From 344e6117ca209febcdb2db6bca6bd9e474cb9beb Mon Sep 17 00:00:00 2001 From: j3ssie Date: Wed, 21 Jan 2026 17:20:52 +0800 Subject: [PATCH] feat: enhance target handling and improve release/install workflows - Add file target support to heuristics with automatic TargetSpace derivation and comprehensive unit tests for file path parsing - Display target space folder location during workflow execution for both module and flow runs - Fix IP address handling in URL parsing to correctly identify and extract root domain for IP targets - Add goreleaser --mark-latest flag to manual release workflow for consistent release tagging - Improve install binary help text and add progress feedback for silent mode installations - Replace deprecated --list-registry-binaries with --list-registry-direct-fetch and --list-registry-nix-build examples --- .github/workflows/manual-release.yaml | 2 +- internal/executor/executor.go | 18 +++++ internal/heuristics/heuristics.go | 31 ++++++++ internal/heuristics/heuristics_test.go | 97 ++++++++++++++++++++++++++ internal/heuristics/url.go | 24 ++++--- pkg/cli/install.go | 9 ++- 6 files changed, 170 insertions(+), 11 deletions(-) create mode 100644 internal/heuristics/heuristics_test.go diff --git a/.github/workflows/manual-release.yaml b/.github/workflows/manual-release.yaml index a73953c..753dfa4 100644 --- a/.github/workflows/manual-release.yaml +++ b/.github/workflows/manual-release.yaml @@ -44,7 +44,7 @@ jobs: with: distribution: goreleaser version: '~> v2' - args: release --clean --skip=validate + args: release --clean --skip=validate --mark-latest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GORELEASER_CURRENT_TAG: ${{ steps.get-tag.outputs.tag }} diff --git a/internal/executor/executor.go b/internal/executor/executor.go index eb21e52..56f3c93 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -859,6 +859,15 @@ func (e *Executor) ExecuteModule(ctx context.Context, module *core.Workflow, par e.printer.WorkflowInfo(module.Name, module.Description, module.Tags, string(module.Runner), len(module.Steps)) } + // Show target space folder location + if !e.dryRun && e.progressBar == nil { + if targetSpace, ok := execCtx.GetVariable("TargetSpace"); ok { + if tsStr, ok := targetSpace.(string); ok && tsStr != "" { + e.printer.Info("Reserving target space folder at: %s", terminal.Cyan(tsStr)) + } + } + } + // Execute steps e.logger.Debug("Starting step execution loop", zap.Int("total_steps", len(module.Steps)), @@ -1414,6 +1423,15 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params printDryRunHeader(flow.Name, string(core.KindFlow), params["target"], tactic, len(flow.Modules), execCtx) } + // Show target space folder location + if !e.dryRun && e.progressBar == nil { + if targetSpace, ok := execCtx.GetVariable("TargetSpace"); ok { + if tsStr, ok := targetSpace.(string); ok && tsStr != "" { + e.printer.Info("Reserving target space folder at: %s", terminal.Cyan(tsStr)) + } + } + } + // Export flow workflow state (write workflow YAML to output) if !e.disableWorkflowState && !e.dryRun { if stateWorkflowFile, ok := execCtx.GetVariable("StateWorkflowFile"); ok { diff --git a/internal/heuristics/heuristics.go b/internal/heuristics/heuristics.go index 2273569..27176d4 100644 --- a/internal/heuristics/heuristics.go +++ b/internal/heuristics/heuristics.go @@ -3,6 +3,8 @@ package heuristics import ( "net" "net/url" + "os" + "path/filepath" "regexp" "strings" ) @@ -14,6 +16,7 @@ const ( TargetTypeURL TargetType = "url" TargetTypeDomain TargetType = "domain" TargetTypeIP TargetType = "ip" + TargetTypeFile TargetType = "file" TargetTypeUnknown TargetType = "unknown" ) @@ -93,6 +96,12 @@ func Analyze(target string, level string) (*TargetInfo, error) { RootDomain: target, } + case TargetTypeFile: + info, err = ParseFileTarget(target) + if err != nil { + return nil, err + } + default: info = &TargetInfo{ Type: TargetTypeUnknown, @@ -107,6 +116,11 @@ func Analyze(target string, level string) (*TargetInfo, error) { func DetectType(target string) TargetType { target = strings.TrimSpace(target) + // Check if it's a file path (exists on disk) - must check before URL + if info, err := os.Stat(target); err == nil && !info.IsDir() { + return TargetTypeFile + } + // Check if it's a URL (has scheme) if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") { return TargetTypeURL @@ -149,3 +163,20 @@ func isValidDomain(s string) bool { domainRegex := regexp.MustCompile(`^([a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$`) return domainRegex.MatchString(s) } + +// ParseFileTarget extracts TargetInfo from a file path +// Derives RootDomain as: basename without extension, _ replaced with -, "-file" suffix +func ParseFileTarget(filePath string) (*TargetInfo, error) { + base := filepath.Base(filePath) + ext := filepath.Ext(base) + nameWithoutExt := strings.TrimSuffix(base, ext) + // Replace _ with - for path friendliness + nameWithoutExt = strings.ReplaceAll(nameWithoutExt, "_", "-") + targetSpace := nameWithoutExt + "-file" + + return &TargetInfo{ + Type: TargetTypeFile, + Original: filePath, + RootDomain: targetSpace, + }, nil +} diff --git a/internal/heuristics/heuristics_test.go b/internal/heuristics/heuristics_test.go new file mode 100644 index 0000000..d94529a --- /dev/null +++ b/internal/heuristics/heuristics_test.go @@ -0,0 +1,97 @@ +package heuristics + +import ( + "os" + "testing" +) + +func TestDetectType_File(t *testing.T) { + // Create temp file + f, err := os.CreateTemp("", "test-file*.txt") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + f.Close() + + got := DetectType(f.Name()) + if got != TargetTypeFile { + t.Errorf("DetectType() = %v, want %v", got, TargetTypeFile) + } +} + +func TestDetectType_DomainNotFile(t *testing.T) { + got := DetectType("example.com") + if got != TargetTypeDomain { + t.Errorf("DetectType() = %v, want %v", got, TargetTypeDomain) + } +} + +func TestDetectType_URL(t *testing.T) { + got := DetectType("https://example.com/path") + if got != TargetTypeURL { + t.Errorf("DetectType() = %v, want %v", got, TargetTypeURL) + } +} + +func TestParseFileTarget(t *testing.T) { + tests := []struct { + name string + filePath string + wantRoot string + }{ + { + name: "simple filename", + filePath: "/tmp/urls-input.txt", + wantRoot: "urls-input-file", + }, + { + name: "underscore replacement", + filePath: "/tmp/my_target_list.txt", + wantRoot: "my-target-list-file", + }, + { + name: "multiple underscores", + filePath: "/path/to/some_long_file_name.csv", + wantRoot: "some-long-file-name-file", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info, err := ParseFileTarget(tt.filePath) + if err != nil { + t.Errorf("ParseFileTarget() error = %v", err) + return + } + if info.RootDomain != tt.wantRoot { + t.Errorf("ParseFileTarget() RootDomain = %v, want %v", info.RootDomain, tt.wantRoot) + } + if info.Type != TargetTypeFile { + t.Errorf("ParseFileTarget() Type = %v, want %v", info.Type, TargetTypeFile) + } + if info.Original != tt.filePath { + t.Errorf("ParseFileTarget() Original = %v, want %v", info.Original, tt.filePath) + } + }) + } +} + +func TestAnalyze_FileTarget(t *testing.T) { + // Create temp file + f, err := os.CreateTemp("", "test_analysis*.txt") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + f.Close() + + info, err := Analyze(f.Name(), "basic") + if err != nil { + t.Errorf("Analyze() error = %v", err) + return + } + if info.Type != TargetTypeFile { + t.Errorf("Analyze() Type = %v, want %v", info.Type, TargetTypeFile) + } +} diff --git a/internal/heuristics/url.go b/internal/heuristics/url.go index 823e7c3..8b98104 100644 --- a/internal/heuristics/url.go +++ b/internal/heuristics/url.go @@ -2,6 +2,7 @@ package heuristics import ( "crypto/tls" + "net" "net/http" "net/url" "path" @@ -45,15 +46,22 @@ func ParseURL(rawURL string) (*TargetInfo, error) { info.Port = port info.Hostname = u.Host // includes port if specified - // Extract root domain using publicsuffix - rootDomain, err := publicsuffix.EffectiveTLDPlusOne(host) - if err != nil { - // Fallback: use the last two parts of the domain - rootDomain = extractRootDomainFallback(host) + // Check if host is an IP address + if net.ParseIP(host) != nil { + info.RootDomain = host + info.TLD = "" + info.SLD = "" + } else { + // Extract root domain using publicsuffix + rootDomain, err := publicsuffix.EffectiveTLDPlusOne(host) + if err != nil { + // Fallback: use the last two parts of the domain + rootDomain = extractRootDomainFallback(host) + } + info.RootDomain = rootDomain + info.TLD, _ = publicsuffix.PublicSuffix(host) + info.SLD = extractSLD(info.RootDomain, info.TLD) } - info.RootDomain = rootDomain - info.TLD, _ = publicsuffix.PublicSuffix(host) - info.SLD = extractSLD(info.RootDomain, info.TLD) // Extract path and file urlPath := u.Path diff --git a/pkg/cli/install.go b/pkg/cli/install.go index 4fcbc40..a65c89b 100644 --- a/pkg/cli/install.go +++ b/pkg/cli/install.go @@ -88,7 +88,8 @@ var installBinaryCmd = &cobra.Command{ Short: "Install binary tools from registry", Long: `Install one or more binary tools from the registry. Skips binaries already available in PATH.`, Example: ` # List binaries in registry - osmedeus install binary --list-registry-binaries + osmedeus install binary --list-registry-direct-fetch + osmedeus install binary --list-registry-nix-build # Install specific binaries osmedeus install binary --name nuclei @@ -1466,7 +1467,8 @@ func printInstallBinaryHelp(cmd *cobra.Command) { fmt.Println(terminal.BoldCyan("◆ Examples")) fmt.Printf(" %s\n", terminal.Green("# List available binaries")) - fmt.Printf(" %s\n\n", terminal.Gray("osmedeus install binary --list-registry-binaries")) + fmt.Printf(" %s\n", terminal.Gray("osmedeus install binary --list-registry-direct-fetch")) + fmt.Printf(" %s\n\n", terminal.Gray("osmedeus install binary --list-registry-nix-build")) fmt.Printf(" %s\n", terminal.Green("# Install specific binaries (auto-detects method from registry)")) fmt.Printf(" %s\n", terminal.Gray("osmedeus install binary --name nuclei")) @@ -1743,6 +1745,9 @@ func installBinariesParallel(names []string, registry installer.BinaryRegistry, go func() { defer wg.Done() for name := range workCh { + if silent { + fmt.Printf(" %s Installing: %s\n", terminal.SymbolBullet, terminal.Cyan(name)) + } err := installer.InstallBinary(name, registry, binariesFolder, headers) statusMu.Lock() if err != nil {