diff --git a/internal/executor/executor.go b/internal/executor/executor.go index ee212b8..66a8e6d 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -568,6 +568,16 @@ func isModuleExcluded(moduleName string, excludeList []string) bool { return false } +// isFuzzyModuleExcluded checks if a module name contains any of the fuzzy exclude patterns +func isFuzzyModuleExcluded(moduleName string, fuzzyList []string) bool { + for _, pattern := range fuzzyList { + if strings.Contains(moduleName, pattern) { + return true + } + } + return false +} + // formatDuration formats a duration in human-readable format func formatDuration(d time.Duration) string { if d < time.Second { @@ -1233,6 +1243,25 @@ func (e *Executor) ExecuteModule(ctx context.Context, module *core.Workflow, par metrics.RecordStepDuration(string(step.Type), string(stepResult.Status), stepResult.Duration.Seconds()) if err != nil { + // Check if this is a skip-module signal + if errors.Is(err, functions.ErrSkipModule) { + msg := "skip() called" + var skipErr *functions.SkipModuleError + if errors.As(err, &skipErr) { + msg = skipErr.Message + } + execCtx.Logger.Info("Module skipped", + zap.String("step", step.Name), + zap.String("message", msg), + ) + result.Status = core.RunStatusSkipped + result.Message = msg + result.Exports = execCtx.Exports + result.EndTime = time.Now() + metrics.RecordWorkflowEnd(module.Name, string(core.KindModule), string(result.Status), result.EndTime.Sub(result.StartTime).Seconds()) + return result, nil + } + execCtx.Logger.Error("Step failed", zap.String("step", step.Name), zap.Error(err), @@ -1475,6 +1504,21 @@ func (e *Executor) executeStepsDAG(ctx context.Context, steps []core.Step, execC } if err != nil { + // Check if this is a skip-module signal + if errors.Is(err, functions.ErrSkipModule) { + if firstError == nil { + firstError = err + } + // Mark all remaining steps as completed to break the main loop + for name := range stepMap { + if !executed[name] { + executed[name] = true + atomic.AddInt32(&completedCount, 1) + } + } + cond.Signal() + return + } failed[sName] = true if firstError == nil && !e.shouldContinueOnError(s) { firstError = err @@ -1502,6 +1546,18 @@ func (e *Executor) executeStepsDAG(ctx context.Context, steps []core.Step, execC result.Steps = collector.Results() if firstError != nil { + // Check if the error was a skip-module signal (not a failure) + if errors.Is(firstError, functions.ErrSkipModule) { + msg := "skip() called" + var skipErr *functions.SkipModuleError + if errors.As(firstError, &skipErr) { + msg = skipErr.Message + } + result.Status = core.RunStatusSkipped + result.Message = msg + result.EndTime = time.Now() + return nil + } result.Status = core.RunStatusFailed result.Error = firstError result.EndTime = time.Now() @@ -1762,6 +1818,7 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params // Parse excluded modules excludeList := parseExcludeList(params["exclude_modules"]) + fuzzyExcludeList := parseExcludeList(params["fuzzy_exclude_modules"]) // Pre-load all modules in parallel for faster startup execCtx.Logger.Debug("Pre-loading modules", zap.Int("count", len(flow.Modules))) @@ -1812,8 +1869,8 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params modRef := moduleMap[modName] - // Check if module is excluded - if isModuleExcluded(modRef.Name, excludeList) { + // Check if module is excluded (exact match or fuzzy substring match) + if isModuleExcluded(modRef.Name, excludeList) || isFuzzyModuleExcluded(modRef.Name, fuzzyExcludeList) { execCtx.Logger.Info("Skipping excluded module", zap.String("module", modRef.Name)) executed[modRef.Name] = true // Unblock dependents even for excluded modules @@ -2110,6 +2167,16 @@ func (e *Executor) executeStep(ctx context.Context, step *core.Step, execCtx *co ) ok, err := e.functionRegistry.EvaluateCondition(renderedCondition, execCtx.GetVariables()) if err != nil { + // Check if skip() was called inside a pre_condition + if errors.Is(err, functions.ErrSkipModule) { + result.Status = core.StepStatusSkipped + result.Error = err + result.EndTime = time.Now() + if e.progressBar == nil { + e.printer.StepSkipped(step.Name) + } + return result, err + } stepLogger.Debug("Pre-condition evaluation failed", zap.Error(err)) result.Status = core.StepStatusFailed result.Error = fmt.Errorf("pre-condition evaluation failed: %w", err) @@ -2243,6 +2310,18 @@ func (e *Executor) executeStep(ctx context.Context, step *core.Step, execCtx *co sp.Stop() } if err != nil { + // Check if this is a skip-module signal (not a failure) + if errors.Is(err, functions.ErrSkipModule) { + result.Status = core.StepStatusSkipped + result.Error = err + result.EndTime = time.Now() + result.Duration = result.EndTime.Sub(result.StartTime) + if e.progressBar == nil { + e.printer.StepSkipped(step.Name) + } + return result, err + } + result.Status = core.StepStatusFailed result.Error = err result.EndTime = time.Now() diff --git a/internal/executor/executor_test.go b/internal/executor/executor_test.go index 58bb368..652757d 100644 --- a/internal/executor/executor_test.go +++ b/internal/executor/executor_test.go @@ -7,6 +7,7 @@ import ( "github.com/j3ssie/osmedeus/v5/internal/config" "github.com/j3ssie/osmedeus/v5/internal/core" + "github.com/j3ssie/osmedeus/v5/internal/parser" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1677,3 +1678,143 @@ func TestExecutor_StepDependencies_FailedDep_SkipsDependent(t *testing.T) { assert.NotEqual(t, core.StepStatusSuccess, stepBExists) } } + +func TestExecutor_SkipModule(t *testing.T) { + ctx := context.Background() + cfg := testConfig(t) + + module := &core.Workflow{ + Name: "test-skip", + Kind: core.KindModule, + Steps: []core.Step{ + { + Name: "step-before", + Type: core.StepTypeFunction, + Function: "log_info('before skip')", + }, + { + Name: "step-skip", + Type: core.StepTypeFunction, + Function: "skip('target not applicable')", + }, + { + Name: "step-after", + Type: core.StepTypeFunction, + Function: "log_info('after skip')", + }, + }, + } + + executor := NewExecutor() + executor.SetDryRun(false) + executor.SetSpinner(false) + + result, err := executor.ExecuteModule(ctx, module, map[string]string{ + "target": "test", + }, cfg) + + // skip() returns nil error (not a failure) + require.NoError(t, err) + assert.Equal(t, core.RunStatusSkipped, result.Status) + assert.Equal(t, "target not applicable", result.Message) + + // step-before should have executed, step-after should NOT + assert.GreaterOrEqual(t, len(result.Steps), 2, "should have at least step-before and step-skip") + + // Find step-after in results - it should not be present + for _, s := range result.Steps { + assert.NotEqual(t, "step-after", s.StepName, "step-after should not have executed") + } +} + +func TestExecutor_SkipModulePreservesExports(t *testing.T) { + ctx := context.Background() + cfg := testConfig(t) + + module := &core.Workflow{ + Name: "test-skip-exports", + Kind: core.KindModule, + Steps: []core.Step{ + { + Name: "set-var", + Type: core.StepTypeFunction, + Function: "set_var('my_key', 'my_value')", + }, + { + Name: "do-skip", + Type: core.StepTypeFunction, + Function: "skip('done early')", + }, + }, + } + + executor := NewExecutor() + executor.SetDryRun(false) + executor.SetSpinner(false) + + result, err := executor.ExecuteModule(ctx, module, map[string]string{ + "target": "test", + }, cfg) + + require.NoError(t, err) + assert.Equal(t, core.RunStatusSkipped, result.Status) + assert.Equal(t, "done early", result.Message) +} + +func TestIsFuzzyModuleExcluded(t *testing.T) { + tests := []struct { + name string + moduleName string + fuzzyList []string + expected bool + }{ + {"exact substring match", "recon-spider", []string{"spider"}, true}, + {"prefix match", "spider-crawl", []string{"spider"}, true}, + {"no match", "recon-dns", []string{"spider"}, false}, + {"empty list", "recon-spider", nil, false}, + {"empty pattern list", "recon-spider", []string{}, false}, + {"multiple patterns first matches", "recon-spider", []string{"spider", "dns"}, true}, + {"multiple patterns second matches", "recon-dns", []string{"spider", "dns"}, true}, + {"multiple patterns none match", "recon-http", []string{"spider", "dns"}, false}, + {"full name as pattern", "recon-spider", []string{"recon-spider"}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isFuzzyModuleExcluded(tt.moduleName, tt.fuzzyList) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestExecutor_FuzzyExcludeModules(t *testing.T) { + ctx := context.Background() + cfg := testConfig(t) + + // Create a flow where all modules will be excluded by fuzzy match + flow := &core.Workflow{ + Name: "test-fuzzy-exclude", + Kind: core.KindFlow, + Modules: []core.ModuleRef{ + {Name: "recon-spider", Path: ""}, + {Name: "spider-crawl", Path: ""}, + }, + } + + loader := parser.NewLoader(cfg.WorkflowsPath) + + exec := NewExecutor() + exec.SetDryRun(true) + exec.SetSpinner(false) + exec.SetLoader(loader) + + // fuzzy_exclude_modules=spider should skip both recon-spider and spider-crawl + result, err := exec.ExecuteFlow(ctx, flow, map[string]string{ + "target": "test.example.com", + "fuzzy_exclude_modules": "spider", + }, cfg) + + require.NoError(t, err) + assert.NotNil(t, result) + assert.Equal(t, core.RunStatusCompleted, result.Status) +} diff --git a/internal/executor/function_executor.go b/internal/executor/function_executor.go index 362f375..f80970a 100644 --- a/internal/executor/function_executor.go +++ b/internal/executor/function_executor.go @@ -2,6 +2,7 @@ package executor import ( "context" + "errors" "fmt" "sync" "time" @@ -68,6 +69,11 @@ func (e *FunctionExecutor) Execute(ctx context.Context, step *core.Step, execCtx result.Duration = result.EndTime.Sub(result.StartTime) if err != nil { + if errors.Is(err, functions.ErrSkipModule) { + result.Status = core.StepStatusSkipped + result.Error = err + return result, err + } result.Status = core.StepStatusFailed result.Error = err return result, err diff --git a/internal/functions/constants.go b/internal/functions/constants.go index a633cc8..6f4cf27 100644 --- a/internal/functions/constants.go +++ b/internal/functions/constants.go @@ -1,5 +1,29 @@ package functions +import ( + "errors" + "fmt" +) + +// ErrSkipModule is a sentinel error used by skip() to signal that the +// remaining steps in the current module should be skipped. The flow +// continues to the next module. +var ErrSkipModule = errors.New("skip module") + +// SkipModuleError carries an optional message and wraps ErrSkipModule +// so that errors.Is(err, ErrSkipModule) works through the chain. +type SkipModuleError struct { + Message string +} + +func (e *SkipModuleError) Error() string { + return fmt.Sprintf("skip module: %s", e.Message) +} + +func (e *SkipModuleError) Unwrap() error { + return ErrSkipModule +} + // Function name constants for easy reference and consistency // This file serves as a central reference for all available workflow functions @@ -79,6 +103,7 @@ const ( FnPrintf = "printf" // printf(message) -> void (print message to stdout) FnCatFile = "cat_file" // cat_file(path) -> void (print file content to stdout) FnExit = "exit" // exit(code) -> void (exit scan with code) + FnSkip = "skip" // skip(message?) -> void (skip remaining steps in current module) FnExecCmd = "exec_cmd" // exec_cmd(command) -> string (alias for bash) FnBash = "bash" FnSleep = "sleep" // sleep(seconds) -> void (pause for n seconds) @@ -391,6 +416,7 @@ func AllFunctions() []string { FnPrintf, FnCatFile, FnExit, + FnSkip, FnExecCmd, FnBash, FnSleep, @@ -727,6 +753,7 @@ func FunctionRegistry() map[string][]FunctionInfo { {FnPrintf, "printf(message)", "Print message to stdout", "void", "printf('Scan started')"}, {FnCatFile, "cat_file(path)", "Print file content to stdout", "void", "cat_file('{{Output}}/results.txt')"}, {FnExit, "exit(code)", "Exit scan with code", "void", "exit(1)"}, + {FnSkip, "skip(message?)", "Skip remaining steps in current module and continue to next module", "void", "skip('target not applicable')"}, {FnBash, "bash(command)", "Execute bash command and return output", "string", "bash('whoami')"}, {FnExecCmd, "exec_cmd(command)", "Alias for bash(command)", "string", "exec_cmd('whoami')"}, {FnSleep, "sleep(seconds)", "Pause for n seconds", "void", "sleep(5)"}, diff --git a/internal/functions/goja_runtime.go b/internal/functions/goja_runtime.go index fc70295..c165ef3 100644 --- a/internal/functions/goja_runtime.go +++ b/internal/functions/goja_runtime.go @@ -112,6 +112,7 @@ func (r *GojaRuntime) registerFunctionsOnVM(vm *goja.Runtime) { _ = vm.Set(FnPrintf, vf.printf) _ = vm.Set(FnCatFile, vf.catFile) _ = vm.Set(FnExit, vf.exit) + _ = vm.Set(FnSkip, vf.skip) _ = vm.Set(FnExecCmd, vf.execCmd) _ = vm.Set(FnBash, vf.bash) _ = vm.Set(FnSleep, vf.sleep) diff --git a/internal/functions/util_functions.go b/internal/functions/util_functions.go index 343fc53..70ce132 100644 --- a/internal/functions/util_functions.go +++ b/internal/functions/util_functions.go @@ -708,6 +708,18 @@ func (vf *vmFunc) exit(call goja.FunctionCall) goja.Value { return goja.Undefined() } +// skip stops execution of the remaining steps in the current module. +// The flow continues to the next module. Accepts an optional message. +// Usage: skip(message?) -> void +func (vf *vmFunc) skip(call goja.FunctionCall) goja.Value { + msg := call.Argument(0).String() + if msg == "undefined" || msg == "" { + msg = "skip() called" + } + logger.Get().Info("Module skip requested: " + msg) + panic(vf.vm.NewGoError(&SkipModuleError{Message: msg})) +} + // execCmd executes a bash command and returns the stdout output // Usage: exec_cmd(command) -> string func (vf *vmFunc) execCmd(call goja.FunctionCall) goja.Value { diff --git a/internal/functions/util_functions_test.go b/internal/functions/util_functions_test.go index 56a89d8..18913d9 100644 --- a/internal/functions/util_functions_test.go +++ b/internal/functions/util_functions_test.go @@ -2,6 +2,7 @@ package functions import ( "bytes" + "errors" "io" "os" "path/filepath" @@ -912,3 +913,49 @@ func TestMoveFile(t *testing.T) { assert.True(t, info.Mode()&0100 != 0, "expected executable permission to be preserved") }) } + +func TestSkip(t *testing.T) { + runtime := NewOttoRuntime() + + t.Run("skip without message returns ErrSkipModule", func(t *testing.T) { + _, err := runtime.Execute(`skip()`, nil) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrSkipModule), "error should wrap ErrSkipModule") + }) + + t.Run("skip with message preserves message", func(t *testing.T) { + _, err := runtime.Execute(`skip('target not applicable')`, nil) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrSkipModule), "error should wrap ErrSkipModule") + var skipErr *SkipModuleError + assert.True(t, errors.As(err, &skipErr), "error should be extractable as SkipModuleError") + assert.Equal(t, "target not applicable", skipErr.Message) + }) + + t.Run("skip default message", func(t *testing.T) { + _, err := runtime.Execute(`skip()`, nil) + require.Error(t, err) + var skipErr *SkipModuleError + if errors.As(err, &skipErr) { + assert.Equal(t, "skip() called", skipErr.Message) + } + }) +} + +func TestSkipModuleError(t *testing.T) { + t.Run("Error returns formatted message", func(t *testing.T) { + err := &SkipModuleError{Message: "test message"} + assert.Equal(t, "skip module: test message", err.Error()) + }) + + t.Run("Unwrap returns ErrSkipModule", func(t *testing.T) { + err := &SkipModuleError{Message: "test"} + assert.Equal(t, ErrSkipModule, err.Unwrap()) + }) + + t.Run("errors.Is works through chain", func(t *testing.T) { + skipErr := &SkipModuleError{Message: "inner"} + wrapped := errors.Join(errors.New("outer"), skipErr) + assert.True(t, errors.Is(wrapped, ErrSkipModule)) + }) +} diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index b83c772..1bf62af 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -127,8 +127,8 @@ func ImportWorkspace(source, workspacesPath string, skipDB bool, cfg *config.Con } // Extract the zip into the workspaces root — zip entries already contain - // the workspace directory prefix (e.g. "shopee.vn/file.txt"), so extracting - // into workspacesPath produces workspacesPath/shopee.vn/file.txt. + // the workspace directory prefix (e.g. "example.com/file.txt"), so extracting + // into workspacesPath produces workspacesPath/example.com/file.txt. filesCount, err := extractZip(zipPath, workspacesPath) if err != nil { return nil, fmt.Errorf("failed to extract archive: %w", err) @@ -222,8 +222,8 @@ func ForceImportWorkspace(source, workspacesPath string, skipDB bool, cfg *confi } // Extract the zip into the workspaces root — zip entries already contain - // the workspace directory prefix (e.g. "shopee.vn/file.txt"), so extracting - // into workspacesPath produces workspacesPath/shopee.vn/file.txt. + // the workspace directory prefix (e.g. "example.com/file.txt"), so extracting + // into workspacesPath produces workspacesPath/example.com/file.txt. filesCount, err := extractZip(zipPath, workspacesPath) if err != nil { return nil, fmt.Errorf("failed to extract archive: %w", err) diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index 9445bfa..3211542 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -217,7 +217,7 @@ func TestExtractZip(t *testing.T) { t.Run("extracts zip with directory prefix without double nesting", func(t *testing.T) { // Simulate what createHighCompressionZip produces: entries prefixed with workspace name - // e.g. "shopee.vn/output.txt", "shopee.vn/subdir/nested.txt" + // e.g. "example.com/output.txt", "example.com/subdir/nested.txt" zipPath := filepath.Join(os.TempDir(), "test-extract-prefix.zip") defer func() { _ = os.Remove(zipPath) }() @@ -227,24 +227,24 @@ func TestExtractZip(t *testing.T) { writer := zip.NewWriter(zipFile) // Directory entry with proper permissions - dirHeader := &zip.FileHeader{Name: "shopee.vn/"} + dirHeader := &zip.FileHeader{Name: "example.com/"} dirHeader.SetMode(0755) _, err = writer.CreateHeader(dirHeader) require.NoError(t, err) // File inside the directory - f, err := writer.Create("shopee.vn/output.txt") + f, err := writer.Create("example.com/output.txt") require.NoError(t, err) _, err = f.Write([]byte("scan results")) require.NoError(t, err) // Nested subdirectory with proper permissions - subDirHeader := &zip.FileHeader{Name: "shopee.vn/subdir/"} + subDirHeader := &zip.FileHeader{Name: "example.com/subdir/"} subDirHeader.SetMode(0755) _, err = writer.CreateHeader(subDirHeader) require.NoError(t, err) - f, err = writer.Create("shopee.vn/subdir/nested.txt") + f, err = writer.Create("example.com/subdir/nested.txt") require.NoError(t, err) _, err = f.Write([]byte("nested content")) require.NoError(t, err) @@ -261,18 +261,18 @@ func TestExtractZip(t *testing.T) { require.NoError(t, err) assert.Equal(t, 2, filesCount) // 2 files (directories are not counted) - // Verify correct structure: workspacesDir/shopee.vn/output.txt (NOT workspacesDir/shopee.vn/shopee.vn/output.txt) - content, err := os.ReadFile(filepath.Join(workspacesDir, "shopee.vn", "output.txt")) + // Verify correct structure: workspacesDir/example.com/output.txt (NOT workspacesDir/example.com/example.com/output.txt) + content, err := os.ReadFile(filepath.Join(workspacesDir, "example.com", "output.txt")) require.NoError(t, err) assert.Equal(t, "scan results", string(content)) - content, err = os.ReadFile(filepath.Join(workspacesDir, "shopee.vn", "subdir", "nested.txt")) + content, err = os.ReadFile(filepath.Join(workspacesDir, "example.com", "subdir", "nested.txt")) require.NoError(t, err) assert.Equal(t, "nested content", string(content)) // Verify NO double nesting - _, err = os.Stat(filepath.Join(workspacesDir, "shopee.vn", "shopee.vn")) - assert.True(t, os.IsNotExist(err), "should not have double-nested shopee.vn/shopee.vn directory") + _, err = os.Stat(filepath.Join(workspacesDir, "example.com", "example.com")) + assert.True(t, os.IsNotExist(err), "should not have double-nested example.com/example.com directory") }) t.Run("fails with non-existent zip", func(t *testing.T) { diff --git a/pkg/cli/run.go b/pkg/cli/run.go index 01ed764..ee80edb 100644 --- a/pkg/cli/run.go +++ b/pkg/cli/run.go @@ -49,6 +49,7 @@ var ( threadsHold int runTactic string excludeModules []string + fuzzyExcludeModules []string spaceName string workspacesFolder string heuristicsCheck string @@ -108,6 +109,7 @@ func init() { runCmd.Flags().IntVarP(&concurrency, "concurrency", "c", 1, "number of targets to run concurrently") runCmd.Flags().StringVarP(&runTactic, "tactic", "B", "default", "run tactic: aggressive, default, gently") runCmd.Flags().StringArrayVarP(&excludeModules, "exclude", "x", nil, "module(s) to exclude from execution (can be specified multiple times)") + runCmd.Flags().StringArrayVarP(&fuzzyExcludeModules, "fuzzy-exclude", "X", nil, "exclude modules whose name contains the given substring (can be specified multiple times)") runCmd.Flags().StringVarP(&spaceName, "space", "S", "", "override {{TargetSpace}} variable") runCmd.Flags().StringVarP(&workspacesFolder, "workspaces-folder", "W", "", "override {{Workspaces}} variable") runCmd.Flags().StringVar(&heuristicsCheck, "heuristics-check", "basic", "heuristics check level: none, basic, advanced") @@ -788,6 +790,7 @@ func executeRunForTargetWithContext(ctx context.Context, workflow *core.Workflow params["tactic"] = runTactic params["threads_hold"] = fmt.Sprintf("%d", threadsHold) params["exclude_modules"] = strings.Join(excludeModules, ",") + params["fuzzy_exclude_modules"] = strings.Join(fuzzyExcludeModules, ",") params["space_name"] = spaceName params["workspaces_folder"] = workspacesFolder params["heuristics_check"] = heuristicsCheck @@ -806,6 +809,7 @@ func executeRunForTargetWithContext(ctx context.Context, workflow *core.Workflow zap.String("tactic", runTactic), zap.Int("threads_hold", threadsHold), zap.Strings("exclude_modules", excludeModules), + zap.Strings("fuzzy_exclude_modules", fuzzyExcludeModules), zap.Int("param_count", len(params)), ) diff --git a/pkg/cli/scan.go b/pkg/cli/scan.go index 14bb361..0867b45 100644 --- a/pkg/cli/scan.go +++ b/pkg/cli/scan.go @@ -40,6 +40,7 @@ func init() { scanCmd.Flags().IntVarP(&concurrency, "concurrency", "c", 1, "number of targets to run concurrently") scanCmd.Flags().StringVarP(&runTactic, "tactic", "B", "default", "run tactic: aggressive, default, gently") scanCmd.Flags().StringArrayVarP(&excludeModules, "exclude", "x", nil, "module(s) to exclude from execution (can be specified multiple times)") + scanCmd.Flags().StringArrayVarP(&fuzzyExcludeModules, "fuzzy-exclude", "X", nil, "exclude modules whose name contains the given substring (can be specified multiple times)") scanCmd.Flags().StringVarP(&spaceName, "space", "S", "", "override {{TargetSpace}} variable") scanCmd.Flags().StringVarP(&workspacesFolder, "workspaces-folder", "W", "", "override {{Workspaces}} variable") scanCmd.Flags().StringVar(&heuristicsCheck, "heuristics-check", "basic", "heuristics check level: none, basic, advanced") diff --git a/test/testdata/sample-jsonl-output/cusom-content-discovery.jsonl b/test/testdata/sample-jsonl-output/cusom-content-discovery.jsonl new file mode 100644 index 0000000..e9a8c30 --- /dev/null +++ b/test/testdata/sample-jsonl-output/cusom-content-discovery.jsonl @@ -0,0 +1,9 @@ +{"timestamp":"2026-02-13T15:21:24+07:00","url":"http://cdn.doitac.example.com/common/","input":"http://cdn.doitac.example.com/common/","scheme":"http","host":"cdn.doitac.example.com","port":"80","path":"/common/","method":"GET","status_code":200,"content_length":0,"content_type":"text/plain","webserver":"UploadServer","header":{"Accept-Ranges":"bytes","Access-Control-Allow-Origin":"*","Access-Control-Expose-Headers":"*","Age":"0","Cache-Control":"public,max-age=3600","Content-Length":"0","Content-Type":"text/plain","Date":"Fri, 13 Feb 2026 08:21:22 GMT","Etag":"\"d41d8cd98f00b204e9800998ecf8427e\"","Last-Modified":"Fri, 27 Aug 2021 12:16:34 GMT","Server":"UploadServer","X-Goog-Generation":"1630066594545907","X-Goog-Hash":"crc32c=AAAAAA==","X-Goog-Metageneration":"1","X-Goog-Storage-Class":"STANDARD","X-Goog-Stored-Content-Encoding":"identity","X-Goog-Stored-Content-Length":"0","X-Guploader-Uploadid":"AJRbA5WZX4sT7V-H95LeV0UP-rnC9J4JB9rKdyGa8IIc8uTiafoZfrbMghIxr7kCtzlg4ZiU"},"words":0,"lines":0,"found_by":"short-dir","depth":1,"type":"directory"} +{"timestamp":"2026-02-13T15:21:32+07:00","url":"http://pciidk.example.com/monitor","input":"http://pciidk.example.com/monitor","scheme":"http","host":"pciidk.example.com","port":"80","path":"/monitor","method":"GET","status_code":200,"content_length":18,"content_type":"text/plain","webserver":"Kestrel","header":{"Cache-Control":"no-cache","Content-Type":"text/plain; charset=utf-8","Date":"Fri, 13 Feb 2026 08:21:30 GMT","Expires":"0","Pragma":"no-cache","Server":"Kestrel","Strict-Transport-Security":"max-age=31536000; preload;","Vary":"Accept-Encoding"},"words":1,"lines":1,"found_by":"long-file-no-ext","depth":1,"type":"file"} +{"timestamp":"2026-02-13T15:22:50+07:00","url":"https://cdn.doitac.example.com/common/","input":"https://cdn.doitac.example.com/common/","scheme":"https","host":"cdn.doitac.example.com","port":"443","path":"/common/","method":"GET","status_code":200,"content_length":0,"content_type":"text/plain","webserver":"UploadServer","header":{"Accept-Ranges":"bytes","Access-Control-Allow-Origin":"*","Access-Control-Expose-Headers":"*","Age":"86","Alt-Svc":"h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000","Cache-Control":"public,max-age=3600","Content-Length":"0","Content-Type":"text/plain","Date":"Fri, 13 Feb 2026 08:21:22 GMT","Etag":"\"d41d8cd98f00b204e9800998ecf8427e\"","Last-Modified":"Fri, 27 Aug 2021 12:16:34 GMT","Server":"UploadServer","X-Goog-Generation":"1630066594545907","X-Goog-Hash":"crc32c=AAAAAA==","X-Goog-Metageneration":"1","X-Goog-Storage-Class":"STANDARD","X-Goog-Stored-Content-Encoding":"identity","X-Goog-Stored-Content-Length":"0","X-Guploader-Uploadid":"AJRbA5WZX4sT7V-H95LeV0UP-rnC9J4JB9rKdyGa8IIc8uTiafoZfrbMghIxr7kCtzlg4ZiU"},"words":0,"lines":0,"found_by":"short-dir","depth":1,"type":"directory"} +{"timestamp":"2026-02-13T15:23:35+07:00","url":"https://openapi.livestream.example.com/live","input":"https://openapi.livestream.example.com/live","scheme":"https","host":"openapi.livestream.example.com","port":"443","path":"/live","method":"GET","status_code":301,"content_length":40,"content_type":"text/html","location":"/live","webserver":"SGW","header":{"Connection":"keep-alive","Content-Length":"40","Content-Type":"text/html; charset=utf-8","Date":"Fri, 13 Feb 2026 08:23:35 GMT","Location":"/live","Server":"SGW"},"words":3,"lines":3,"found_by":"redirect","depth":1,"type":"file"} +{"timestamp":"2026-02-13T15:23:36+07:00","url":"https://openapi.livestream.example.com/live/","input":"https://openapi.livestream.example.com/live/","scheme":"https","host":"openapi.livestream.example.com","port":"443","path":"/live/","method":"GET","status_code":301,"content_length":40,"content_type":"text/html","location":"/live","webserver":"SGW","header":{"Connection":"keep-alive","Content-Length":"40","Content-Type":"text/html; charset=utf-8","Date":"Fri, 13 Feb 2026 08:23:35 GMT","Location":"/live","Server":"SGW"},"words":3,"lines":3,"found_by":"long-dir","depth":1,"type":"directory"} +{"timestamp":"2026-02-13T15:24:15+07:00","url":"http://e-procurement.example.com/static","input":"http://e-procurement.example.com/static","scheme":"http","host":"e-procurement.example.com","port":"80","path":"/static","method":"GET","status_code":200,"content_length":16640,"content_type":"text/html","webserver":"SGW","header":{"Cache-Control":"no-cache,no-store","Connection":"keep-alive","Content-Type":"text/html; charset=utf-8","Date":"Fri, 13 Feb 2026 08:24:15 GMT","Etag":"W/\"8a12d89b7752b67e85954dd17b51bef1\"","Last-Modified":"Wed, 28 Jan 2026 12:42:02 GMT","Originalmd5":"8a12d89b7752b67e85954dd17b51bef1","Server":"SGW","Vary":"Accept-Encoding","X-Cache-Status":"STALE","X-Cdn":"staticcache","X-Origin":"uss","X-Ratelimit-Limit":"1250","X-Ratelimit-Remaining":"1236","X-Request-Id":"36119c1c-70c1-4b89-8f9b-6619540a55ef","X-Uri":"/shopee-scs-live-sg/static/index.html","X-Via":"52.199"},"words":949,"lines":106,"found_by":"observed-no-ext","depth":1,"type":"file","remarks":["Modern-App"]} +{"timestamp":"2026-02-13T15:24:16+07:00","url":"http://e-procurement.example.com/static/","input":"http://e-procurement.example.com/static/","scheme":"http","host":"e-procurement.example.com","port":"80","path":"/static/","method":"GET","status_code":200,"content_length":16640,"content_type":"text/html","webserver":"SGW","header":{"Cache-Control":"no-cache,no-store","Connection":"keep-alive","Content-Type":"text/html; charset=utf-8","Date":"Fri, 13 Feb 2026 08:24:15 GMT","Etag":"W/\"8a12d89b7752b67e85954dd17b51bef1\"","Last-Modified":"Wed, 28 Jan 2026 12:42:02 GMT","Originalmd5":"8a12d89b7752b67e85954dd17b51bef1","Server":"SGW","Vary":"Accept-Encoding","X-Cache-Status":"HIT","X-Cdn":"staticcache","X-Origin":"uss","X-Ratelimit-Limit":"1250","X-Ratelimit-Remaining":"1236","X-Request-Id":"36119c1c-70c1-4b89-8f9b-6619540a55ef","X-Uri":"/shopee-scs-live-sg/static/index.html","X-Via":"52.199"},"words":949,"lines":106,"found_by":"observed-dir","depth":1,"type":"directory","remarks":["Modern-App"]} +{"timestamp":"2026-02-13T15:24:32+07:00","url":"http://e-procurement.example.com/oauth","input":"http://e-procurement.example.com/oauth","scheme":"http","host":"e-procurement.example.com","port":"80","path":"/oauth","method":"GET","status_code":200,"content_length":4713,"content_type":"text/html","title":"SCS Open Platform Authorization","webserver":"SGW","header":{"Cache-Control":"no-cache,no-store","Connection":"keep-alive","Content-Type":"text/html; charset=utf-8","Date":"Fri, 13 Feb 2026 08:24:32 GMT","Etag":"W/\"43e9aea279110e2f8a086697c0da1c14\"","Last-Modified":"Wed, 28 Jan 2026 12:42:01 GMT","Originalmd5":"43e9aea279110e2f8a086697c0da1c14","Server":"SGW","Vary":"Accept-Encoding","X-Cache-Status":"STALE","X-Cdn":"staticcache","X-Origin":"uss","X-Ratelimit-Limit":"1250","X-Ratelimit-Remaining":"1219","X-Request-Id":"bd43f4c6-38c1-4221-996a-8cac52602be7","X-Uri":"/shopee-scs-live-sg/oauth/index.html","X-Via":"44.5"},"words":118,"lines":1,"found_by":"long-file-no-ext","depth":1,"type":"file","remarks":["Modern-App"]} +{"timestamp":"2026-02-13T15:24:45+07:00","url":"http://e-procurement.example.com/oauth/","input":"http://e-procurement.example.com/oauth/","scheme":"http","host":"e-procurement.example.com","port":"80","path":"/oauth/","method":"GET","status_code":200,"content_length":4713,"content_type":"text/html","title":"SCS Open Platform Authorization","webserver":"SGW","header":{"Cache-Control":"no-cache,no-store","Connection":"keep-alive","Content-Type":"text/html; charset=utf-8","Date":"Fri, 13 Feb 2026 08:24:44 GMT","Etag":"W/\"43e9aea279110e2f8a086697c0da1c14\"","Last-Modified":"Wed, 28 Jan 2026 12:42:01 GMT","Originalmd5":"43e9aea279110e2f8a086697c0da1c14","Server":"SGW","Vary":"Accept-Encoding","X-Cache-Status":"STALE","X-Cdn":"staticcache","X-Origin":"uss","X-Ratelimit-Limit":"1250","X-Ratelimit-Remaining":"1236","X-Request-Id":"2afbd034-956c-4cfc-bc71-41a3bf90e425","X-Uri":"/shopee-scs-live-sg/oauth/index.html","X-Via":"52.199"},"words":118,"lines":1,"found_by":"long-dir","depth":1,"type":"directory","remarks":["Modern-App"]} \ No newline at end of file