From 9ed02e7eeeeaf86ffb0ff08e20a2123cd3618376 Mon Sep 17 00:00:00 2001 From: j3ssie Date: Mon, 19 Jan 2026 01:17:04 +0800 Subject: [PATCH] refactor: code quality improvements, Docker updates, and setup enhancements - chore: fix code style and remove unused error handling (add blank checks for closed file handles) - chore: align struct field padding in multiple files for consistency - fix: add 386 architecture support to goreleaser build config - feat: add docker-publish target to Makefile for publishing to Docker Hub - feat: refactor first-time setup into reusable installRequiredBinaries helper function - feat: add initialization marker tracking for first-time setup completion - feat: enhance workflow YAML detection to skip non-workflow files and hidden directories - feat: improve database column display defaults (assets and vulnerabilities tables) - feat: add fallback mechanism to install.sh for version detection failures - fix: correct tarball filename generation by stripping 'v' prefix - chore: update Docker base image from golang:1.22 to golang:1.25 - chore: update goreleaser release flags and simplify Docker build naming - chore: fix import ordering across multiple files (alphabetical consistency) - chore: improve install script with better version display formatting - chore: reduce binaries per row from 10 to 6 in CLI output for better readability --- .goreleaser.yaml | 4 + Makefile | 17 +- README.md | 5 +- build/DEPLOYMENT.md | 3 +- build/docker/Dockerfile | 4 +- internal/console/capture.go | 18 +- internal/executor/executor.go | 4 +- internal/functions/constants.go | 22 +- internal/functions/goja_runtime.go | 4 +- internal/functions/url_functions_test.go | 20 +- internal/installer/nix.go | 2 +- internal/parser/loader.go | 41 ++- internal/state/types.go | 22 +- internal/storage/storage.go | 2 +- internal/updater/mock_source.go | 12 +- lib/options.go | 1 - pkg/cli/db.go | 3 +- pkg/cli/function.go | 1 - pkg/cli/health.go | 10 +- pkg/cli/install.go | 75 +++-- pkg/cli/root.go | 291 ++++++++++-------- pkg/cli/run.go | 48 +-- pkg/cli/workflow.go | 5 +- pkg/server/server.go | 2 +- public/mics/install.sh | 52 ++-- .../registry-metadata-direct-fetch.json | 2 +- 26 files changed, 406 insertions(+), 264 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 494b4eb..2b6a39d 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -19,6 +19,10 @@ builds: goarch: - amd64 - arm64 + - 386 + ignore: + - goos: darwin + goarch: 386 ldflags: - -s -w - -X main.BuildTime={{.Date}} diff --git a/Makefile b/Makefile index 97ba32c..66996a5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build run test test-unit test-integration test-workflow-integration test-e2e test-e2e-verbose test-e2e-ssh test-e2e-api test-e2e-nix test-e2e-install test-docker test-ssh test-distributed test-all test-summary test-ci clean install-gotestsum lint fmt db-seed db-clean db-migrate run-server-debug swagger update-ui snapshot-release github-release docker-toolbox docker-toolbox-run docker-toolbox-shell +.PHONY: build run test test-unit test-integration test-workflow-integration test-e2e test-e2e-verbose test-e2e-ssh test-e2e-api test-e2e-nix test-e2e-install test-docker test-ssh test-distributed test-all test-summary test-ci clean install-gotestsum lint fmt db-seed db-clean db-migrate run-server-debug swagger update-ui snapshot-release github-release docker-toolbox docker-toolbox-run docker-toolbox-shell docker-publish # Go parameters GOCMD=go @@ -268,6 +268,18 @@ docker-toolbox-run: docker-toolbox-shell: docker exec -it osmedeus-toolbox bash +# Docker publish (build and push to Docker Hub) +docker-publish: + @echo "$(PREFIX) Building Docker image j3ssie/osmedeus:latest..." + docker build -t j3ssie/osmedeus:latest \ + -f build/docker/Dockerfile \ + --build-arg BUILD_TIME=$(BUILD_TIME) \ + --build-arg COMMIT_HASH=$(COMMIT_HASH) \ + . + @echo "$(PREFIX) Pushing to Docker Hub..." + docker push j3ssie/osmedeus:latest + @echo "$(PREFIX) Published j3ssie/osmedeus:latest successfully!" + # Release commands (GoReleaser) snapshot-release: @echo "$(PREFIX) Building $(BINARY_NAME)..." @@ -278,7 +290,7 @@ snapshot-release: @echo "$(PREFIX) Update registry-metadata-direct-fetch.json..." cp ../osmedeus-registry/registry-metadata-direct-fetch.json public/presets/registry-metadata-direct-fetch.json @echo "$(PREFIX) Building snapshot release..." - export GORELEASER_CURRENT_TAG="$(VERSION)" && goreleaser release --snapshot --clean + export GORELEASER_CURRENT_TAG="$(VERSION)" && goreleaser release --clean --skip=announce,publish,validate @echo "$(PREFIX) Install script copied to dist/install.sh" cp ../osmedeus-registry/install.sh dist/install.sh @echo "$(PREFIX) Prepare registry-metadata-direct-fetch.json" @@ -341,6 +353,7 @@ help: @echo "\033[33m DOCKER\033[0m" @echo " make docker-build Build Docker image" @echo " make docker-run Run Docker container" + @echo " make docker-publish Build and push j3ssie/osmedeus:latest to Docker Hub" @echo " make docker-toolbox Build toolbox image (all tools pre-installed)" @echo " make docker-toolbox-run Start toolbox container" @echo " make docker-toolbox-shell Enter toolbox container shell" diff --git a/README.md b/README.md index 5623f44..12d08bb 100644 --- a/README.md +++ b/README.md @@ -75,16 +75,15 @@ osmedeus --usage-example ```bash # Show help -docker run --rm osmedeus:latest --help +docker run --rm j3ssie/osmedeus:latest --help # Run a scan docker run --rm -v $(pwd)/output:/root/workspaces-osmedeus \ - osmedeus:latest run -f general -t example.com + j3ssie/osmedeus:latest run -f general -t example.com ``` For more CLI usage and example commands, refer to the [CLI Reference](https://docs.osmedeus.org/getting-started/cli). - | CLI Usage | Web UI Assets | Web UI Workflow | |-----------|--------------|-----------------| | ![CLI Usage](https://raw.githubusercontent.com/osmedeus/assets/refs/heads/main/demo-images/cli-run-with-verbose-output.png) | ![Web UI Assets](https://raw.githubusercontent.com/osmedeus/assets/refs/heads/main/demo-images/web-ui-assets.png) | ![Web UI Workflow](https://raw.githubusercontent.com/osmedeus/assets/refs/heads/main/demo-images/web-ui-workflow.png) | diff --git a/build/DEPLOYMENT.md b/build/DEPLOYMENT.md index 54f922b..5553108 100644 --- a/build/DEPLOYMENT.md +++ b/build/DEPLOYMENT.md @@ -44,8 +44,7 @@ make build-windows # Windows amd64 ### Docker Build ```bash -# Production image (minimal, ~50MB) -docker build -t osmedeus:5.0.0 -f build/docker/Dockerfile . +docker build -t osmedeus:latest -f build/docker/Dockerfile . # Development image (with hot-reload) docker build -t osmedeus:dev -f build/docker/Dockerfile.dev . diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 5eb7f30..0e70897 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -2,7 +2,7 @@ # Ubuntu/Debian-based image with essential tools for security scanning # Stage 1: Build osmedeus binary -FROM golang:1.22-bookworm AS builder +FROM golang:1.25-bookworm AS builder WORKDIR /app @@ -22,7 +22,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build \ -o /app/bin/osmedeus ./cmd/osmedeus # Stage 2: Runtime with essential tools -FROM golang:1.22-bookworm +FROM golang:1.25-bookworm # Install essential tools RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/internal/console/capture.go b/internal/console/capture.go index 6c49b55..9277c81 100644 --- a/internal/console/capture.go +++ b/internal/console/capture.go @@ -44,15 +44,15 @@ func StartCapture(filePath string) (*Capture, error) { // Create pipes for stdout and stderr outReader, outWriter, err := os.Pipe() if err != nil { - file.Close() + _ = file.Close() return nil, err } errReader, errWriter, err := os.Pipe() if err != nil { - file.Close() - outReader.Close() - outWriter.Close() + _ = file.Close() + _ = outReader.Close() + _ = outWriter.Close() return nil, err } @@ -146,10 +146,10 @@ func (c *Capture) Stop() error { // Close pipe writers to signal EOF to tee goroutines if c.outWriter != nil { - c.outWriter.Close() + _ = c.outWriter.Close() } if c.errWriter != nil { - c.errWriter.Close() + _ = c.errWriter.Close() } // Restore original stdout/stderr immediately @@ -161,10 +161,10 @@ func (c *Capture) Stop() error { // Close readers if c.outReader != nil { - c.outReader.Close() + _ = c.outReader.Close() } if c.errReader != nil { - c.errReader.Close() + _ = c.errReader.Close() } // Close file @@ -172,7 +172,7 @@ func (c *Capture) Stop() error { defer c.mu.Unlock() if c.file != nil { _ = c.file.Sync() - c.file.Close() + _ = c.file.Close() c.file = nil } diff --git a/internal/executor/executor.go b/internal/executor/executor.go index fc2deba..0141fa0 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -660,7 +660,7 @@ func (e *Executor) ExecuteModule(ctx context.Context, module *core.Workflow, par } if e.consoleCapture != nil { defer func() { - e.consoleCapture.Stop() + _ = e.consoleCapture.Stop() e.consoleCapture = nil }() } @@ -970,7 +970,7 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params } if e.consoleCapture != nil { defer func() { - e.consoleCapture.Stop() + _ = e.consoleCapture.Stop() e.consoleCapture = nil }() } diff --git a/internal/functions/constants.go b/internal/functions/constants.go index 1c588d9..92f872a 100644 --- a/internal/functions/constants.go +++ b/internal/functions/constants.go @@ -21,11 +21,11 @@ const ( FnMoveFile = "moveFile" // moveFile(source, dest) -> bool FnGlob = "glob" // glob(pattern) -> []string - FnGrepStringToFile = "grep_string_to_file" // grep_string_to_file(dest, source, str) -> bool - FnGrepRegexToFile = "grep_regex_to_file" // grep_regex_to_file(dest, source, pattern) -> bool - FnGrepString = "grep_string" // grep_string(source, str) -> string - FnGrepRegex = "grep_regex" // grep_regex(source, pattern) -> string - FnRemoveBlankLines = "remove_blank_lines" // remove_blank_lines(path) -> bool (in-place) + FnGrepStringToFile = "grep_string_to_file" // grep_string_to_file(dest, source, str) -> bool + FnGrepRegexToFile = "grep_regex_to_file" // grep_regex_to_file(dest, source, pattern) -> bool + FnGrepString = "grep_string" // grep_string(source, str) -> string + FnGrepRegex = "grep_regex" // grep_regex(source, pattern) -> string + FnRemoveBlankLines = "remove_blank_lines" // remove_blank_lines(path) -> bool (in-place) ) // String Functions - String manipulation operations @@ -156,12 +156,12 @@ const ( // Markdown Functions - Markdown rendering and conversion const ( - FnRenderMarkdownFromFile = "render_markdown_from_file" // render_markdown_from_file(path) -> string (rendered markdown) - FnPrintMarkdownFromFile = "print_markdown_from_file" // print_markdown_from_file(path) -> void (print with syntax highlight) - FnConvertJSONLToMarkdown = "convert_jsonl_to_markdown" // convert_jsonl_to_markdown(input_path, output_path) -> bool (writes markdown table to file) - FnConvertCSVToMarkdown = "convert_csv_to_markdown" // convert_csv_to_markdown(path) -> string (markdown table) - FnRenderMarkdownReport = "render_markdown_report" // render_markdown_report(template_path, output_path) -> bool - FnGenerateSecurityReport = "generate_security_report" // generate_security_report(template_path) -> bool (output to {{Output}}/security-report.md) + FnRenderMarkdownFromFile = "render_markdown_from_file" // render_markdown_from_file(path) -> string (rendered markdown) + FnPrintMarkdownFromFile = "print_markdown_from_file" // print_markdown_from_file(path) -> void (print with syntax highlight) + FnConvertJSONLToMarkdown = "convert_jsonl_to_markdown" // convert_jsonl_to_markdown(input_path, output_path) -> bool (writes markdown table to file) + FnConvertCSVToMarkdown = "convert_csv_to_markdown" // convert_csv_to_markdown(path) -> string (markdown table) + FnRenderMarkdownReport = "render_markdown_report" // render_markdown_report(template_path, output_path) -> bool + FnGenerateSecurityReport = "generate_security_report" // generate_security_report(template_path) -> bool (output to {{Output}}/security-report.md) ) // Database Functions - Database update and import operations diff --git a/internal/functions/goja_runtime.go b/internal/functions/goja_runtime.go index 2b4bb71..5790726 100644 --- a/internal/functions/goja_runtime.go +++ b/internal/functions/goja_runtime.go @@ -10,8 +10,8 @@ import ( // GojaRuntime wraps the Goja JavaScript interpreter with VM pooling. // Uses a pool of Goja VMs for parallel execution without global mutex. type GojaRuntime struct { - pool *VMPool // Pool of configured VMs - mu sync.Mutex // Only used for custom function registration + pool *VMPool // Pool of configured VMs + mu sync.Mutex // Only used for custom function registration } // vmFunc provides VM access to all function implementations. diff --git a/internal/functions/url_functions_test.go b/internal/functions/url_functions_test.go index fe4219f..75121c4 100644 --- a/internal/functions/url_functions_test.go +++ b/internal/functions/url_functions_test.go @@ -250,16 +250,16 @@ func TestIsNoiseURL(t *testing.T) { url string expected bool }{ - {"https://example.com/blog/2022/post", true}, // blog path - {"https://example.com/news/article", true}, // news path - {"https://example.com/2022/01/02/page", true}, // calendar date - {"https://example.com/data/12345.html", true}, // numeric-only path - {"https://example.com/data/12345", true}, // numeric-only path no ext - {"https://example.com/api/v1/users", false}, // API endpoint - {"https://example.com/product/details", true}, // product is noise path - {"https://example.com/articles/tech-news", true}, // articles path - {"https://example.com/careers/engineer", true}, // careers path - {"https://example.com/search?q=test", false}, // search endpoint + {"https://example.com/blog/2022/post", true}, // blog path + {"https://example.com/news/article", true}, // news path + {"https://example.com/2022/01/02/page", true}, // calendar date + {"https://example.com/data/12345.html", true}, // numeric-only path + {"https://example.com/data/12345", true}, // numeric-only path no ext + {"https://example.com/api/v1/users", false}, // API endpoint + {"https://example.com/product/details", true}, // product is noise path + {"https://example.com/articles/tech-news", true}, // articles path + {"https://example.com/careers/engineer", true}, // careers path + {"https://example.com/search?q=test", false}, // search endpoint } for _, tt := range tests { diff --git a/internal/installer/nix.go b/internal/installer/nix.go index 648a01e..9d3f718 100644 --- a/internal/installer/nix.go +++ b/internal/installer/nix.go @@ -10,8 +10,8 @@ import ( "sort" "strings" - "github.com/orivej/go-nix/nix/parser" "github.com/j3ssie/osmedeus/v5/internal/logger" + "github.com/orivej/go-nix/nix/parser" "go.uber.org/zap" ) diff --git a/internal/parser/loader.go b/internal/parser/loader.go index b1aa599..3e07104 100644 --- a/internal/parser/loader.go +++ b/internal/parser/loader.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" "sync" @@ -183,6 +184,31 @@ func (l *Loader) LoadAllWorkflows() ([]*core.Workflow, error) { return workflows, nil } +// shouldSkipPath returns true for paths that are obviously not workflow files +func shouldSkipPath(path string) bool { + // Skip hidden directories (except the workflow root itself) + parts := strings.Split(path, string(os.PathSeparator)) + for _, part := range parts { + if strings.HasPrefix(part, ".") && part != "." { + return true // Skip .github/, .gitlab/, .git/, etc. + } + } + return false +} + +// isWorkflowYAML checks if a YAML file contains kind: module or kind: flow +func isWorkflowYAML(path string) bool { + content, err := os.ReadFile(path) + if err != nil { + return false + } + if !strings.Contains(string(content), "kind:") { + return false + } + kindPattern := regexp.MustCompile(`(?m)^kind:\s*['"]?(module|flow)['"]?\s*$`) + return kindPattern.Match(content) +} + // findYAMLFiles finds all YAML files in a directory func (l *Loader) findYAMLFiles(dir string, recursive bool) ([]string, error) { var files []string @@ -193,6 +219,12 @@ func (l *Loader) findYAMLFiles(dir string, recursive bool) ([]string, error) { return err } if !info.IsDir() && (strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) { + if shouldSkipPath(path) { + return nil // Skip hidden directories + } + if !isWorkflowYAML(path) { + return nil // Skip non-workflow YAML files + } files = append(files, path) } return nil @@ -207,7 +239,14 @@ func (l *Loader) findYAMLFiles(dir string, recursive bool) ([]string, error) { for _, entry := range entries { if !entry.IsDir() && (strings.HasSuffix(entry.Name(), ".yaml") || strings.HasSuffix(entry.Name(), ".yml")) { - files = append(files, filepath.Join(dir, entry.Name())) + path := filepath.Join(dir, entry.Name()) + if shouldSkipPath(path) { + continue // Skip hidden directories + } + if !isWorkflowYAML(path) { + continue // Skip non-workflow YAML files + } + files = append(files, path) } } diff --git a/internal/state/types.go b/internal/state/types.go index 85c4738..c4989d7 100644 --- a/internal/state/types.go +++ b/internal/state/types.go @@ -12,18 +12,18 @@ type StateExport struct { // RunInfo contains run information for export (mirrors database.Run fields) type RunInfo struct { - RunID string `json:"run_id"` - WorkflowName string `json:"workflow_name"` - WorkflowKind string `json:"workflow_kind"` - Target string `json:"target"` + RunID string `json:"run_id"` + WorkflowName string `json:"workflow_name"` + WorkflowKind string `json:"workflow_kind"` + Target string `json:"target"` Params map[string]any `json:"params,omitempty"` - Status string `json:"status"` - WorkspacePath string `json:"workspace_path"` - StartedAt *time.Time `json:"started_at,omitempty"` - CompletedAt *time.Time `json:"completed_at,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` - TotalSteps int `json:"total_steps"` - CompletedSteps int `json:"completed_steps"` + Status string `json:"status"` + WorkspacePath string `json:"workspace_path"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + TotalSteps int `json:"total_steps"` + CompletedSteps int `json:"completed_steps"` } // WorkspaceInfo contains workspace information for export diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 1824b05..2d805e3 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -7,9 +7,9 @@ import ( "os" "path/filepath" + "github.com/j3ssie/osmedeus/v5/internal/config" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/j3ssie/osmedeus/v5/internal/config" ) // Provider represents a cloud storage provider diff --git a/internal/updater/mock_source.go b/internal/updater/mock_source.go index 83601d7..4d02ecc 100644 --- a/internal/updater/mock_source.go +++ b/internal/updater/mock_source.go @@ -8,12 +8,12 @@ import ( // MockSource is a mock implementation of Source for testing type MockSource struct { - Releases []*Release - DetectError error - UpdateError error - DetectCalled int - UpdateCalled int - LastUpdateTo *Release + Releases []*Release + DetectError error + UpdateError error + DetectCalled int + UpdateCalled int + LastUpdateTo *Release } // NewMockSource creates a mock source with predefined releases diff --git a/lib/options.go b/lib/options.go index ee1d1b1..2c7bb81 100644 --- a/lib/options.go +++ b/lib/options.go @@ -143,4 +143,3 @@ func (o *EvalOptions) WithTarget(target string) *EvalOptions { copy.Target = target return © } - diff --git a/pkg/cli/db.go b/pkg/cli/db.go index 1eec319..4a2ff5c 100644 --- a/pkg/cli/db.go +++ b/pkg/cli/db.go @@ -36,7 +36,8 @@ var defaultHiddenColumns = []string{"id", "created_at", "updated_at", "completed // tableDefaultColumns defines default columns for specific tables var tableDefaultColumns = map[string][]string{ - "assets": {"host", "host_ip", "title", "status_code", "words", "tech"}, + "assets": {"asset_value", "host_ip", "title", "status_code", "words", "tech"}, + "vulnerabilities": {"asset_value", "asset_type", "severity", "confidence", "vuln_title", "vuln_info"}, } // dbCmd - parent command for database management diff --git a/pkg/cli/function.go b/pkg/cli/function.go index 27e343d..e9019b6 100644 --- a/pkg/cli/function.go +++ b/pkg/cli/function.go @@ -236,4 +236,3 @@ func runFunctionList(cmd *cobra.Command, args []string) error { } return nil } - diff --git a/pkg/cli/health.go b/pkg/cli/health.go index c1ec4fc..2cac20d 100644 --- a/pkg/cli/health.go +++ b/pkg/cli/health.go @@ -286,6 +286,11 @@ func findWorkflowYAMLFiles(root string) ([]string, error) { continue } + // Skip hidden directories and files + if strings.HasPrefix(name, ".") { + continue + } + fullPath := filepath.Join(dir, name) if entry.IsDir() { @@ -303,7 +308,10 @@ func findWorkflowYAMLFiles(root string) ([]string, error) { lower := strings.ToLower(name) if strings.HasSuffix(lower, ".yaml") || strings.HasSuffix(lower, ".yml") { - out = append(out, fullPath) + // Only include files that are actually workflow YAML files + if isWorkflowYAML(fullPath) { + out = append(out, fullPath) + } } } } diff --git a/pkg/cli/install.go b/pkg/cli/install.go index 6d692bc..7d503de 100644 --- a/pkg/cli/install.go +++ b/pkg/cli/install.go @@ -19,23 +19,23 @@ import ( ) var ( - registryPath string - installAll bool - binaryNames []string - checkOnly bool - customHeaders []string - nixBuildInstall bool - nixInstallation bool - nixPkgs []string - installOptional bool - descMaxWidth int - hideBinaryTags bool - baseSample bool - basePreset bool - workflowPreset bool - validateSample bool - validatePreset bool - installEnvAll bool + registryPath string + installAll bool + binaryNames []string + checkOnly bool + customHeaders []string + nixBuildInstall bool + nixInstallation bool + nixPkgs []string + installOptional bool + descMaxWidth int + hideBinaryTags bool + baseSample bool + basePreset bool + workflowPreset bool + validateSample bool + validatePreset bool + installEnvAll bool goGetterSources []string goGetterDest string listRegistryNixBuild bool @@ -227,14 +227,53 @@ func runInstallBase(cmd *cobra.Command, args []string) error { if err := inst.InstallBase(presetURL); err != nil { return err } + printer.Success("Base installed from: %s", terminal.Cyan(presetURL)) + printer.Newline() + // Install workflows from OSM_WORKFLOW_URL or DEFAULT_WORKFLOW_REPO + workflowURL := os.Getenv("OSM_WORKFLOW_URL") + if workflowURL != "" { + printer.Info("Using workflow URL from OSM_WORKFLOW_URL environment variable") + printer.Println(" %s %s", terminal.SymbolBullet, terminal.Gray(workflowURL)) + } else { + workflowURL = core.DEFAULT_WORKFLOW_REPO + printer.Info("Using default workflow URL") + printer.Println(" %s %s", terminal.SymbolBullet, terminal.Gray(workflowURL)) + } + + if err := inst.InstallWorkflow(workflowURL); err != nil { + printer.Warning("Failed to install workflows: %s", err) + // Continue - workflow installation failure shouldn't block base setup + } else { + printer.Success("Workflows installed from: %s", terminal.Cyan(workflowURL)) + } + printer.Newline() + + // Reload config after base and workflow installation reloaded, err := config.Load(cfg.BaseFolder) if err == nil { config.Set(reloaded) + cfg = reloaded if reloaded.BinariesPath != "" { binariesFolder = reloaded.BinariesPath } } + + // Check if first-time setup is needed (install binaries if so) + if isFirstTimeSetupNeeded(cfg.BaseFolder) { + printer.Println("%s %s", terminal.BoldBlue(terminal.SymbolLightning), terminal.HiBlue("First-time setup detected. Installing binaries...")) + printer.Newline() + + // Install required binaries + installRequiredBinaries(cfg, printer) + + // Create initialization marker + _ = createInitializationMarker(printer) + + printer.Newline() + printer.Println("%s %s", terminal.Green(terminal.SymbolSuccess), terminal.BoldGreen("First-time setup complete!")) + } + ensureBinariesPathInEnv(printer, binariesFolder, true) return nil } @@ -1565,7 +1604,7 @@ func truncatePad(s string, width int) string { const defaultParallelWorkers = 3 // binariesPerRow is the number of binary names displayed per row -const binariesPerRow = 10 +const binariesPerRow = 6 // splitIntoRows splits binary names into rows of specified size func splitIntoRows(names []string, perRow int) [][]string { diff --git a/pkg/cli/root.go b/pkg/cli/root.go index 1ec8970..dc650f2 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -292,6 +292,143 @@ func init() { rootCmd.AddCommand(updateCmd) } +// installRequiredBinaries installs all required binaries from the registry. +// Returns counts of installed, skipped, and failed binaries. +// This is a shared helper used by both runFirstTimeSetup and runInstallBase --preset. +func installRequiredBinaries(cfg *config.Config, printer *terminal.Printer) (installed, skipped, failed int) { + // Check for registry URL override + registryURL := os.Getenv("OSM_REGISTRY_URL") + registryDisplay := "the default registry" + if registryURL != "" { + registryDisplay = registryURL + } + + printer.Println("%s %s", terminal.BoldBlue(terminal.SymbolLightning), + terminal.HiBlue(fmt.Sprintf("Installing security binaries from %s", terminal.Cyan(registryDisplay)))) + printer.Println(" %s This may take a few minutes depending on your network speed", terminal.SymbolBullet) + printer.Newline() + + // Show spinner while loading registry + loadingSpinner := terminal.LoadingSpinner("Loading binary registry") + loadingSpinner.Start() + registry, err := installer.LoadRegistry(registryURL, nil) + loadingSpinner.Stop() + if err != nil { + printer.Warning("Failed to load binary registry: %s", err) + printer.Println(" %s", terminal.Gray("You can manually run: osmedeus install binary --all")) + return 0, 0, 0 + } + + binariesFolder := cfg.BinariesPath + if binariesFolder == "" { + binariesFolder = filepath.Join(cfg.BaseFolder, "external-binaries") + } + + // Create binaries folder if it doesn't exist + if err := os.MkdirAll(binariesFolder, 0755); err != nil { + printer.Warning("Failed to create binaries folder: %s", err) + } + + // Count binaries to install + var toInstall []string + for name, entry := range registry { + isOptional := false + for _, tag := range entry.Tags { + if tag == "optional" { + isOptional = true + break + } + } + if !isOptional && !installer.IsBinaryInPath(name) { + toInstall = append(toInstall, name) + } + } + + if len(toInstall) > 0 { + printer.Println(" %s Installing %s binaries to: %s", terminal.SymbolBullet, terminal.Green(fmt.Sprintf("%d", len(toInstall))), terminal.Cyan(binariesFolder)) + printer.Newline() + } + + // Sort binary names for consistent display + sort.Strings(toInstall) + + // Set silent mode for binary installation to reduce noise + _ = os.Setenv("OSMEDEUS_SILENT", "1") + defer func() { _ = os.Unsetenv("OSMEDEUS_SILENT") }() + + // Suppress logger output during binary installation + logCfg := logger.DefaultConfig() + logCfg.Silent = true + _ = logger.Init(logCfg) + defer func() { + // Restore normal logging after installation + logCfg.Silent = false + _ = logger.Init(logCfg) + }() + + // Install binaries in parallel with multi-row spinner display + var failedNames []string + if len(toInstall) > 0 { + failedNames = installBinariesParallel(toInstall, registry, binariesFolder, nil, printer, false) + } + + // Count results + installedCount := len(toInstall) - len(failedNames) + failedCount := len(failedNames) + + // Count skipped (already in PATH) - these weren't in toInstall + skippedCount := 0 + for name, entry := range registry { + isOptional := false + for _, tag := range entry.Tags { + if tag == "optional" { + isOptional = true + break + } + } + if !isOptional && installer.IsBinaryInPath(name) { + skippedCount++ + } + } + + printer.Newline() + + // Show summary + printer.Success("Installed %s binaries (%s skipped, %s failed)", + terminal.Green(fmt.Sprintf("%d", installedCount)), + terminal.Yellow(fmt.Sprintf("%d", skippedCount)), + terminal.Red(fmt.Sprintf("%d", failedCount))) + + // Ensure binaries path is in environment + ensureBinariesPathInEnv(printer, binariesFolder, false) + + return installedCount, skippedCount, failedCount +} + +// createInitializationMarker creates the $HOME/.osmedeus/initialized marker file. +// This marker indicates that first-time setup has been completed. +func createInitializationMarker(printer *terminal.Printer) error { + homeDir, err := os.UserHomeDir() + if err != nil { + printer.Warning("Failed to get home directory: %s", err) + return err + } + + osmDir := filepath.Join(homeDir, ".osmedeus") + if err := os.MkdirAll(osmDir, 0755); err != nil { + printer.Warning("Failed to create osmedeus config directory: %s", err) + return err + } + + markerFile := filepath.Join(osmDir, "initialized") + if err := os.WriteFile(markerFile, []byte("initialized\n"), 0644); err != nil { + printer.Warning("Failed to create initialization marker: %s", err) + return err + } + + return nil +} + // versionCmd shows version information var versionCmd = &cobra.Command{ Use: "version", @@ -392,15 +529,16 @@ func runFirstTimeSetup(baseFolder string, cfg *config.Config) error { } printer.Println(" %s Preset URL: %s", terminal.SymbolBullet, terminal.Cyan(presetURL)) - // Check for custom workflow URL from environment + // Check for custom workflow URL from environment (default to DEFAULT_WORKFLOW_REPO) workflowURL := os.Getenv("OSM_WORKFLOW_URL") - if workflowURL != "" { - printer.Println(" %s Workflow URL: %s", terminal.SymbolBullet, terminal.Cyan(workflowURL)) + if workflowURL == "" { + workflowURL = core.DEFAULT_WORKFLOW_REPO } + printer.Println(" %s Workflow URL: %s", terminal.SymbolBullet, terminal.Cyan(workflowURL)) printer.Newline() - // Step 2: Install preset base folder (workflows, etc.) - printer.Println("%s %s", terminal.BoldMagenta(terminal.SymbolLightning), terminal.HiMagenta("Installing preset workflows...")) + // Step 2: Install preset base folder + printer.Println("%s %s", terminal.BoldMagenta(terminal.SymbolLightning), terminal.HiMagenta("Installing preset base folder...")) printer.Println(" %s Downloading from: %s", terminal.SymbolBullet, terminal.Cyan(presetURL)) inst := installer.NewInstaller( baseFolder, @@ -409,24 +547,23 @@ func runFirstTimeSetup(baseFolder string, cfg *config.Config) error { nil, ) if err := inst.InstallBase(presetURL); err != nil { - printer.Warning("Failed to install preset workflows: %s", err) + printer.Warning("Failed to install preset base: %s", err) printer.Println(" %s", terminal.Gray("You can manually run: osmedeus install validate --preset")) return err } - printer.Success("Preset workflows installed to: %s", terminal.Cyan(filepath.Join(baseFolder, "workflows"))) + printer.Success("Base folder installed to: %s", terminal.Cyan(baseFolder)) printer.Newline() - // Step 3: Install workflows from separate URL if specified - if workflowURL != "" { - printer.Println("%s %s", terminal.BoldBlue(terminal.SymbolLightning), terminal.HiBlue("Installing workflows from OSM_WORKFLOW_URL...")) - printer.Println(" %s Downloading from: %s", terminal.SymbolBullet, terminal.Cyan(workflowURL)) - if err := inst.InstallWorkflow(workflowURL); err != nil { - printer.Warning("Failed to install workflows from OSM_WORKFLOW_URL: %s", err) - } else { - printer.Success("Workflows installed from: %s", terminal.Cyan(workflowURL)) - } - printer.Newline() + // Step 3: Install workflows from workflow URL + printer.Println("%s %s", terminal.BoldBlue(terminal.SymbolLightning), terminal.HiBlue("Installing workflows...")) + printer.Println(" %s Downloading from: %s", terminal.SymbolBullet, terminal.Cyan(workflowURL)) + if err := inst.InstallWorkflow(workflowURL); err != nil { + printer.Warning("Failed to install workflows: %s", err) + printer.Println(" %s", terminal.Gray("You can manually run: osmedeus install workflow --preset")) + } else { + printer.Success("Workflows installed to: %s", terminal.Cyan(filepath.Join(baseFolder, "workflows"))) } + printer.Newline() // Step 4: Reload config and load workflows printer.Println("%s %s", terminal.BoldMagenta(terminal.SymbolLightning), terminal.HiMagenta("Loading workflows...")) @@ -446,123 +583,11 @@ func runFirstTimeSetup(baseFolder string, cfg *config.Config) error { } printer.Newline() - // Step 5: Install all binaries - // Check for registry URL override - registryURL := os.Getenv("OSM_REGISTRY_URL") - registryDisplay := "the default registry" - if registryURL != "" { - registryDisplay = registryURL - } + // Step 5: Install all binaries using shared helper + installRequiredBinaries(cfg, printer) - printer.Println("%s %s", terminal.BoldBlue(terminal.SymbolLightning), - terminal.HiBlue(fmt.Sprintf("Installing security binaries from %s", terminal.Cyan(registryDisplay)))) - printer.Println(" %s This may take a few minutes depending on your network speed", terminal.SymbolBullet) - printer.Newline() - - // Show spinner while loading registry - loadingSpinner := terminal.LoadingSpinner("Loading binary registry") - loadingSpinner.Start() - registry, err := installer.LoadRegistry(registryURL, nil) - loadingSpinner.Stop() - if err != nil { - printer.Warning("Failed to load binary registry: %s", err) - printer.Println(" %s", terminal.Gray("You can manually run: osmedeus install binary --all")) - return err - } - - binariesFolder := cfg.BinariesPath - if binariesFolder == "" { - binariesFolder = filepath.Join(baseFolder, "external-binaries") - } - - // Create binaries folder if it doesn't exist - if err := os.MkdirAll(binariesFolder, 0755); err != nil { - printer.Warning("Failed to create binaries folder: %s", err) - } - - // Count binaries to install - var toInstall []string - for name, entry := range registry { - isOptional := false - for _, tag := range entry.Tags { - if tag == "optional" { - isOptional = true - break - } - } - if !isOptional && !installer.IsBinaryInPath(name) { - toInstall = append(toInstall, name) - } - } - - if len(toInstall) > 0 { - printer.Println(" %s Installing %s binaries to: %s", terminal.SymbolBullet, terminal.Green(fmt.Sprintf("%d", len(toInstall))), terminal.Cyan(binariesFolder)) - printer.Newline() - } - - // Sort binary names for consistent display - sort.Strings(toInstall) - - // Set silent mode for binary installation to reduce noise - _ = os.Setenv("OSMEDEUS_SILENT", "1") - defer func() { _ = os.Unsetenv("OSMEDEUS_SILENT") }() - - // Suppress logger output during binary installation - logCfg := logger.DefaultConfig() - logCfg.Silent = true - _ = logger.Init(logCfg) - defer func() { - // Restore normal logging after installation - logCfg.Silent = false - _ = logger.Init(logCfg) - }() - - // Install binaries in parallel with multi-row spinner display - var failed []string - if len(toInstall) > 0 { - failed = installBinariesParallel(toInstall, registry, binariesFolder, nil, printer, false) - } - - // Count results - installedCount := len(toInstall) - len(failed) - failedCount := len(failed) - - // Count skipped (already in PATH) - these weren't in toInstall - skippedCount := 0 - for name, entry := range registry { - isOptional := false - for _, tag := range entry.Tags { - if tag == "optional" { - isOptional = true - break - } - } - if !isOptional && installer.IsBinaryInPath(name) { - skippedCount++ - } - } - - printer.Newline() - - // Show summary - printer.Success("Installed %s binaries (%s skipped, %s failed)", - terminal.Green(fmt.Sprintf("%d", installedCount)), - terminal.Yellow(fmt.Sprintf("%d", skippedCount)), - terminal.Red(fmt.Sprintf("%d", failedCount))) - - // Ensure binaries path is in environment - ensureBinariesPathInEnv(printer, binariesFolder, false) - - // Create initialization marker file in $HOME/.osmedeus/ - homeDir, _ := os.UserHomeDir() - osmDir := filepath.Join(homeDir, ".osmedeus") - if err := os.MkdirAll(osmDir, 0755); err != nil { - printer.Warning("Failed to create osmedeus config directory: %s", err) - } - markerFile := filepath.Join(osmDir, "initialized") - if err := os.WriteFile(markerFile, []byte("initialized\n"), 0644); err != nil { - printer.Warning("Failed to create initialization marker: %s", err) - } + // Create initialization marker file + _ = createInitializationMarker(printer) // Print completion message printer.Newline() diff --git a/pkg/cli/run.go b/pkg/cli/run.go index 0a505d6..5de0d4b 100644 --- a/pkg/cli/run.go +++ b/pkg/cli/run.go @@ -31,30 +31,30 @@ import ( ) var ( - flowName string - moduleNames []string - targets []string - targetFile string - paramFlags []string - paramsFile string - workspacePath string - dryRun bool - threadsHold int - runTactic string - excludeModules []string - spaceName string - workspacesFolder string - heuristicsCheck string - distributedRun bool - redisURLRun string - concurrency int - repeatRun bool - repeatWaitTime string - runTimeout string - stdModule bool - emptyTarget bool - progressBar bool - disableWorkflowState bool + flowName string + moduleNames []string + targets []string + targetFile string + paramFlags []string + paramsFile string + workspacePath string + dryRun bool + threadsHold int + runTactic string + excludeModules []string + spaceName string + workspacesFolder string + heuristicsCheck string + distributedRun bool + redisURLRun string + concurrency int + repeatRun bool + repeatWaitTime string + runTimeout string + stdModule bool + emptyTarget bool + progressBar bool + disableWorkflowState bool // explicitFlags tracks which CLI flags were explicitly set by the user // Used to determine precedence when applying workflow preferences diff --git a/pkg/cli/workflow.go b/pkg/cli/workflow.go index 8653a4b..381c06d 100644 --- a/pkg/cli/workflow.go +++ b/pkg/cli/workflow.go @@ -1310,9 +1310,10 @@ func printValidationTable(results []ValidationResult, baseDir string) { } statusColor := terminal.Green - if r.Status == "failed" { + switch r.Status { + case "failed": statusColor = terminal.Red - } else if r.Status == "warning" { + case "warning": statusColor = terminal.Yellow } diff --git a/pkg/server/server.go b/pkg/server/server.go index a41f685..456ba6f 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -15,7 +15,6 @@ import ( "github.com/gofiber/fiber/v2/middleware/logger" "github.com/gofiber/fiber/v2/middleware/recover" "github.com/gofiber/swagger" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/j3ssie/osmedeus/v5/internal/config" "github.com/j3ssie/osmedeus/v5/internal/core" "github.com/j3ssie/osmedeus/v5/internal/database" @@ -24,6 +23,7 @@ import ( "github.com/j3ssie/osmedeus/v5/pkg/server/handlers" "github.com/j3ssie/osmedeus/v5/pkg/server/middleware" "github.com/j3ssie/osmedeus/v5/public" + "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/zap" _ "github.com/j3ssie/osmedeus/v5/docs/api-swagger" // swagger docs diff --git a/public/mics/install.sh b/public/mics/install.sh index 0b884df..4cb4125 100755 --- a/public/mics/install.sh +++ b/public/mics/install.sh @@ -9,6 +9,7 @@ OSM_HOME="${OSM_HOME:-$HOME/.osmedeus}" BIN_DIR="$HOME/.local/bin" GITHUB_REPO="j3ssie/osmedeus" GITHUB_RELEASES="https://github.com/${GITHUB_REPO}/releases" +FALLBACK_VERSION="v5.0.0-beta" OSM_URL_ENV_SET=0 if [[ -n "${OSM_URL+x}" ]]; then OSM_URL_ENV_SET=1 @@ -144,8 +145,13 @@ downloader() { download_file() { local url="$1" local output_file="$2" + local version="${3:-}" - log "Downloading $(basename "$output_file")..." + if [[ -n "$version" ]]; then + log "Downloading $(basename "$output_file") (${LIGHT_GREEN}${version}${NC})..." + else + log "Downloading $(basename "$output_file")..." + fi # Use secure temporary file local temp_file @@ -173,23 +179,31 @@ verify_checksum() { success "Checksum verified" } -# Fetch latest CLI version from GitHub +# Fetch latest CLI version from GitHub API with fallback fetch_latest_version() { - local api_url="https://api.github.com/repos/${GITHUB_REPO}/releases/latest" - local version - local tmp_file + local api_url="https://api.github.com/repos/${GITHUB_REPO}/releases/latest" + local version + local tmp_file - log "Fetching latest version from GitHub..." - tmp_file=$(mktemp) - downloader "$api_url" "$tmp_file" - version=$(grep '"tag_name":' "$tmp_file" | head -n 1 | sed -E 's/.*"([^"]+)".*/\1/') - rm -f "$tmp_file" + log "Fetching latest version from GitHub..." + tmp_file=$(mktemp) - if [[ -z "$version" ]]; then - error "Failed to fetch latest version from GitHub" - fi + # Try to fetch from GitHub API + if downloader "$api_url" "$tmp_file" 2>/dev/null; then + version=$(grep '"tag_name":' "$tmp_file" | head -n 1 | sed -E 's/.*"([^"]+)".*/\1/') + rm -f "$tmp_file" - echo "$version" + if [[ -n "$version" ]]; then + echo "$version" + return + fi + fi + + rm -f "$tmp_file" 2>/dev/null + + # Fall back to hardcoded version + warn "Failed to fetch from GitHub API, using fallback: $FALLBACK_VERSION" + echo "$FALLBACK_VERSION" } fetch_latest_version_from_metadata() { @@ -259,9 +273,11 @@ install_osmedeus_binary() { if [[ "$version" != v* ]]; then version="v${version}" fi - log "Installing version: $version" + log "Installing version: ${LIGHT_GREEN}${version}${NC}" - local tarball_name="osmedeus_${version}_${platform}.tar.gz" + # Strip 'v' prefix for tarball filename (e.g., v5.0.0 -> 5.0.0) + local version_no_v="${version#v}" + local tarball_name="osmedeus_${version_no_v}_${platform}.tar.gz" local base_url if [[ $OSM_URL_ENV_SET -eq 1 && -n "${OSM_URL}" ]]; then base_url="${OSM_URL%/}" @@ -282,7 +298,7 @@ install_osmedeus_binary() { mkdir -p "$extract_dir" # Download checksum first - download_file "$checksum_url" "$checksum_path" + download_file "$checksum_url" "$checksum_path" "$version" # Extract expected checksum for our tarball local expected_checksum @@ -293,7 +309,7 @@ install_osmedeus_binary() { fi # Download tarball - download_file "$tarball_url" "$tarball_path" + download_file "$tarball_url" "$tarball_path" "$version" # Verify checksum verify_checksum "$tarball_path" "$expected_checksum" diff --git a/public/presets/registry-metadata-direct-fetch.json b/public/presets/registry-metadata-direct-fetch.json index c9a5798..dbbad04 100644 --- a/public/presets/registry-metadata-direct-fetch.json +++ b/public/presets/registry-metadata-direct-fetch.json @@ -553,7 +553,7 @@ "multi-commands-linux": [ "git clone --depth=1 https://github.com/blechschmidt/massdns.git /tmp/massdns", "cd /tmp/massdns && make", - "cp /tmp/massdns/bin/massdns $HOME/.local/bin/" + "mkdir -p $HOME/.local/bin/ && cp /tmp/massdns/bin/massdns $HOME/.local/bin/" ], "multi-commands-darwin": [ " install massdns"