WEBDIENSTE/osmedeus
1
0
Fork 0
mirror of https://github.com/j3ssie/osmedeus.git synced 2026-09-11 04:07:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor: code quality improvements, Docker updates, and setup enhancements

Browse Source 
- chore: fix code style and remove unused error handling (add blank checks for closed file handles)
- chore: align struct field padding in multiple files for consistency
- fix: add 386 architecture support to goreleaser build config
- feat: add docker-publish target to Makefile for publishing to Docker Hub
- feat: refactor first-time setup into reusable installRequiredBinaries helper function
- feat: add initialization marker tracking for first-time setup completion
- feat: enhance workflow YAML detection to skip non-workflow files and hidden directories
- feat: improve database column display defaults (assets and vulnerabilities tables)
- feat: add fallback mechanism to install.sh for version detection failures
- fix: correct tarball filename generation by stripping 'v' prefix
- chore: update Docker base image from golang:1.22 to golang:1.25
- chore: update goreleaser release flags and simplify Docker build naming
- chore: fix import ordering across multiple files (alphabetical consistency)
- chore: improve install script with better version display formatting
- chore: reduce binaries per row from 10 to 6 in CLI output for better readability
This commit is contained in:
j3ssie
2026-01-19 01:17:04 +08:00
parent 7a2c5a5dc9
commit 9ed02e7eee
26 changed files with 406 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.goreleaser.yaml
+4
View File
@@ -19,6 +19,10 @@ builds:
goarch:
- amd64
- arm64
- 386
ignore:
- goos: darwin
goarch: 386
ldflags:
- -s -w
- -X main.BuildTime={{.Date}}
Makefile
+15 -2
View File
@@ -1,4 +1,4 @@
.PHONY: build run test test-unit test-integration test-workflow-integration test-e2e test-e2e-verbose test-e2e-ssh test-e2e-api test-e2e-nix test-e2e-install test-docker test-ssh test-distributed test-all test-summary test-ci clean install-gotestsum lint fmt db-seed db-clean db-migrate run-server-debug swagger update-ui snapshot-release github-release docker-toolbox docker-toolbox-run docker-toolbox-shell
.PHONY: build run test test-unit test-integration test-workflow-integration test-e2e test-e2e-verbose test-e2e-ssh test-e2e-api test-e2e-nix test-e2e-install test-docker test-ssh test-distributed test-all test-summary test-ci clean install-gotestsum lint fmt db-seed db-clean db-migrate run-server-debug swagger update-ui snapshot-release github-release docker-toolbox docker-toolbox-run docker-toolbox-shell docker-publish
# Go parameters
GOCMD=go
@@ -268,6 +268,18 @@ docker-toolbox-run:
docker-toolbox-shell:
docker exec -it osmedeus-toolbox bash
# Docker publish (build and push to Docker Hub)
docker-publish:
@echo "$(PREFIX) Building Docker image j3ssie/osmedeus:latest..."
docker build -t j3ssie/osmedeus:latest \
-f build/docker/Dockerfile \
--build-arg BUILD_TIME=$(BUILD_TIME) \
--build-arg COMMIT_HASH=$(COMMIT_HASH) \
.
@echo "$(PREFIX) Pushing to Docker Hub..."
docker push j3ssie/osmedeus:latest
@echo "$(PREFIX) Published j3ssie/osmedeus:latest successfully!"
# Release commands (GoReleaser)
snapshot-release:
@echo "$(PREFIX) Building $(BINARY_NAME)..."
@@ -278,7 +290,7 @@ snapshot-release:
@echo "$(PREFIX) Update registry-metadata-direct-fetch.json..."
cp ../osmedeus-registry/registry-metadata-direct-fetch.json public/presets/registry-metadata-direct-fetch.json
@echo "$(PREFIX) Building snapshot release..."
export GORELEASER_CURRENT_TAG="$(VERSION)" && goreleaser release --snapshot --clean
export GORELEASER_CURRENT_TAG="$(VERSION)" && goreleaser release --clean --skip=announce,publish,validate
@echo "$(PREFIX) Install script copied to dist/install.sh"
cp ../osmedeus-registry/install.sh dist/install.sh
@echo "$(PREFIX) Prepare registry-metadata-direct-fetch.json"
@@ -341,6 +353,7 @@ help:
@echo "\033[33m DOCKER\033[0m"
@echo " make docker-build Build Docker image"
@echo " make docker-run Run Docker container"
@echo " make docker-publish Build and push j3ssie/osmedeus:latest to Docker Hub"
@echo " make docker-toolbox Build toolbox image (all tools pre-installed)"
@echo " make docker-toolbox-run Start toolbox container"
@echo " make docker-toolbox-shell Enter toolbox container shell"
README.md
+2 -3
View File
@@ -75,16 +75,15 @@ osmedeus --usage-example
```bash
# Show help
docker run --rm osmedeus:latest --help
docker run --rm j3ssie/osmedeus:latest --help
# Run a scan
docker run --rm -v $(pwd)/output:/root/workspaces-osmedeus \
osmedeus:latest run -f general -t example.com
j3ssie/osmedeus:latest run -f general -t example.com
```
For more CLI usage and example commands, refer to the [CLI Reference](https://docs.osmedeus.org/getting-started/cli).
| CLI Usage | Web UI Assets | Web UI Workflow |
|-----------|--------------|-----------------|
| ![CLI Usage](https://raw.githubusercontent.com/osmedeus/assets/refs/heads/main/demo-images/cli-run-with-verbose-output.png) | ![Web UI Assets](https://raw.githubusercontent.com/osmedeus/assets/refs/heads/main/demo-images/web-ui-assets.png) | ![Web UI Workflow](https://raw.githubusercontent.com/osmedeus/assets/refs/heads/main/demo-images/web-ui-workflow.png) |
build/DEPLOYMENT.md
+1 -2
View File
@@ -44,8 +44,7 @@ make build-windows # Windows amd64
### Docker Build
```bash
# Production image (minimal, ~50MB)
docker build -t osmedeus:5.0.0 -f build/docker/Dockerfile .
docker build -t osmedeus:latest -f build/docker/Dockerfile .
# Development image (with hot-reload)
docker build -t osmedeus:dev -f build/docker/Dockerfile.dev .
build/docker/Dockerfile
+2 -2
View File
@@ -2,7 +2,7 @@
# Ubuntu/Debian-based image with essential tools for security scanning
# Stage 1: Build osmedeus binary
FROM golang:1.22-bookworm AS builder
FROM golang:1.25-bookworm AS builder
WORKDIR /app
@@ -22,7 +22,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build \
-o /app/bin/osmedeus ./cmd/osmedeus
# Stage 2: Runtime with essential tools
FROM golang:1.22-bookworm
FROM golang:1.25-bookworm
# Install essential tools
RUN apt-get update && apt-get install -y --no-install-recommends \
internal/console/capture.go
+9 -9
View File
@@ -44,15 +44,15 @@ func StartCapture(filePath string) (*Capture, error) {
// Create pipes for stdout and stderr
outReader, outWriter, err := os.Pipe()
if err != nil {
file.Close()
_ = file.Close()
return nil, err
}
errReader, errWriter, err := os.Pipe()
if err != nil {
file.Close()
outReader.Close()
outWriter.Close()
_ = file.Close()
_ = outReader.Close()
_ = outWriter.Close()
return nil, err
}
@@ -146,10 +146,10 @@ func (c *Capture) Stop() error {
// Close pipe writers to signal EOF to tee goroutines
if c.outWriter != nil {
c.outWriter.Close()
_ = c.outWriter.Close()
}
if c.errWriter != nil {
c.errWriter.Close()
_ = c.errWriter.Close()
}
// Restore original stdout/stderr immediately
@@ -161,10 +161,10 @@ func (c *Capture) Stop() error {
// Close readers
if c.outReader != nil {
c.outReader.Close()
_ = c.outReader.Close()
}
if c.errReader != nil {
c.errReader.Close()
_ = c.errReader.Close()
}
// Close file
@@ -172,7 +172,7 @@ func (c *Capture) Stop() error {
defer c.mu.Unlock()
if c.file != nil {
_ = c.file.Sync()
c.file.Close()
_ = c.file.Close()
c.file = nil
}
internal/executor/executor.go
+2 -2
View File
@@ -660,7 +660,7 @@ func (e *Executor) ExecuteModule(ctx context.Context, module *core.Workflow, par
}
if e.consoleCapture != nil {
defer func() {
e.consoleCapture.Stop()
_ = e.consoleCapture.Stop()
e.consoleCapture = nil
}()
}
@@ -970,7 +970,7 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params
}
if e.consoleCapture != nil {
defer func() {
e.consoleCapture.Stop()
_ = e.consoleCapture.Stop()
e.consoleCapture = nil
}()
}
internal/functions/constants.go
+11 -11
View File
@@ -21,11 +21,11 @@ const (
FnMoveFile = "moveFile" // moveFile(source, dest) -> bool
FnGlob = "glob" // glob(pattern) -> []string
FnGrepStringToFile = "grep_string_to_file" // grep_string_to_file(dest, source, str) -> bool
FnGrepRegexToFile = "grep_regex_to_file" // grep_regex_to_file(dest, source, pattern) -> bool
FnGrepString = "grep_string" // grep_string(source, str) -> string
FnGrepRegex = "grep_regex" // grep_regex(source, pattern) -> string
FnRemoveBlankLines = "remove_blank_lines" // remove_blank_lines(path) -> bool (in-place)
FnGrepStringToFile = "grep_string_to_file" // grep_string_to_file(dest, source, str) -> bool
FnGrepRegexToFile = "grep_regex_to_file" // grep_regex_to_file(dest, source, pattern) -> bool
FnGrepString = "grep_string" // grep_string(source, str) -> string
FnGrepRegex = "grep_regex" // grep_regex(source, pattern) -> string
FnRemoveBlankLines = "remove_blank_lines" // remove_blank_lines(path) -> bool (in-place)
)
// String Functions - String manipulation operations
@@ -156,12 +156,12 @@ const (
// Markdown Functions - Markdown rendering and conversion
const (
FnRenderMarkdownFromFile = "render_markdown_from_file" // render_markdown_from_file(path) -> string (rendered markdown)
FnPrintMarkdownFromFile = "print_markdown_from_file" // print_markdown_from_file(path) -> void (print with syntax highlight)
FnConvertJSONLToMarkdown = "convert_jsonl_to_markdown" // convert_jsonl_to_markdown(input_path, output_path) -> bool (writes markdown table to file)
FnConvertCSVToMarkdown = "convert_csv_to_markdown" // convert_csv_to_markdown(path) -> string (markdown table)
FnRenderMarkdownReport = "render_markdown_report" // render_markdown_report(template_path, output_path) -> bool
FnGenerateSecurityReport = "generate_security_report" // generate_security_report(template_path) -> bool (output to {{Output}}/security-report.md)
FnRenderMarkdownFromFile = "render_markdown_from_file" // render_markdown_from_file(path) -> string (rendered markdown)
FnPrintMarkdownFromFile = "print_markdown_from_file" // print_markdown_from_file(path) -> void (print with syntax highlight)
FnConvertJSONLToMarkdown = "convert_jsonl_to_markdown" // convert_jsonl_to_markdown(input_path, output_path) -> bool (writes markdown table to file)
FnConvertCSVToMarkdown = "convert_csv_to_markdown" // convert_csv_to_markdown(path) -> string (markdown table)
FnRenderMarkdownReport = "render_markdown_report" // render_markdown_report(template_path, output_path) -> bool
FnGenerateSecurityReport = "generate_security_report" // generate_security_report(template_path) -> bool (output to {{Output}}/security-report.md)
)
// Database Functions - Database update and import operations
internal/functions/goja_runtime.go
+2 -2
View File
@@ -10,8 +10,8 @@ import (
// GojaRuntime wraps the Goja JavaScript interpreter with VM pooling.
// Uses a pool of Goja VMs for parallel execution without global mutex.
type GojaRuntime struct {
pool *VMPool // Pool of configured VMs
mu sync.Mutex // Only used for custom function registration
pool *VMPool // Pool of configured VMs
mu sync.Mutex // Only used for custom function registration
}
// vmFunc provides VM access to all function implementations.
internal/functions/url_functions_test.go
+10 -10
View File
@@ -250,16 +250,16 @@ func TestIsNoiseURL(t *testing.T) {
url string
expected bool
}{
{"https://example.com/blog/2022/post", true}, // blog path
{"https://example.com/news/article", true}, // news path
{"https://example.com/2022/01/02/page", true}, // calendar date
{"https://example.com/data/12345.html", true}, // numeric-only path
{"https://example.com/data/12345", true}, // numeric-only path no ext
{"https://example.com/api/v1/users", false}, // API endpoint
{"https://example.com/product/details", true}, // product is noise path
{"https://example.com/articles/tech-news", true}, // articles path
{"https://example.com/careers/engineer", true}, // careers path
{"https://example.com/search?q=test", false}, // search endpoint
{"https://example.com/blog/2022/post", true}, // blog path
{"https://example.com/news/article", true}, // news path
{"https://example.com/2022/01/02/page", true}, // calendar date
{"https://example.com/data/12345.html", true}, // numeric-only path
{"https://example.com/data/12345", true}, // numeric-only path no ext
{"https://example.com/api/v1/users", false}, // API endpoint
{"https://example.com/product/details", true}, // product is noise path
{"https://example.com/articles/tech-news", true}, // articles path
{"https://example.com/careers/engineer", true}, // careers path
{"https://example.com/search?q=test", false}, // search endpoint
}
for _, tt := range tests {
internal/installer/nix.go
+1 -1
View File
@@ -10,8 +10,8 @@ import (
"sort"
"strings"
"github.com/orivej/go-nix/nix/parser"
"github.com/j3ssie/osmedeus/v5/internal/logger"
"github.com/orivej/go-nix/nix/parser"
"go.uber.org/zap"
)
internal/parser/loader.go
+40 -1
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
@@ -183,6 +184,31 @@ func (l *Loader) LoadAllWorkflows() ([]*core.Workflow, error) {
return workflows, nil
}
// shouldSkipPath returns true for paths that are obviously not workflow files
func shouldSkipPath(path string) bool {
// Skip hidden directories (except the workflow root itself)
parts := strings.Split(path, string(os.PathSeparator))
for _, part := range parts {
if strings.HasPrefix(part, ".") && part != "." {
return true // Skip .github/, .gitlab/, .git/, etc.
}
}
return false
}
// isWorkflowYAML checks if a YAML file contains kind: module or kind: flow
func isWorkflowYAML(path string) bool {
content, err := os.ReadFile(path)
if err != nil {
return false
}
if !strings.Contains(string(content), "kind:") {
return false
}
kindPattern := regexp.MustCompile(`(?m)^kind:\s*['"]?(module|flow)['"]?\s*$`)
return kindPattern.Match(content)
}
// findYAMLFiles finds all YAML files in a directory
func (l *Loader) findYAMLFiles(dir string, recursive bool) ([]string, error) {
var files []string
@@ -193,6 +219,12 @@ func (l *Loader) findYAMLFiles(dir string, recursive bool) ([]string, error) {
return err
}
if !info.IsDir() && (strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) {
if shouldSkipPath(path) {
return nil // Skip hidden directories
}
if !isWorkflowYAML(path) {
return nil // Skip non-workflow YAML files
}
files = append(files, path)
}
return nil
@@ -207,7 +239,14 @@ func (l *Loader) findYAMLFiles(dir string, recursive bool) ([]string, error) {
for _, entry := range entries {
if !entry.IsDir() && (strings.HasSuffix(entry.Name(), ".yaml") || strings.HasSuffix(entry.Name(), ".yml")) {
files = append(files, filepath.Join(dir, entry.Name()))
path := filepath.Join(dir, entry.Name())
if shouldSkipPath(path) {
continue // Skip hidden directories
}
if !isWorkflowYAML(path) {
continue // Skip non-workflow YAML files
}
files = append(files, path)
}
}
internal/state/types.go
+11 -11
View File
@@ -12,18 +12,18 @@ type StateExport struct {
// RunInfo contains run information for export (mirrors database.Run fields)
type RunInfo struct {
RunID string `json:"run_id"`
WorkflowName string `json:"workflow_name"`
WorkflowKind string `json:"workflow_kind"`
Target string `json:"target"`
RunID string `json:"run_id"`
WorkflowName string `json:"workflow_name"`
WorkflowKind string `json:"workflow_kind"`
Target string `json:"target"`
Params map[string]any `json:"params,omitempty"`
Status string `json:"status"`
WorkspacePath string `json:"workspace_path"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
TotalSteps int `json:"total_steps"`
CompletedSteps int `json:"completed_steps"`
Status string `json:"status"`
WorkspacePath string `json:"workspace_path"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
TotalSteps int `json:"total_steps"`
CompletedSteps int `json:"completed_steps"`
}
// WorkspaceInfo contains workspace information for export
internal/storage/storage.go
+1 -1
View File
@@ -7,9 +7,9 @@ import (
"os"
"path/filepath"
"github.com/j3ssie/osmedeus/v5/internal/config"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/j3ssie/osmedeus/v5/internal/config"
)
// Provider represents a cloud storage provider
internal/updater/mock_source.go
+6 -6
View File
@@ -8,12 +8,12 @@ import (
// MockSource is a mock implementation of Source for testing
type MockSource struct {
Releases []*Release
DetectError error
UpdateError error
DetectCalled int
UpdateCalled int
LastUpdateTo *Release
Releases []*Release
DetectError error
UpdateError error
DetectCalled int
UpdateCalled int
LastUpdateTo *Release
}
// NewMockSource creates a mock source with predefined releases
lib/options.go
-1
View File
@@ -143,4 +143,3 @@ func (o *EvalOptions) WithTarget(target string) *EvalOptions {
copy.Target = target
return &copy
}
pkg/cli/db.go
+2 -1
View File
@@ -36,7 +36,8 @@ var defaultHiddenColumns = []string{"id", "created_at", "updated_at", "completed
// tableDefaultColumns defines default columns for specific tables
var tableDefaultColumns = map[string][]string{
"assets": {"host", "host_ip", "title", "status_code", "words", "tech"},
"assets": {"asset_value", "host_ip", "title", "status_code", "words", "tech"},
"vulnerabilities": {"asset_value", "asset_type", "severity", "confidence", "vuln_title", "vuln_info"},
}
// dbCmd - parent command for database management
pkg/cli/function.go
-1
View File
@@ -236,4 +236,3 @@ func runFunctionList(cmd *cobra.Command, args []string) error {
}
return nil
}
pkg/cli/health.go
+9 -1
View File
@@ -286,6 +286,11 @@ func findWorkflowYAMLFiles(root string) ([]string, error) {
continue
}
// Skip hidden directories and files
if strings.HasPrefix(name, ".") {
continue
}
fullPath := filepath.Join(dir, name)
if entry.IsDir() {
@@ -303,7 +308,10 @@ func findWorkflowYAMLFiles(root string) ([]string, error) {
lower := strings.ToLower(name)
if strings.HasSuffix(lower, ".yaml") || strings.HasSuffix(lower, ".yml") {
out = append(out, fullPath)
// Only include files that are actually workflow YAML files
if isWorkflowYAML(fullPath) {
out = append(out, fullPath)
}
}
}
}
pkg/cli/install.go
+57 -18
View File
@@ -19,23 +19,23 @@ import (
)
var (
registryPath string
installAll bool
binaryNames []string
checkOnly bool
customHeaders []string
nixBuildInstall bool
nixInstallation bool
nixPkgs []string
installOptional bool
descMaxWidth int
hideBinaryTags bool
baseSample bool
basePreset bool
workflowPreset bool
validateSample bool
validatePreset bool
installEnvAll bool
registryPath string
installAll bool
binaryNames []string
checkOnly bool
customHeaders []string
nixBuildInstall bool
nixInstallation bool
nixPkgs []string
installOptional bool
descMaxWidth int
hideBinaryTags bool
baseSample bool
basePreset bool
workflowPreset bool
validateSample bool
validatePreset bool
installEnvAll bool
goGetterSources []string
goGetterDest string
listRegistryNixBuild bool
@@ -227,14 +227,53 @@ func runInstallBase(cmd *cobra.Command, args []string) error {
if err := inst.InstallBase(presetURL); err != nil {
return err
}
printer.Success("Base installed from: %s", terminal.Cyan(presetURL))
printer.Newline()
// Install workflows from OSM_WORKFLOW_URL or DEFAULT_WORKFLOW_REPO
workflowURL := os.Getenv("OSM_WORKFLOW_URL")
if workflowURL != "" {
printer.Info("Using workflow URL from OSM_WORKFLOW_URL environment variable")
printer.Println(" %s %s", terminal.SymbolBullet, terminal.Gray(workflowURL))
} else {
workflowURL = core.DEFAULT_WORKFLOW_REPO
printer.Info("Using default workflow URL")
printer.Println(" %s %s", terminal.SymbolBullet, terminal.Gray(workflowURL))
}
if err := inst.InstallWorkflow(workflowURL); err != nil {
printer.Warning("Failed to install workflows: %s", err)
// Continue - workflow installation failure shouldn't block base setup
} else {
printer.Success("Workflows installed from: %s", terminal.Cyan(workflowURL))
}
printer.Newline()
// Reload config after base and workflow installation
reloaded, err := config.Load(cfg.BaseFolder)
if err == nil {
config.Set(reloaded)
cfg = reloaded
if reloaded.BinariesPath != "" {
binariesFolder = reloaded.BinariesPath
}
}
// Check if first-time setup is needed (install binaries if so)
if isFirstTimeSetupNeeded(cfg.BaseFolder) {
printer.Println("%s %s", terminal.BoldBlue(terminal.SymbolLightning), terminal.HiBlue("First-time setup detected. Installing binaries..."))
printer.Newline()
// Install required binaries
installRequiredBinaries(cfg, printer)
// Create initialization marker
_ = createInitializationMarker(printer)
printer.Newline()
printer.Println("%s %s", terminal.Green(terminal.SymbolSuccess), terminal.BoldGreen("First-time setup complete!"))
}
ensureBinariesPathInEnv(printer, binariesFolder, true)
return nil
}
@@ -1565,7 +1604,7 @@ func truncatePad(s string, width int) string {
const defaultParallelWorkers = 3
// binariesPerRow is the number of binary names displayed per row
const binariesPerRow = 10
const binariesPerRow = 6
// splitIntoRows splits binary names into rows of specified size
func splitIntoRows(names []string, perRow int) [][]string {
pkg/cli/root.go
+158 -133
View File
@@ -292,6 +292,143 @@ func init() {
rootCmd.AddCommand(updateCmd)
}
// installRequiredBinaries installs all required binaries from the registry.
// Returns counts of installed, skipped, and failed binaries.
// This is a shared helper used by both runFirstTimeSetup and runInstallBase --preset.
func installRequiredBinaries(cfg *config.Config, printer *terminal.Printer) (installed, skipped, failed int) {
// Check for registry URL override
registryURL := os.Getenv("OSM_REGISTRY_URL")
registryDisplay := "the default registry"
if registryURL != "" {
registryDisplay = registryURL
}
printer.Println("%s %s", terminal.BoldBlue(terminal.SymbolLightning),
terminal.HiBlue(fmt.Sprintf("Installing security binaries from %s", terminal.Cyan(registryDisplay))))
printer.Println(" %s This may take a few minutes depending on your network speed", terminal.SymbolBullet)
printer.Newline()
// Show spinner while loading registry
loadingSpinner := terminal.LoadingSpinner("Loading binary registry")
loadingSpinner.Start()
registry, err := installer.LoadRegistry(registryURL, nil)
loadingSpinner.Stop()
if err != nil {
printer.Warning("Failed to load binary registry: %s", err)
printer.Println(" %s", terminal.Gray("You can manually run: osmedeus install binary --all"))
return 0, 0, 0
}
binariesFolder := cfg.BinariesPath
if binariesFolder == "" {
binariesFolder = filepath.Join(cfg.BaseFolder, "external-binaries")
}
// Create binaries folder if it doesn't exist
if err := os.MkdirAll(binariesFolder, 0755); err != nil {
printer.Warning("Failed to create binaries folder: %s", err)
}
// Count binaries to install
var toInstall []string
for name, entry := range registry {
isOptional := false
for _, tag := range entry.Tags {
if tag == "optional" {
isOptional = true
break
}
}
if !isOptional && !installer.IsBinaryInPath(name) {
toInstall = append(toInstall, name)
}
}
if len(toInstall) > 0 {
printer.Println(" %s Installing %s binaries to: %s", terminal.SymbolBullet, terminal.Green(fmt.Sprintf("%d", len(toInstall))), terminal.Cyan(binariesFolder))
printer.Newline()
}
// Sort binary names for consistent display
sort.Strings(toInstall)
// Set silent mode for binary installation to reduce noise
_ = os.Setenv("OSMEDEUS_SILENT", "1")
defer func() { _ = os.Unsetenv("OSMEDEUS_SILENT") }()
// Suppress logger output during binary installation
logCfg := logger.DefaultConfig()
logCfg.Silent = true
_ = logger.Init(logCfg)
defer func() {
// Restore normal logging after installation
logCfg.Silent = false
_ = logger.Init(logCfg)
}()
// Install binaries in parallel with multi-row spinner display
var failedNames []string
if len(toInstall) > 0 {
failedNames = installBinariesParallel(toInstall, registry, binariesFolder, nil, printer, false)
}
// Count results
installedCount := len(toInstall) - len(failedNames)
failedCount := len(failedNames)
// Count skipped (already in PATH) - these weren't in toInstall
skippedCount := 0
for name, entry := range registry {
isOptional := false
for _, tag := range entry.Tags {
if tag == "optional" {
isOptional = true
break
}
}
if !isOptional && installer.IsBinaryInPath(name) {
skippedCount++
}
}
printer.Newline()
// Show summary
printer.Success("Installed %s binaries (%s skipped, %s failed)",
terminal.Green(fmt.Sprintf("%d", installedCount)),
terminal.Yellow(fmt.Sprintf("%d", skippedCount)),
terminal.Red(fmt.Sprintf("%d", failedCount)))
// Ensure binaries path is in environment
ensureBinariesPathInEnv(printer, binariesFolder, false)
return installedCount, skippedCount, failedCount
}
// createInitializationMarker creates the $HOME/.osmedeus/initialized marker file.
// This marker indicates that first-time setup has been completed.
func createInitializationMarker(printer *terminal.Printer) error {
homeDir, err := os.UserHomeDir()
if err != nil {
printer.Warning("Failed to get home directory: %s", err)
return err
}
osmDir := filepath.Join(homeDir, ".osmedeus")
if err := os.MkdirAll(osmDir, 0755); err != nil {
printer.Warning("Failed to create osmedeus config directory: %s", err)
return err
}
markerFile := filepath.Join(osmDir, "initialized")
if err := os.WriteFile(markerFile, []byte("initialized\n"), 0644); err != nil {
printer.Warning("Failed to create initialization marker: %s", err)
return err
}
return nil
}
// versionCmd shows version information
var versionCmd = &cobra.Command{
Use: "version",
@@ -392,15 +529,16 @@ func runFirstTimeSetup(baseFolder string, cfg *config.Config) error {
}
printer.Println(" %s Preset URL: %s", terminal.SymbolBullet, terminal.Cyan(presetURL))
// Check for custom workflow URL from environment
// Check for custom workflow URL from environment (default to DEFAULT_WORKFLOW_REPO)
workflowURL := os.Getenv("OSM_WORKFLOW_URL")
if workflowURL != "" {
printer.Println(" %s Workflow URL: %s", terminal.SymbolBullet, terminal.Cyan(workflowURL))
if workflowURL == "" {
workflowURL = core.DEFAULT_WORKFLOW_REPO
}
printer.Println(" %s Workflow URL: %s", terminal.SymbolBullet, terminal.Cyan(workflowURL))
printer.Newline()
// Step 2: Install preset base folder (workflows, etc.)
printer.Println("%s %s", terminal.BoldMagenta(terminal.SymbolLightning), terminal.HiMagenta("Installing preset workflows..."))
// Step 2: Install preset base folder
printer.Println("%s %s", terminal.BoldMagenta(terminal.SymbolLightning), terminal.HiMagenta("Installing preset base folder..."))
printer.Println(" %s Downloading from: %s", terminal.SymbolBullet, terminal.Cyan(presetURL))
inst := installer.NewInstaller(
baseFolder,
@@ -409,24 +547,23 @@ func runFirstTimeSetup(baseFolder string, cfg *config.Config) error {
nil,
)
if err := inst.InstallBase(presetURL); err != nil {
printer.Warning("Failed to install preset workflows: %s", err)
printer.Warning("Failed to install preset base: %s", err)
printer.Println(" %s", terminal.Gray("You can manually run: osmedeus install validate --preset"))
return err
}
printer.Success("Preset workflows installed to: %s", terminal.Cyan(filepath.Join(baseFolder, "workflows")))
printer.Success("Base folder installed to: %s", terminal.Cyan(baseFolder))
printer.Newline()
// Step 3: Install workflows from separate URL if specified
if workflowURL != "" {
printer.Println("%s %s", terminal.BoldBlue(terminal.SymbolLightning), terminal.HiBlue("Installing workflows from OSM_WORKFLOW_URL..."))
printer.Println(" %s Downloading from: %s", terminal.SymbolBullet, terminal.Cyan(workflowURL))
if err := inst.InstallWorkflow(workflowURL); err != nil {
printer.Warning("Failed to install workflows from OSM_WORKFLOW_URL: %s", err)
} else {
printer.Success("Workflows installed from: %s", terminal.Cyan(workflowURL))
}
printer.Newline()
// Step 3: Install workflows from workflow URL
printer.Println("%s %s", terminal.BoldBlue(terminal.SymbolLightning), terminal.HiBlue("Installing workflows..."))
printer.Println(" %s Downloading from: %s", terminal.SymbolBullet, terminal.Cyan(workflowURL))
if err := inst.InstallWorkflow(workflowURL); err != nil {
printer.Warning("Failed to install workflows: %s", err)
printer.Println(" %s", terminal.Gray("You can manually run: osmedeus install workflow --preset"))
} else {
printer.Success("Workflows installed to: %s", terminal.Cyan(filepath.Join(baseFolder, "workflows")))
}
printer.Newline()
// Step 4: Reload config and load workflows
printer.Println("%s %s", terminal.BoldMagenta(terminal.SymbolLightning), terminal.HiMagenta("Loading workflows..."))
@@ -446,123 +583,11 @@ func runFirstTimeSetup(baseFolder string, cfg *config.Config) error {
}
printer.Newline()
// Step 5: Install all binaries
// Check for registry URL override
registryURL := os.Getenv("OSM_REGISTRY_URL")
registryDisplay := "the default registry"
if registryURL != "" {
registryDisplay = registryURL
}
// Step 5: Install all binaries using shared helper
installRequiredBinaries(cfg, printer)
printer.Println("%s %s", terminal.BoldBlue(terminal.SymbolLightning),
terminal.HiBlue(fmt.Sprintf("Installing security binaries from %s", terminal.Cyan(registryDisplay))))
printer.Println(" %s This may take a few minutes depending on your network speed", terminal.SymbolBullet)
printer.Newline()
// Show spinner while loading registry
loadingSpinner := terminal.LoadingSpinner("Loading binary registry")
loadingSpinner.Start()
registry, err := installer.LoadRegistry(registryURL, nil)
loadingSpinner.Stop()
if err != nil {
printer.Warning("Failed to load binary registry: %s", err)
printer.Println(" %s", terminal.Gray("You can manually run: osmedeus install binary --all"))
return err
}
binariesFolder := cfg.BinariesPath
if binariesFolder == "" {
binariesFolder = filepath.Join(baseFolder, "external-binaries")
}
// Create binaries folder if it doesn't exist
if err := os.MkdirAll(binariesFolder, 0755); err != nil {
printer.Warning("Failed to create binaries folder: %s", err)
}
// Count binaries to install
var toInstall []string
for name, entry := range registry {
isOptional := false
for _, tag := range entry.Tags {
if tag == "optional" {
isOptional = true
break
}
}
if !isOptional && !installer.IsBinaryInPath(name) {
toInstall = append(toInstall, name)
}
}
if len(toInstall) > 0 {
printer.Println(" %s Installing %s binaries to: %s", terminal.SymbolBullet, terminal.Green(fmt.Sprintf("%d", len(toInstall))), terminal.Cyan(binariesFolder))
printer.Newline()
}
// Sort binary names for consistent display
sort.Strings(toInstall)
// Set silent mode for binary installation to reduce noise
_ = os.Setenv("OSMEDEUS_SILENT", "1")
defer func() { _ = os.Unsetenv("OSMEDEUS_SILENT") }()
// Suppress logger output during binary installation
logCfg := logger.DefaultConfig()
logCfg.Silent = true
_ = logger.Init(logCfg)
defer func() {
// Restore normal logging after installation
logCfg.Silent = false
_ = logger.Init(logCfg)
}()
// Install binaries in parallel with multi-row spinner display
var failed []string
if len(toInstall) > 0 {
failed = installBinariesParallel(toInstall, registry, binariesFolder, nil, printer, false)
}
// Count results
installedCount := len(toInstall) - len(failed)
failedCount := len(failed)
// Count skipped (already in PATH) - these weren't in toInstall
skippedCount := 0
for name, entry := range registry {
isOptional := false
for _, tag := range entry.Tags {
if tag == "optional" {
isOptional = true
break
}
}
if !isOptional && installer.IsBinaryInPath(name) {
skippedCount++
}
}
printer.Newline()
// Show summary
printer.Success("Installed %s binaries (%s skipped, %s failed)",
terminal.Green(fmt.Sprintf("%d", installedCount)),
terminal.Yellow(fmt.Sprintf("%d", skippedCount)),
terminal.Red(fmt.Sprintf("%d", failedCount)))
// Ensure binaries path is in environment
ensureBinariesPathInEnv(printer, binariesFolder, false)
// Create initialization marker file in $HOME/.osmedeus/
homeDir, _ := os.UserHomeDir()
osmDir := filepath.Join(homeDir, ".osmedeus")
if err := os.MkdirAll(osmDir, 0755); err != nil {
printer.Warning("Failed to create osmedeus config directory: %s", err)
}
markerFile := filepath.Join(osmDir, "initialized")
if err := os.WriteFile(markerFile, []byte("initialized\n"), 0644); err != nil {
printer.Warning("Failed to create initialization marker: %s", err)
}
// Create initialization marker file
_ = createInitializationMarker(printer)
// Print completion message
printer.Newline()
pkg/cli/run.go
+24 -24
View File
@@ -31,30 +31,30 @@ import (
)
var (
flowName string
moduleNames []string
targets []string
targetFile string
paramFlags []string
paramsFile string
workspacePath string
dryRun bool
threadsHold int
runTactic string
excludeModules []string
spaceName string
workspacesFolder string
heuristicsCheck string
distributedRun bool
redisURLRun string
concurrency int
repeatRun bool
repeatWaitTime string
runTimeout string
stdModule bool
emptyTarget bool
progressBar bool
disableWorkflowState bool
flowName string
moduleNames []string
targets []string
targetFile string
paramFlags []string
paramsFile string
workspacePath string
dryRun bool
threadsHold int
runTactic string
excludeModules []string
spaceName string
workspacesFolder string
heuristicsCheck string
distributedRun bool
redisURLRun string
concurrency int
repeatRun bool
repeatWaitTime string
runTimeout string
stdModule bool
emptyTarget bool
progressBar bool
disableWorkflowState bool
// explicitFlags tracks which CLI flags were explicitly set by the user
// Used to determine precedence when applying workflow preferences
pkg/cli/workflow.go
+3 -2
View File
@@ -1310,9 +1310,10 @@ func printValidationTable(results []ValidationResult, baseDir string) {
}
statusColor := terminal.Green
if r.Status == "failed" {
switch r.Status {
case "failed":
statusColor = terminal.Red
} else if r.Status == "warning" {
case "warning":
statusColor = terminal.Yellow
}
pkg/server/server.go
+1 -1
View File
@@ -15,7 +15,6 @@ import (
"github.com/gofiber/fiber/v2/middleware/logger"
"github.com/gofiber/fiber/v2/middleware/recover"
"github.com/gofiber/swagger"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/j3ssie/osmedeus/v5/internal/config"
"github.com/j3ssie/osmedeus/v5/internal/core"
"github.com/j3ssie/osmedeus/v5/internal/database"
@@ -24,6 +23,7 @@ import (
"github.com/j3ssie/osmedeus/v5/pkg/server/handlers"
"github.com/j3ssie/osmedeus/v5/pkg/server/middleware"
"github.com/j3ssie/osmedeus/v5/public"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
_ "github.com/j3ssie/osmedeus/v5/docs/api-swagger" // swagger docs
public/mics/install.sh
+34 -18
View File
@@ -9,6 +9,7 @@ OSM_HOME="${OSM_HOME:-$HOME/.osmedeus}"
BIN_DIR="$HOME/.local/bin"
GITHUB_REPO="j3ssie/osmedeus"
GITHUB_RELEASES="https://github.com/${GITHUB_REPO}/releases"
FALLBACK_VERSION="v5.0.0-beta"
OSM_URL_ENV_SET=0
if [[ -n "${OSM_URL+x}" ]]; then
OSM_URL_ENV_SET=1
@@ -144,8 +145,13 @@ downloader() {
download_file() {
local url="$1"
local output_file="$2"
local version="${3:-}"
log "Downloading $(basename "$output_file")..."
if [[ -n "$version" ]]; then
log "Downloading $(basename "$output_file") (${LIGHT_GREEN}${version}${NC})..."
else
log "Downloading $(basename "$output_file")..."
fi
# Use secure temporary file
local temp_file
@@ -173,23 +179,31 @@ verify_checksum() {
success "Checksum verified"
}
# Fetch latest CLI version from GitHub
# Fetch latest CLI version from GitHub API with fallback
fetch_latest_version() {
local api_url="https://api.github.com/repos/${GITHUB_REPO}/releases/latest"
local version
local tmp_file
local api_url="https://api.github.com/repos/${GITHUB_REPO}/releases/latest"
local version
local tmp_file
log "Fetching latest version from GitHub..."
tmp_file=$(mktemp)
downloader "$api_url" "$tmp_file"
version=$(grep '"tag_name":' "$tmp_file" | head -n 1 | sed -E 's/.*"([^"]+)".*/\1/')
rm -f "$tmp_file"
log "Fetching latest version from GitHub..."
tmp_file=$(mktemp)
if [[ -z "$version" ]]; then
error "Failed to fetch latest version from GitHub"
fi
# Try to fetch from GitHub API
if downloader "$api_url" "$tmp_file" 2>/dev/null; then
version=$(grep '"tag_name":' "$tmp_file" | head -n 1 | sed -E 's/.*"([^"]+)".*/\1/')
rm -f "$tmp_file"
echo "$version"
if [[ -n "$version" ]]; then
echo "$version"
return
fi
fi
rm -f "$tmp_file" 2>/dev/null
# Fall back to hardcoded version
warn "Failed to fetch from GitHub API, using fallback: $FALLBACK_VERSION"
echo "$FALLBACK_VERSION"
}
fetch_latest_version_from_metadata() {
@@ -259,9 +273,11 @@ install_osmedeus_binary() {
if [[ "$version" != v* ]]; then
version="v${version}"
fi
log "Installing version: $version"
log "Installing version: ${LIGHT_GREEN}${version}${NC}"
local tarball_name="osmedeus_${version}_${platform}.tar.gz"
# Strip 'v' prefix for tarball filename (e.g., v5.0.0 -> 5.0.0)
local version_no_v="${version#v}"
local tarball_name="osmedeus_${version_no_v}_${platform}.tar.gz"
local base_url
if [[ $OSM_URL_ENV_SET -eq 1 && -n "${OSM_URL}" ]]; then
base_url="${OSM_URL%/}"
@@ -282,7 +298,7 @@ install_osmedeus_binary() {
mkdir -p "$extract_dir"
# Download checksum first
download_file "$checksum_url" "$checksum_path"
download_file "$checksum_url" "$checksum_path" "$version"
# Extract expected checksum for our tarball
local expected_checksum
@@ -293,7 +309,7 @@ install_osmedeus_binary() {
fi
# Download tarball
download_file "$tarball_url" "$tarball_path"
download_file "$tarball_url" "$tarball_path" "$version"
# Verify checksum
verify_checksum "$tarball_path" "$expected_checksum"
public/presets/registry-metadata-direct-fetch.json
+1 -1
View File
@@ -553,7 +553,7 @@
"multi-commands-linux": [
"git clone --depth=1 https://github.com/blechschmidt/massdns.git /tmp/massdns",
"cd /tmp/massdns && make",
"cp /tmp/massdns/bin/massdns $HOME/.local/bin/"
"mkdir -p $HOME/.local/bin/ && cp /tmp/massdns/bin/massdns $HOME/.local/bin/"
],
"multi-commands-darwin": [
"<auto_detect_package_manager> install massdns"