mirror of
https://github.com/j3ssie/osmedeus.git
synced 2026-08-20 06:35:48 +02:00
- Add worker eval command for distributed function execution with Redis hooks registration - Add worker set command to update worker fields (alias, public-ip, ssh-enabled, ssh-keys-path) - Enhance worker status with JSON output, search filtering, and column selection (--columns, --exclude-columns, --search) - Add --keep-setting flag to install base/validate commands to preserve osm-settings.yaml after base installation - Fix binary installation in Nix: replace CopyInstalledBinaryToFolder with SymlinkInstalledBinaryToFolder - Add --clean-ws flag to db clean command for removing workspace data - Add HooksEnabled field to Run records when creating runs from CLI and API - Add comprehensive test coverage for hook execution (pre/post hooks, execution order, failure handling) - Add test coverage for worker commands (eval, set, status with JSON) and db clean operations - Improve usage documentation for worker subcommands and db operations
373 lines
11 KiB
Go
373 lines
11 KiB
Go
package executor
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/j3ssie/osmedeus/v5/internal/core"
|
|
"github.com/j3ssie/osmedeus/v5/internal/metrics"
|
|
"github.com/j3ssie/osmedeus/v5/internal/runner"
|
|
"github.com/j3ssie/osmedeus/v5/internal/template"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
// BashExecutor executes bash steps
|
|
type BashExecutor struct {
|
|
templateEngine template.TemplateEngine
|
|
runner runner.Runner
|
|
}
|
|
|
|
// NewBashExecutor creates a new bash executor
|
|
func NewBashExecutor(engine template.TemplateEngine) *BashExecutor {
|
|
return &BashExecutor{
|
|
templateEngine: engine,
|
|
}
|
|
}
|
|
|
|
// Name returns the executor name for logging/debugging
|
|
func (e *BashExecutor) Name() string {
|
|
return "bash"
|
|
}
|
|
|
|
// StepTypes returns the step types this executor handles
|
|
func (e *BashExecutor) StepTypes() []core.StepType {
|
|
return []core.StepType{core.StepTypeBash}
|
|
}
|
|
|
|
// SetRunner sets the runner for command execution
|
|
func (e *BashExecutor) SetRunner(r runner.Runner) {
|
|
e.runner = r
|
|
}
|
|
|
|
// assembleCommand joins the command with structured args in order:
|
|
// command + speed_args + config_args + input_args + output_args
|
|
func assembleCommand(command, speedArgs, configArgs, inputArgs, outputArgs string) string {
|
|
parts := []string{command}
|
|
if speedArgs != "" {
|
|
parts = append(parts, speedArgs)
|
|
}
|
|
if configArgs != "" {
|
|
parts = append(parts, configArgs)
|
|
}
|
|
if inputArgs != "" {
|
|
parts = append(parts, inputArgs)
|
|
}
|
|
if outputArgs != "" {
|
|
parts = append(parts, outputArgs)
|
|
}
|
|
return strings.Join(parts, " ")
|
|
}
|
|
|
|
// writeStdFile writes command output to the specified file
|
|
func writeStdFile(path, content string) error {
|
|
// Ensure parent directory exists
|
|
if dir := filepath.Dir(path); dir != "" && dir != "." {
|
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
|
return fmt.Errorf("failed to create directory: %w", err)
|
|
}
|
|
}
|
|
return os.WriteFile(path, []byte(content), 0644)
|
|
}
|
|
|
|
// extractToolName extracts the tool/binary name from a command string.
|
|
// Returns the base name of the first word in the command.
|
|
func extractToolName(command string) string {
|
|
parts := strings.Fields(command)
|
|
if len(parts) == 0 {
|
|
return "unknown"
|
|
}
|
|
return filepath.Base(parts[0])
|
|
}
|
|
|
|
// Execute executes a bash step
|
|
func (e *BashExecutor) Execute(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) {
|
|
result := &core.StepResult{
|
|
StepName: step.Name,
|
|
Status: core.StepStatusRunning,
|
|
StartTime: time.Now(),
|
|
}
|
|
|
|
timeout, err := step.Timeout.Duration()
|
|
if err != nil {
|
|
result.Status = core.StepStatusFailed
|
|
result.Error = err
|
|
result.EndTime = time.Now()
|
|
result.Duration = result.EndTime.Sub(result.StartTime)
|
|
return result, err
|
|
}
|
|
|
|
// Extract binaries path for fallback resolution
|
|
binariesPath := ""
|
|
if bp, ok := execCtx.GetVariable("Binaries"); ok {
|
|
if bpStr, ok := bp.(string); ok {
|
|
binariesPath = bpStr
|
|
}
|
|
}
|
|
|
|
var output string
|
|
|
|
// Determine execution mode
|
|
if len(step.ParallelCommands) > 0 {
|
|
output, err = e.executeParallel(ctx, step.ParallelCommands, timeout, binariesPath)
|
|
} else if len(step.Commands) > 0 {
|
|
output, err = e.executeSequential(ctx, step.Commands, timeout, binariesPath)
|
|
} else if step.Command != "" {
|
|
// Assemble command with structured args if present
|
|
finalCmd := assembleCommand(step.Command, step.SpeedArgs, step.ConfigArgs, step.InputArgs, step.OutputArgs)
|
|
output, err = e.executeCommandWithFallback(ctx, finalCmd, timeout, binariesPath)
|
|
} else {
|
|
err = fmt.Errorf("no command specified")
|
|
}
|
|
|
|
result.Output = output
|
|
result.EndTime = time.Now()
|
|
result.Duration = result.EndTime.Sub(result.StartTime)
|
|
|
|
// Write stdout/stderr to file if std_file is specified
|
|
if step.StdFile != "" {
|
|
if writeErr := writeStdFile(step.StdFile, output); writeErr != nil {
|
|
// Log warning but don't fail the step
|
|
execCtx.Logger.Warn("Failed to write std_file",
|
|
zap.String("path", step.StdFile),
|
|
zap.Error(writeErr))
|
|
}
|
|
}
|
|
|
|
if err != nil {
|
|
result.Status = core.StepStatusFailed
|
|
result.Error = err
|
|
return result, err
|
|
}
|
|
|
|
result.Status = core.StepStatusSuccess
|
|
return result, nil
|
|
}
|
|
|
|
// executeCommand executes a single command
|
|
|
|
func (e *BashExecutor) executeCommand(ctx context.Context, command string, timeout time.Duration) (string, error) {
|
|
// Track execution timing for metrics
|
|
startTime := time.Now()
|
|
toolName := extractToolName(command)
|
|
|
|
// Apply timeout if specified
|
|
if timeout > 0 {
|
|
var cancel context.CancelFunc
|
|
ctx, cancel = context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
}
|
|
|
|
// Use runner if available, otherwise fall back to local execution
|
|
if e.runner != nil {
|
|
result, err := e.runner.Execute(ctx, command)
|
|
duration := time.Since(startTime).Seconds()
|
|
|
|
// Check for context cancellation first (Ctrl+C or timeout)
|
|
// This must be checked before other errors because runners return nil error
|
|
// but set result.ExitCode to -1 when context is cancelled
|
|
if ctx.Err() != nil {
|
|
if ctx.Err() == context.DeadlineExceeded {
|
|
metrics.RecordToolExecution(toolName, "timeout", duration)
|
|
return result.Output, fmt.Errorf("command timed out after %s", timeout)
|
|
}
|
|
metrics.RecordToolExecution(toolName, "cancelled", duration)
|
|
return result.Output, ctx.Err()
|
|
}
|
|
|
|
if err != nil {
|
|
metrics.RecordToolExecution(toolName, "error", duration)
|
|
return result.Output, fmt.Errorf("command failed: %w", err)
|
|
}
|
|
if result.ExitCode != 0 {
|
|
metrics.RecordToolExecution(toolName, "failed", duration)
|
|
return result.Output, newExitCodeErrorf(result.ExitCode, "command exited with code %d", result.ExitCode)
|
|
}
|
|
metrics.RecordToolExecution(toolName, "success", duration)
|
|
return strings.TrimSpace(result.Output), nil
|
|
}
|
|
|
|
// Fallback to local execution
|
|
// @NOTE: yes yes, I know this is a security risk. This is the intended behavior as it suppose to be that flexible so you should think twice of what you will run or design your workflow better
|
|
cmd := exec.CommandContext(ctx, "sh", "-c", command)
|
|
|
|
stdout := runner.NewLimitedBuffer(runner.MaxOutputSize)
|
|
stderr := runner.NewLimitedBuffer(runner.MaxStderrSize)
|
|
cmd.Stdout = stdout
|
|
cmd.Stderr = stderr
|
|
|
|
err := cmd.Run()
|
|
duration := time.Since(startTime).Seconds()
|
|
|
|
output := string(stdout.Bytes())
|
|
if stderr.Len() > 0 {
|
|
output += "\n" + string(stderr.Bytes())
|
|
}
|
|
if stdout.Overflow() || stderr.Overflow() {
|
|
output += "\n[output truncated]"
|
|
}
|
|
|
|
// Check for context cancellation first (Ctrl+C or timeout)
|
|
if ctx.Err() != nil {
|
|
if ctx.Err() == context.DeadlineExceeded {
|
|
metrics.RecordToolExecution(toolName, "timeout", duration)
|
|
return output, fmt.Errorf("command timed out after %s", timeout)
|
|
}
|
|
metrics.RecordToolExecution(toolName, "cancelled", duration)
|
|
return output, ctx.Err()
|
|
}
|
|
|
|
if err != nil {
|
|
// Extract exit code if available for fallback detection
|
|
var exitErr *exec.ExitError
|
|
if errors.As(err, &exitErr) {
|
|
exitCode := exitErr.ExitCode()
|
|
metrics.RecordToolExecution(toolName, "failed", duration)
|
|
return output, newExitCodeErrorf(exitCode, "command failed with exit code %d\nstderr: %s", exitCode, string(stderr.Bytes()))
|
|
}
|
|
metrics.RecordToolExecution(toolName, "error", duration)
|
|
return output, fmt.Errorf("command failed: %w\nstderr: %s", err, string(stderr.Bytes()))
|
|
}
|
|
|
|
metrics.RecordToolExecution(toolName, "success", duration)
|
|
return strings.TrimSpace(output), nil
|
|
}
|
|
|
|
// executeCommandWithFallback wraps executeCommand with automatic retry on exit code 127.
|
|
// Fallback 1: strip timeout prefix. Fallback 2: prepend binariesPath to the binary.
|
|
func (e *BashExecutor) executeCommandWithFallback(ctx context.Context, command string, timeout time.Duration, binariesPath string) (string, error) {
|
|
output, err := e.executeCommand(ctx, command, timeout)
|
|
if err == nil {
|
|
return output, nil
|
|
}
|
|
|
|
// Only attempt fallback on exit code 127 (command not found)
|
|
var ecErr *exitCodeError
|
|
if !errors.As(err, &ecErr) || ecErr.code != 127 {
|
|
return output, err
|
|
}
|
|
|
|
// Don't retry if context is already cancelled
|
|
if ctx.Err() != nil {
|
|
return output, err
|
|
}
|
|
|
|
currentCmd := command
|
|
|
|
// Fallback 1: strip timeout prefix
|
|
if result := stripTimeoutPrefix(currentCmd); result.stripped && result.command != "" {
|
|
// Use parsed duration from timeout prefix as fallback if step timeout is not set
|
|
retryTimeout := timeout
|
|
if retryTimeout == 0 && result.duration > 0 {
|
|
retryTimeout = result.duration
|
|
}
|
|
output, err = e.executeCommand(ctx, result.command, retryTimeout)
|
|
if err == nil {
|
|
return output, nil
|
|
}
|
|
// Check if still 127 for next fallback
|
|
if !errors.As(err, &ecErr) || ecErr.code != 127 {
|
|
return output, err
|
|
}
|
|
currentCmd = result.command
|
|
}
|
|
|
|
// Fallback 2: prepend binaries path
|
|
if prepended, ok := prependBinariesPath(currentCmd, binariesPath); ok {
|
|
output, err = e.executeCommand(ctx, prepended, timeout)
|
|
return output, err
|
|
}
|
|
|
|
return output, err
|
|
}
|
|
|
|
// executeSequential executes commands sequentially
|
|
func (e *BashExecutor) executeSequential(ctx context.Context, commands []string, timeout time.Duration, binariesPath string) (string, error) {
|
|
var outputs []string
|
|
|
|
for _, cmd := range commands {
|
|
output, err := e.executeCommandWithFallback(ctx, cmd, timeout, binariesPath)
|
|
outputs = append(outputs, output)
|
|
if err != nil {
|
|
return strings.Join(outputs, "\n"), err
|
|
}
|
|
}
|
|
|
|
return strings.Join(outputs, "\n"), nil
|
|
}
|
|
|
|
// executeParallel executes commands in parallel with bounded concurrency.
|
|
// Uses a worker pool capped at runtime.NumCPU()*2 to prevent unbounded goroutine/memory growth.
|
|
func (e *BashExecutor) executeParallel(ctx context.Context, commands []string, timeout time.Duration, binariesPath string) (string, error) {
|
|
type result struct {
|
|
index int
|
|
output string
|
|
err error
|
|
}
|
|
|
|
maxWorkers := runtime.NumCPU() * 2
|
|
if maxWorkers > len(commands) {
|
|
maxWorkers = len(commands)
|
|
}
|
|
|
|
type workItem struct {
|
|
index int
|
|
command string
|
|
}
|
|
|
|
workQueue := make(chan workItem, maxWorkers)
|
|
results := make(chan result, len(commands))
|
|
var wg sync.WaitGroup
|
|
|
|
// Start bounded worker pool
|
|
for range maxWorkers {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for work := range workQueue {
|
|
output, err := e.executeCommandWithFallback(ctx, work.command, timeout, binariesPath)
|
|
results <- result{index: work.index, output: output, err: err}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Feed work items
|
|
go func() {
|
|
for i, cmd := range commands {
|
|
workQueue <- workItem{index: i, command: cmd}
|
|
}
|
|
close(workQueue)
|
|
}()
|
|
|
|
// Wait for all workers to complete
|
|
go func() {
|
|
wg.Wait()
|
|
close(results)
|
|
}()
|
|
|
|
// Collect results
|
|
outputs := make([]string, len(commands))
|
|
var firstError error
|
|
|
|
for r := range results {
|
|
outputs[r.index] = r.output
|
|
if r.err != nil && firstError == nil {
|
|
firstError = r.err
|
|
}
|
|
}
|
|
|
|
return strings.Join(outputs, "\n"), firstError
|
|
}
|
|
|
|
// CanHandle returns true if this executor can handle the given step type
|
|
func (e *BashExecutor) CanHandle(stepType core.StepType) bool {
|
|
return stepType == core.StepTypeBash
|
|
}
|