feat: add LLM step executor with vision and tool support, event workflow system, and inheritance

- Add LLM executor supporting OpenAI vision, tool calling, embeddings, and structured outputs
- Introduce event emitter/receiver workflows with deduplication and filtering (generate_event functions)
- Add workflow extends/override system enabling inheritance chains and step merge modes
- Update function naming to snake_case across all testdata (fileExists→file_exists, etc.)
- Add comprehensive test fixtures for linter, events, CDN, step dependencies, and extends workflows
This commit is contained in:
j3ssie
2026-01-20 18:23:57 +08:00
parent 9ed02e7eee
commit 1403d20a4d
219 changed files with 25734 additions and 1696 deletions
+138 -15
View File
@@ -1,10 +1,12 @@
package cli
import (
"bufio"
"fmt"
"io"
"os"
"strings"
"sync"
"github.com/j3ssie/osmedeus/v5/internal/config"
"github.com/j3ssie/osmedeus/v5/internal/database"
@@ -23,6 +25,11 @@ var (
funcSearchFilter string
funcColumnWidth int
funcShowExample bool
// Bulk processing flags
funcTargetsFile string
funcFunctionFile string
funcConcurrency int
)
// functionCmd is the parent command for function operations
@@ -57,8 +64,13 @@ func init() {
functionEvalCmd.Flags().BoolVar(&evalStdin, "stdin", false, "read script from stdin")
functionEvalCmd.Flags().StringVarP(&evalFunctionName, "function", "f", "", "function name to call (remaining args become function arguments)")
// Bulk processing flags
functionEvalCmd.Flags().StringVarP(&funcTargetsFile, "targets", "T", "", "file containing targets (one per line)")
functionEvalCmd.Flags().StringVar(&funcFunctionFile, "function-file", "", "file containing the function/script to execute")
functionEvalCmd.Flags().IntVarP(&funcConcurrency, "concurrency", "c", 1, "number of concurrent executions")
functionListCmd.Flags().StringVarP(&funcSearchFilter, "search", "s", "", "filter functions by name or description")
functionListCmd.Flags().IntVar(&funcColumnWidth, "width", 0, "max column width (wraps lines instead of truncating)")
functionListCmd.Flags().IntVar(&funcColumnWidth, "width", 60, "max column width (wraps lines instead of truncating)")
functionListCmd.Flags().BoolVar(&funcShowExample, "example", false, "show example usage below each function description")
functionCmd.AddCommand(functionEvalCmd)
@@ -81,11 +93,19 @@ func runFunctionEval(cmd *cobra.Command, args []string) error {
}
}
// Determine script source: -f flag > positional arg > -e flag > stdin
// Determine script source: --function-file > -f flag > positional arg > -e flag > stdin
var script string
// Handle -f/--function flag: build script from function name + positional args
if evalFunctionName != "" {
// Read script from function file if provided
if funcFunctionFile != "" {
data, err := os.ReadFile(funcFunctionFile)
if err != nil {
printer.Error("Failed to read function file: %s", err)
return fmt.Errorf("failed to read function file: %w", err)
}
script = strings.TrimSpace(string(data))
} else if evalFunctionName != "" {
// Handle -f/--function flag: build script from function name + positional args
var quotedArgs []string
for _, arg := range args {
quotedArgs = append(quotedArgs, fmt.Sprintf("%q", arg))
@@ -118,16 +138,86 @@ func runFunctionEval(cmd *cobra.Command, args []string) error {
}
if script == "" {
return fmt.Errorf("no script provided: use positional argument, -e flag, or --stdin")
return fmt.Errorf("no script provided: use positional argument, -e flag, --function-file, or --stdin")
}
// Use the resolved script
evalScript = script
// Bulk processing mode: process multiple targets from file
if funcTargetsFile != "" {
return runBulkFunctionEval(printer, script)
}
// 1. Build context with target and params
// Single target execution (existing behavior)
return executeFunctionForTarget(printer, script, evalTarget)
}
// runBulkFunctionEval processes the script for multiple targets from a file
func runBulkFunctionEval(printer *terminal.Printer, script string) error {
// Read targets from file
targets, err := readFuncTargetsFromFile(funcTargetsFile)
if err != nil {
printer.Error("Failed to read targets file: %s", err)
return fmt.Errorf("failed to read targets file: %w", err)
}
if len(targets) == 0 {
printer.Warning("No targets found in file: %s", funcTargetsFile)
return nil
}
// Deduplicate targets
targets = deduplicateFuncTargets(targets)
if verbose {
printer.Info("Processing %d targets with concurrency %d", len(targets), funcConcurrency)
}
// Ensure concurrency is at least 1
maxConcurrency := funcConcurrency
if maxConcurrency < 1 {
maxConcurrency = 1
}
// Process targets concurrently
sem := make(chan struct{}, maxConcurrency)
var wg sync.WaitGroup
var errCount int
var errMu sync.Mutex
for _, target := range targets {
wg.Add(1)
go func(t string) {
defer wg.Done()
sem <- struct{}{} // Acquire semaphore
defer func() { <-sem }() // Release semaphore
if err := executeFunctionForTarget(printer, script, t); err != nil {
errMu.Lock()
errCount++
errMu.Unlock()
if verbose {
printer.Error("Failed for target %s: %s", t, err)
}
}
}(target)
}
wg.Wait()
if errCount > 0 {
printer.Warning("Completed with %d errors out of %d targets", errCount, len(targets))
} else if verbose {
printer.Success("Successfully processed %d targets", len(targets))
}
return nil
}
// executeFunctionForTarget executes the script for a single target
func executeFunctionForTarget(printer *terminal.Printer, script, target string) error {
// Build context with target and params
ctx := make(map[string]interface{})
if evalTarget != "" {
ctx["target"] = evalTarget
if target != "" {
ctx["target"] = target
}
for _, p := range evalParams {
@@ -137,20 +227,20 @@ func runFunctionEval(cmd *cobra.Command, args []string) error {
}
}
// 2. Render template variables ({{target}}, etc.)
// Render template variables ({{target}}, etc.)
templateEngine := template.NewEngine()
renderedScript, err := templateEngine.Render(evalScript, ctx)
renderedScript, err := templateEngine.Render(script, ctx)
if err != nil {
printer.Error("Template rendering failed: %s", err)
return fmt.Errorf("template rendering failed: %w", err)
}
// Show rendered script if different from original (verbose mode)
if verbose && renderedScript != evalScript {
if verbose && renderedScript != script {
printer.Info("Rendered script: %s", renderedScript)
}
// 3. Execute as JavaScript using Otto runtime
// Execute as JavaScript using Otto runtime
registry := functions.NewRegistry()
result, err := registry.Execute(renderedScript, ctx)
if err != nil {
@@ -158,7 +248,7 @@ func runFunctionEval(cmd *cobra.Command, args []string) error {
return fmt.Errorf("execution failed: %w", err)
}
// 4. Print result
// Print result
if result != nil {
fmt.Println(result)
}
@@ -166,6 +256,39 @@ func runFunctionEval(cmd *cobra.Command, args []string) error {
return nil
}
// readFuncTargetsFromFile reads targets from a file, one per line
func readFuncTargetsFromFile(filepath string) ([]string, error) {
file, err := os.Open(filepath)
if err != nil {
return nil, err
}
defer func() { _ = file.Close() }()
var result []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" && !strings.HasPrefix(line, "#") {
result = append(result, line)
}
}
return result, scanner.Err()
}
// deduplicateFuncTargets removes duplicates and empty strings
func deduplicateFuncTargets(inputTargets []string) []string {
seen := make(map[string]bool)
var result []string
for _, t := range inputTargets {
t = strings.TrimSpace(t)
if t != "" && !seen[t] {
seen[t] = true
result = append(result, t)
}
}
return result
}
func runFunctionList(cmd *cobra.Command, args []string) error {
printer := terminal.NewPrinter()