osmedeus/internal/executor/foreach_executor.go

package executor

import (
	"bufio"
	"context"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/j3ssie/osmedeus/v5/internal/core"
	"github.com/j3ssie/osmedeus/v5/internal/template"
)

// ForeachExecutor executes foreach steps
type ForeachExecutor struct {
	dispatcher     *StepDispatcher
	templateEngine template.TemplateEngine
}

// NewForeachExecutor creates a new foreach executor
func NewForeachExecutor(dispatcher *StepDispatcher, engine template.TemplateEngine) *ForeachExecutor {
	return &ForeachExecutor{
		dispatcher:     dispatcher,
		templateEngine: engine,
	}
}

// Name returns the executor name for logging/debugging
func (e *ForeachExecutor) Name() string {
	return "foreach"
}

// StepTypes returns the step types this executor handles
func (e *ForeachExecutor) StepTypes() []core.StepType {
	return []core.StepType{core.StepTypeForeach}
}

// Execute executes a foreach step
func (e *ForeachExecutor) Execute(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) {
	result := &core.StepResult{
		StepName:  step.Name,
		Status:    core.StepStatusRunning,
		StartTime: time.Now(),
		Exports:   make(map[string]interface{}),
	}

	if step.Step == nil {
		result.Status = core.StepStatusFailed
		result.Error = fmt.Errorf("foreach step has no inner step")
		result.EndTime = time.Now()
		return result, result.Error
	}

	threads, err := step.Threads.Int()
	if err != nil {
		result.Status = core.StepStatusFailed
		result.Error = err
		result.EndTime = time.Now()
		result.Duration = result.EndTime.Sub(result.StartTime)
		return result, err
	}
	if threads <= 0 {
		threads = 1
	}

	// Execute with streaming worker pool
	outputs, err := e.executeWithWorkerPool(ctx, step, step.Input, threads, execCtx)

	result.Output = strings.Join(outputs, "\n")
	result.EndTime = time.Now()
	result.Duration = result.EndTime.Sub(result.StartTime)

	if err != nil {
		result.Status = core.StepStatusFailed
		result.Error = err
		return result, err
	}

	result.Status = core.StepStatusSuccess
	return result, nil
}

// LineIterator provides streaming access to lines in a file
type LineIterator struct {
	file    *os.File
	scanner *bufio.Scanner
	current string
	err     error
}

// NewLineIterator creates an iterator for reading lines from a file
func NewLineIterator(path string) (*LineIterator, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(file)
	// Increase scanner buffer for long lines
	scanner.Buffer(make([]byte, 64*1024), 10*1024*1024)

	return &LineIterator{
		file:    file,
		scanner: scanner,
	}, nil
}

// Next advances to the next non-empty line, returns false when done
func (it *LineIterator) Next() bool {
	for it.scanner.Scan() {
		line := strings.TrimSpace(it.scanner.Text())
		if line != "" {
			it.current = line
			return true
		}
	}
	it.err = it.scanner.Err()
	return false
}

// Value returns the current line
func (it *LineIterator) Value() string {
	return it.current
}

// Err returns any error encountered during iteration
func (it *LineIterator) Err() error {
	return it.err
}

// Close closes the underlying file
func (it *LineIterator) Close() error {
	if it.file != nil {
		return it.file.Close()
	}
	return nil
}

// countInputLines counts non-empty lines in an input file (for result slice allocation)
func countInputLines(path string) (int, error) {
	file, err := os.Open(path)
	if err != nil {
		return 0, err
	}
	defer func() { _ = file.Close() }()

	count := 0
	scanner := bufio.NewScanner(file)
	scanner.Buffer(make([]byte, 64*1024), 10*1024*1024)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line != "" {
			count++
		}
	}
	return count, scanner.Err()
}

// renderSecondaryTemplates clones the step and renders [[ ]] templates with loop context
func (e *ForeachExecutor) renderSecondaryTemplates(step *core.Step, execCtx *core.ExecutionContext) *core.Step {
	// Clone the step to avoid modifying the original
	cloned := step.Clone()
	ctx := execCtx.GetVariables()

	// Render Command if it has secondary variables
	if e.templateEngine.HasSecondaryVariable(cloned.Command) {
		rendered, err := e.templateEngine.RenderSecondary(cloned.Command, ctx)
		if err == nil {
			cloned.Command = rendered
		}
	}

	// Render Commands array
	for i, cmd := range cloned.Commands {
		if e.templateEngine.HasSecondaryVariable(cmd) {
			rendered, err := e.templateEngine.RenderSecondary(cmd, ctx)
			if err == nil {
				cloned.Commands[i] = rendered
			}
		}
	}

	// Render Input (for nested foreach)
	if e.templateEngine.HasSecondaryVariable(cloned.Input) {
		rendered, err := e.templateEngine.RenderSecondary(cloned.Input, ctx)
		if err == nil {
			cloned.Input = rendered
		}
	}

	return cloned
}

// workItem represents a single item to process in the worker pool
type workItem struct {
	index int
	value string
}

// workResult represents the result of processing a work item
type workResult struct {
	index  int
	output string
	err    error
}

// executeWithWorkerPool executes the inner step using a streaming worker pool pattern
// This is memory-efficient: creates only 'threads' goroutines instead of N goroutines
// and streams input lines on-demand instead of loading all into memory
func (e *ForeachExecutor) executeWithWorkerPool(ctx context.Context, step *core.Step, inputPath string, threads int, execCtx *core.ExecutionContext) ([]string, error) {
	// Count lines first for result slice allocation (fast, O(n) with minimal memory)
	lineCount, err := countInputLines(inputPath)
	if err != nil {
		return nil, fmt.Errorf("failed to count input lines: %w", err)
	}

	if lineCount == 0 {
		return nil, nil
	}

	// Create bounded work queue - buffer 2x thread count for smooth flow
	workQueue := make(chan workItem, threads*2)
	results := make(chan workResult, threads*2)

	// Track completion
	var workerWg sync.WaitGroup
	var producerErr error

	// Start fixed worker pool (only 'threads' goroutines, not N)
	for i := 0; i < threads; i++ {
		workerWg.Add(1)
		go func() {
			defer workerWg.Done()
			for work := range workQueue {
				// Check context cancellation
				if ctx.Err() != nil {
					results <- workResult{index: work.index, err: ctx.Err()}
					continue
				}

				// Create optimized child context with loop variables pre-set
				childCtx := execCtx.CloneForLoop(step.Variable, work.value, work.index+1)

				// Clone inner step and render secondary templates [[ ]]
				innerStep := e.renderSecondaryTemplates(step.Step, childCtx)

				// Execute inner step
				stepResult, err := e.dispatcher.Dispatch(ctx, innerStep, childCtx)

				var output string
				if stepResult != nil {
					output = stepResult.Output
				}

				results <- workResult{index: work.index, output: output, err: err}
			}
		}()
	}

	// Producer: stream lines into work queue (separate goroutine)
	go func() {
		defer close(workQueue)

		iter, err := NewLineIterator(inputPath)
		if err != nil {
			producerErr = err
			return
		}
		defer func() { _ = iter.Close() }()

		idx := 0
		for iter.Next() {
			select {
			case workQueue <- workItem{index: idx, value: iter.Value()}:
				idx++
			case <-ctx.Done():
				producerErr = ctx.Err()
				return
			}
		}

		if iter.Err() != nil {
			producerErr = iter.Err()
		}
	}()

	// Collector: close results when all workers done
	go func() {
		workerWg.Wait()
		close(results)
	}()

	// Collect results in order
	outputs := make([]string, lineCount)
	var firstError error

	for r := range results {
		if r.index < len(outputs) {
			outputs[r.index] = r.output
		}
		if r.err != nil && firstError == nil {
			firstError = r.err
		}
	}

	// Check for producer error
	if producerErr != nil && firstError == nil {
		firstError = producerErr
	}

	return outputs, firstError
}

// CanHandle returns true if this executor can handle the given step type
func (e *ForeachExecutor) CanHandle(stepType core.StepType) bool {
	return stepType == core.StepTypeForeach
}