diff --git a/HACKING.md b/HACKING.md index abe64f4..c048071 100644 --- a/HACKING.md +++ b/HACKING.md @@ -23,113 +23,43 @@ This document describes the technical architecture and development practices for ## Project Structure ``` -osmedeus-ng/ -├── cmd/ -│ └── osmedeus/ -│ └── main.go # Application entry point -├── internal/ # Private packages -│ ├── config/ # Configuration management -│ │ ├── config.go # Config loading and defaults -│ │ ├── settings.go # Settings struct definitions -│ │ └── secrets.go # Secret management -│ ├── core/ # Core types and interfaces -│ │ ├── workflow.go # Workflow and RunnerConfig structs -│ │ ├── step.go # Step definition -│ │ ├── trigger.go # Trigger types and events -│ │ ├── types.go # Status types, constants -│ │ ├── context.go # Execution context -│ │ └── constants.go # Project metadata -│ ├── parser/ # Workflow parsing -│ │ ├── parser.go # YAML parser -│ │ ├── loader.go # Workflow loader with caching -│ │ └── validator.go # Workflow validation -│ ├── executor/ # Workflow execution -│ │ ├── executor.go # Main executor -│ │ ├── dispatcher.go # Step dispatcher -│ │ ├── bash_executor.go # Bash step handler -│ │ ├── function_executor.go # Function step handler -│ │ ├── foreach_executor.go # Foreach step handler -│ │ ├── parallel_executor.go # Parallel step handler -│ │ ├── remote_bash_executor.go # Remote-bash step handler -│ │ ├── http_executor.go # HTTP request handler -│ │ └── llm_executor.go # LLM step handler -│ ├── runner/ # Execution environments -│ │ ├── runner.go # Runner interface -│ │ ├── host_runner.go # Local execution -│ │ ├── docker_runner.go # Docker execution -│ │ └── ssh_runner.go # SSH remote execution -│ ├── template/ # Template rendering -│ │ ├── engine.go # Template engine -│ │ ├── generators.go # Value generators -│ │ └── context.go # Template context -│ ├── functions/ # Utility functions -│ │ ├── registry.go # Function registry -│ │ ├── goja_runtime.go # JavaScript runtime (Goja VM) -│ │ ├── file_functions.go # File operations -│ │ ├── string_functions.go # String operations -│ │ ├── util_functions.go # Utility functions -│ │ ├── event_functions.go # Event generation functions -│ │ └── jq.go # JSON query functions -│ ├── scheduler/ # Trigger scheduling -│ │ └── scheduler.go # Cron, event, watch triggers -│ ├── database/ # Data persistence -│ │ ├── database.go # Connection management -│ │ ├── models.go # Data models -│ │ ├── jsonl.go # JSONL import/export -│ │ └── repository/ # Data access layer -│ ├── heuristics/ # Target analysis -│ │ ├── heuristics.go # Target type detection -│ │ ├── url.go # URL parsing -│ │ └── domain.go # Domain analysis -│ ├── linter/ # Workflow linting -│ │ ├── linter.go # Main linter logic -│ │ ├── rules.go # Linting rules (built-in variables, etc.) -│ │ ├── formatter.go # Output formatters (pretty, JSON, GitHub) -│ │ ├── ast.go # Workflow AST for position tracking -│ │ └── types.go # Linter types and severity levels -│ ├── workspace/ # Workspace management -│ │ └── workspace.go # Workspace creation -│ ├── snapshot/ # Workspace snapshots -│ │ └── snapshot.go # Export/import ZIP archives -│ ├── installer/ # Binary installation -│ │ ├── installer.go # Core installer logic -│ │ ├── nix.go # Nix package manager support -│ │ └── registry.go # Binary registry management -│ ├── state/ # Run state export -│ │ ├── export.go # State export functionality -│ │ └── types.go # State types -│ ├── updater/ # Self-update functionality -│ │ ├── updater.go # Update logic -│ │ └── github_source.go # GitHub releases source -│ ├── terminal/ # Terminal UI -│ │ ├── printer.go # Output formatting -│ │ ├── colors.go # ANSI colors -│ │ ├── symbols.go # Unicode symbols -│ │ ├── spinner.go # Loading spinners -│ │ └── table.go # Table rendering -│ └── logger/ # Structured logging -│ └── logger.go # Zap logger wrapper -├── pkg/ # Public packages -│ ├── cli/ # CLI commands -│ │ ├── root.go # Root command -│ │ ├── run.go # Run command (scan) -│ │ ├── workflow.go # Workflow command -│ │ ├── function.go # Function command (with bulk processing) -│ │ ├── db.go # Database CLI command -│ │ └── server.go # Server command -│ └── server/ # REST API -│ ├── server.go # Server setup -│ ├── handlers/ # Request handlers -│ └── middleware/ # Auth middleware (JWT, API Key) -├── test/ # Test suites -│ ├── e2e/ # E2E CLI tests -│ ├── integration/ # Integration tests -│ └── testdata/ -│ ├── workflows/ # Test workflow fixtures -│ └── complex-workflows/ # Complex workflow examples -└── build/ # Build artifacts - ├── docker/ # Docker files - └── DEPLOYMENT.md # Deployment guide +osmedeus/ +├── cmd/osmedeus/ # Application entry point +├── internal/ # Private packages +│ ├── client/ # Remote API client +│ ├── config/ # Configuration management +│ ├── console/ # Console output capture +│ ├── core/ # Core types (Workflow, Step, Trigger, etc.) +│ ├── database/ # SQLite/PostgreSQL via Bun ORM +│ ├── distributed/ # Distributed execution (master/worker) +│ ├── executor/ # Workflow execution engine +│ ├── functions/ # Utility functions (Goja JS runtime) +│ ├── heuristics/ # Target type detection +│ ├── installer/ # Binary installation (direct/Nix) +│ ├── linter/ # Workflow linting and validation +│ ├── logger/ # Structured logging (Zap) +│ ├── parser/ # YAML parsing and caching +│ ├── runner/ # Execution environments (host/docker/ssh) +│ ├── scheduler/ # Trigger scheduling (cron/event/watch) +│ ├── snapshot/ # Workspace export/import +│ ├── state/ # Run state export +│ ├── template/ # {{Variable}} interpolation engine +│ ├── terminal/ # Terminal UI (colors, tables, spinners) +│ ├── updater/ # Self-update via GitHub releases +│ └── workspace/ # Workspace management +├── lib/ # Shared library utilities +├── pkg/ # Public packages +│ ├── cli/ # Cobra CLI commands +│ └── server/ # Fiber REST API server +│ ├── handlers/ # Request handlers +│ └── middleware/ # Auth middleware (JWT, API Key) +├── public/ # Public assets (examples, presets, UI) +├── test/ # Test suites +│ ├── e2e/ # E2E CLI tests +│ ├── integration/ # Integration tests +│ └── testdata/ # Test workflow fixtures +├── docs/ # API documentation +└── build/ # Build artifacts and Docker files ``` ## Architecture Overview @@ -574,7 +504,7 @@ func (e *Executor) injectBuiltinVariables(cfg *config.Config, params map[string] execCtx.SetVariable("Target", params["target"]) execCtx.SetVariable("Output", filepath.Join(workspacesPath, targetSpace)) execCtx.SetVariable("threads", threads) - execCtx.SetVariable("TaskID", execCtx.RunID) + execCtx.SetVariable("RunUUID", execCtx.RunUUID) // ... more variables } ``` @@ -846,7 +776,7 @@ The linter recognizes all runtime-injected variables to avoid false positives. T **Output Variables**: `Output`, `output`, `Workspace`, `workspace` -**Metadata Variables**: `Version`, `TaskID`, `TaskDate`, `TimeStamp`, `Today`, `RandomString` +**Metadata Variables**: `Version`, `RunUUID`, `TaskDate`, `TimeStamp`, `Today`, `RandomString` **Heuristic Variables**: `TargetType`, `TargetRootDomain`, `TargetTLD`, `Org`, `TargetHost`, `TargetPort`, etc. @@ -935,7 +865,7 @@ type Run struct { Target string Params map[string]string Status string // "pending", "running", "completed", "failed" - WorkspacePath string + Workspace string // Logical workspace name (same as TargetSpace) StartedAt time.Time CompletedAt time.Time ErrorMessage string diff --git a/docs/api/README.md b/docs/api/README.md index b443ad2..add0800 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -10,7 +10,11 @@ The Osmedeus API provides a RESTful interface for managing security automation w ## Authentication -Most API endpoints require JWT authentication. First, obtain a token via the login endpoint, then include it in subsequent requests using the `Authorization: Bearer ` header. +Most API endpoints require authentication. Two methods are supported: + +1. **JWT Token**: Obtain a token via the login endpoint, then include it in requests using the `Authorization: Bearer ` header. + +2. **API Key**: Use a static API key via the `x-osm-api-key` header. Configure in `~/osmedeus-base/osm-settings.yaml` under `server.auth_api_key`. See [Authentication](authentication.md) for details. @@ -28,6 +32,7 @@ See [Authentication](authentication.md) for details. | [Assets](assets.md) | View discovered assets | | [Vulnerabilities](vulnerabilities.md) | View and manage vulnerabilities | | [Event Logs](event-logs.md) | View execution event logs | +| [Step Results](steps.md) | Query step execution results | | [Functions](functions.md) | Execute and list utility functions | | [System Statistics](system.md) | Get aggregated system stats | | [Settings](settings.md) | Manage server configuration | diff --git a/docs/api/authentication.md b/docs/api/authentication.md index 96e4677..d4a517a 100644 --- a/docs/api/authentication.md +++ b/docs/api/authentication.md @@ -97,6 +97,53 @@ curl http://localhost:8002/osm/api/workflows \ } ``` +## API Key Authentication + +As an alternative to JWT tokens, you can authenticate using a static API key via the `x-osm-api-key` header. This is useful for scripts, CI/CD pipelines, or integrations where managing JWT token refresh is impractical. + +### Configuration + +API key authentication is configured in `~/osmedeus-base/osm-settings.yaml`: + +```yaml +server: + # Enable API key authentication (default: true) + enabled_auth_api: true + # API key for x-osm-api-key header authentication + # A random 32-character key is generated on first run + auth_api_key: "your-api-key-here" +``` + +### Using the API Key + +Include the API key in requests using the `x-osm-api-key` header: + +```bash +# Store API key in environment variable +export OSM_API_KEY="your-api-key-here" + +# Use in API requests +curl http://localhost:8002/osm/api/workflows \ + -H "x-osm-api-key: $OSM_API_KEY" +``` + +### Error Response + +**401 Unauthorized** - Invalid or missing API key: +```json +{ + "error": true, + "message": "Invalid or missing API key" +} +``` + +### Notes + +- API key authentication takes priority over JWT when enabled +- A random 32-character API key is automatically generated on first server start +- The API key is stored in plain text in the settings file; ensure appropriate file permissions +- Empty, whitespace-only, or placeholder values (`null`, `undefined`, `nil`) are rejected + ## Disabling Authentication Authentication can be disabled by starting the server with the `--no-auth` flag: diff --git a/docs/api/steps.md b/docs/api/steps.md new file mode 100644 index 0000000..3c32088 --- /dev/null +++ b/docs/api/steps.md @@ -0,0 +1,138 @@ +# Step Results + +## List Step Results + +Get a paginated list of step execution results with optional filtering. + +**List all step results:** +```bash +curl http://localhost:8002/osm/api/step-results \ + -H "Authorization: Bearer $TOKEN" +``` + +**With pagination:** +```bash +curl "http://localhost:8002/osm/api/step-results?offset=0&limit=50" \ + -H "Authorization: Bearer $TOKEN" +``` + +**Filter by workspace:** +```bash +curl "http://localhost:8002/osm/api/step-results?workspace=example_com" \ + -H "Authorization: Bearer $TOKEN" +``` + +**Filter by status:** +```bash +curl "http://localhost:8002/osm/api/step-results?status=completed" \ + -H "Authorization: Bearer $TOKEN" +``` + +**Filter by step type:** +```bash +curl "http://localhost:8002/osm/api/step-results?step_type=bash" \ + -H "Authorization: Bearer $TOKEN" +``` + +**Filter by run ID:** +```bash +curl "http://localhost:8002/osm/api/step-results?run_id=123" \ + -H "Authorization: Bearer $TOKEN" +``` + +**Multiple filters:** +```bash +curl "http://localhost:8002/osm/api/step-results?workspace=example_com&status=completed&limit=100" \ + -H "Authorization: Bearer $TOKEN" +``` + +**Query Parameters:** + +| Parameter | Type | Description | +|-----------|------|-------------| +| `workspace` | string | Filter by workspace name | +| `status` | string | Filter by status (pending, running, completed, failed) | +| `step_type` | string | Filter by step type (bash, function, foreach, parallel-steps, remote-bash, http, llm) | +| `run_id` | int | Filter by run ID | +| `offset` | int | Pagination offset (default: 0) | +| `limit` | int | Maximum records to return (default: 20, max: 10000) | + +**Response:** +```json +{ + "data": [ + { + "id": 1, + "run_id": 123, + "step_name": "run-subfinder", + "step_type": "bash", + "status": "completed", + "command": "subfinder -d example.com -o {{Output}}/subdomains.txt", + "output": "Found 150 subdomains", + "error": "", + "started_at": "2025-01-15T10:00:00Z", + "completed_at": "2025-01-15T10:01:30Z", + "duration_ms": 90000, + "created_at": "2025-01-15T10:00:00Z", + "updated_at": "2025-01-15T10:01:30Z" + }, + { + "id": 2, + "run_id": 123, + "step_name": "run-httpx", + "step_type": "bash", + "status": "completed", + "command": "httpx -l {{Output}}/subdomains.txt -o {{Output}}/httpx.txt", + "output": "Probed 150 hosts, 120 alive", + "error": "", + "started_at": "2025-01-15T10:01:30Z", + "completed_at": "2025-01-15T10:03:00Z", + "duration_ms": 90000, + "created_at": "2025-01-15T10:01:30Z", + "updated_at": "2025-01-15T10:03:00Z" + }, + { + "id": 3, + "run_id": 124, + "step_name": "process-results", + "step_type": "function", + "status": "completed", + "command": "", + "output": "true", + "error": "", + "started_at": "2025-01-15T11:00:00Z", + "completed_at": "2025-01-15T11:00:01Z", + "duration_ms": 1000, + "created_at": "2025-01-15T11:00:00Z", + "updated_at": "2025-01-15T11:00:01Z" + } + ], + "pagination": { + "total": 250, + "offset": 0, + "limit": 20 + } +} +``` + +**Step Status Values:** + +| Status | Description | +|--------|-------------| +| `pending` | Step is queued but not yet started | +| `running` | Step is currently executing | +| `completed` | Step finished successfully | +| `failed` | Step failed with an error | +| `skipped` | Step was skipped (pre_condition not met) | + +**Step Types:** + +| Type | Description | +|------|-------------| +| `bash` | Shell command execution | +| `function` | Utility function execution | +| `foreach` | Loop over input items | +| `parallel-steps` | Execute steps in parallel | +| `remote-bash` | Remote command execution (Docker/SSH) | +| `http` | HTTP request step | +| `llm` | LLM/AI step | diff --git a/internal/client/client.go b/internal/client/client.go new file mode 100644 index 0000000..e89e191 --- /dev/null +++ b/internal/client/client.go @@ -0,0 +1,206 @@ +// Package client provides HTTP client utilities for interacting with a remote osmedeus server. +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +const ( + // EnvRemoteURL is the environment variable for the remote server URL + EnvRemoteURL = "OSM_REMOTE_URL" + // EnvAuthKey is the environment variable for the API authentication key + EnvAuthKey = "OSM_REMOTE_AUTH_KEY" + // APIBasePath is the base path for API endpoints + APIBasePath = "/osm/api" + // DefaultTimeout is the default HTTP request timeout + DefaultTimeout = 30 * time.Second +) + +// Client is an HTTP client for interacting with a remote osmedeus server +type Client struct { + baseURL string + authKey string + httpClient *http.Client +} + +// NewClient creates a new client with the given base URL and auth key. +// If baseURL or authKey are empty, they will be read from environment variables. +func NewClient(baseURL, authKey string) (*Client, error) { + // Resolve base URL + if baseURL == "" { + baseURL = os.Getenv(EnvRemoteURL) + } + if baseURL == "" { + return nil, fmt.Errorf("remote URL is required (set %s or use --remote-url)", EnvRemoteURL) + } + + // Validate URL format + parsedURL, err := url.Parse(baseURL) + if err != nil { + return nil, fmt.Errorf("invalid remote URL: %w", err) + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return nil, fmt.Errorf("invalid URL scheme: %s (must be http or https)", parsedURL.Scheme) + } + + // Remove trailing slash from base URL + baseURL = strings.TrimRight(baseURL, "/") + + // Resolve auth key (optional - server may allow unauthenticated access) + if authKey == "" { + authKey = os.Getenv(EnvAuthKey) + } + + return &Client{ + baseURL: baseURL, + authKey: authKey, + httpClient: &http.Client{ + Timeout: DefaultTimeout, + }, + }, nil +} + +// buildURL constructs the full URL for an API endpoint +func (c *Client) buildURL(path string, query url.Values) string { + fullPath := c.baseURL + APIBasePath + path + if len(query) > 0 { + fullPath += "?" + query.Encode() + } + return fullPath +} + +// doRequest performs an HTTP request with authentication +func (c *Client) doRequest(ctx context.Context, method, fullURL string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, fullURL, body) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + // Set headers + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + if c.authKey != "" { + req.Header.Set("x-osm-api-key", c.authKey) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + + return resp, nil +} + +// handleResponse reads and parses the response, returning an error for non-2xx status codes +func handleResponse(resp *http.Response, result interface{}) error { + defer func() { _ = resp.Body.Close() }() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + // Check for error status codes + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + var errResp ErrorResponse + if json.Unmarshal(bodyBytes, &errResp) == nil && errResp.Message != "" { + return &APIError{ + StatusCode: resp.StatusCode, + Message: errResp.Message, + } + } + return &APIError{ + StatusCode: resp.StatusCode, + Message: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, string(bodyBytes)), + } + } + + // Parse successful response + if result != nil && len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, result); err != nil { + return fmt.Errorf("failed to parse response: %w", err) + } + } + + return nil +} + +// Get performs a GET request to the given path with optional query parameters +func (c *Client) Get(ctx context.Context, path string, query url.Values, result interface{}) error { + fullURL := c.buildURL(path, query) + resp, err := c.doRequest(ctx, http.MethodGet, fullURL, nil) + if err != nil { + return err + } + return handleResponse(resp, result) +} + +// Post performs a POST request to the given path with a JSON body +func (c *Client) Post(ctx context.Context, path string, body interface{}, result interface{}) error { + var bodyReader io.Reader + if body != nil { + bodyBytes, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("failed to marshal request body: %w", err) + } + bodyReader = bytes.NewReader(bodyBytes) + } + + fullURL := c.buildURL(path, nil) + resp, err := c.doRequest(ctx, http.MethodPost, fullURL, bodyReader) + if err != nil { + return err + } + return handleResponse(resp, result) +} + +// Delete performs a DELETE request to the given path +func (c *Client) Delete(ctx context.Context, path string, result interface{}) error { + fullURL := c.buildURL(path, nil) + resp, err := c.doRequest(ctx, http.MethodDelete, fullURL, nil) + if err != nil { + return err + } + return handleResponse(resp, result) +} + +// GetRaw performs a GET request and returns the raw response body +func (c *Client) GetRaw(ctx context.Context, path string, query url.Values) ([]byte, error) { + fullURL := c.buildURL(path, query) + resp, err := c.doRequest(ctx, http.MethodGet, fullURL, nil) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + // Check for error status codes + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + var errResp ErrorResponse + if json.Unmarshal(bodyBytes, &errResp) == nil && errResp.Message != "" { + return nil, &APIError{ + StatusCode: resp.StatusCode, + Message: errResp.Message, + } + } + return nil, &APIError{ + StatusCode: resp.StatusCode, + Message: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, string(bodyBytes)), + } + } + + return bodyBytes, nil +} diff --git a/internal/client/types.go b/internal/client/types.go new file mode 100644 index 0000000..7dd6bc6 --- /dev/null +++ b/internal/client/types.go @@ -0,0 +1,318 @@ +package client + +import ( + "fmt" + "time" +) + +// ErrorResponse represents an error response from the API +type ErrorResponse struct { + Error bool `json:"error"` + Message string `json:"message"` +} + +// APIError represents an API error with status code and message +type APIError struct { + StatusCode int + Message string +} + +// Error implements the error interface +func (e *APIError) Error() string { + return fmt.Sprintf("API error (status %d): %s", e.StatusCode, e.Message) +} + +// Pagination represents pagination information in API responses +type Pagination struct { + Total int `json:"total"` + Offset int `json:"offset"` + Limit int `json:"limit"` +} + +// PaginatedResponse represents a paginated API response +type PaginatedResponse struct { + Data interface{} `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// RunsResponse represents the response from listing runs +type RunsResponse struct { + Data []Run `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// Run represents a workflow run from the API +type Run struct { + ID int64 `json:"id"` + RunUUID string `json:"run_uuid"` + WorkflowName string `json:"workflow_name"` + WorkflowKind string `json:"workflow_kind"` + Target string `json:"target"` + Params map[string]interface{} `json:"params,omitempty"` + Status string `json:"status"` + TriggerType string `json:"trigger_type,omitempty"` + RunGroupID string `json:"run_group_id,omitempty"` + TotalSteps int `json:"total_steps"` + CompletedSteps int `json:"completed_steps"` + Workspace string `json:"workspace"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// CreateRunRequest represents a request to create a new run +type CreateRunRequest struct { + Flow string `json:"flow,omitempty"` + Module string `json:"module,omitempty"` + Target string `json:"target,omitempty"` + Targets []string `json:"targets,omitempty"` + Params map[string]string `json:"params,omitempty"` + Concurrency int `json:"concurrency,omitempty"` + Priority string `json:"priority,omitempty"` + Timeout int `json:"timeout,omitempty"` + RunnerType string `json:"runner_type,omitempty"` + ThreadsHold int `json:"threads_hold,omitempty"` + EmptyTarget bool `json:"empty_target,omitempty"` + Repeat bool `json:"repeat,omitempty"` + RepeatWaitTime string `json:"repeat_wait_time,omitempty"` + HeuristicsCheck string `json:"heuristics_check,omitempty"` +} + +// CreateRunResponse represents the response from creating a run +type CreateRunResponse struct { + Message string `json:"message"` + Workflow string `json:"workflow"` + Kind string `json:"kind"` + TargetCount int `json:"target_count"` + Priority string `json:"priority"` + JobID string `json:"job_id"` + Status string `json:"status"` + PollURL string `json:"poll_url"` + Target string `json:"target,omitempty"` + RunID string `json:"run_id,omitempty"` + Targets []string `json:"targets,omitempty"` + Concurrency int `json:"concurrency,omitempty"` +} + +// CancelRunResponse represents the response from cancelling a run +type CancelRunResponse struct { + Message string `json:"message"` + ID int64 `json:"id"` + RunID string `json:"run_id"` +} + +// AssetsResponse represents the response from listing assets +type AssetsResponse struct { + Data []Asset `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// Asset represents an asset from the API +type Asset struct { + ID int64 `json:"id"` + Workspace string `json:"workspace"` + AssetType string `json:"asset_type"` + AssetValue string `json:"asset_value"` + URL string `json:"url,omitempty"` + Title string `json:"title,omitempty"` + StatusCode int `json:"status_code,omitempty"` + HostIP string `json:"host_ip,omitempty"` + TechStack []string `json:"tech_stack,omitempty"` + ContentType string `json:"content_type,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// WorkspacesResponse represents the response from listing workspaces +type WorkspacesResponse struct { + Data []Workspace `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// Workspace represents a workspace from the API +type Workspace struct { + ID int64 `json:"id,omitempty"` + Name string `json:"name"` + LocalPath string `json:"local_path,omitempty"` + DataSource string `json:"data_source,omitempty"` + TotalAssets int `json:"total_assets,omitempty"` + Tags []string `json:"tags,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +// VulnerabilitiesResponse represents the response from listing vulnerabilities +type VulnerabilitiesResponse struct { + Data []Vulnerability `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// Vulnerability represents a vulnerability from the API +type Vulnerability struct { + ID int64 `json:"id"` + Workspace string `json:"workspace"` + VulnInfo string `json:"vuln_info,omitempty"` + VulnTitle string `json:"vuln_title,omitempty"` + VulnDesc string `json:"vuln_desc,omitempty"` + VulnPOC string `json:"vuln_poc,omitempty"` + Severity string `json:"severity,omitempty"` + Confidence string `json:"confidence,omitempty"` + AssetType string `json:"asset_type,omitempty"` + AssetValue string `json:"asset_value,omitempty"` + Tags []string `json:"tags,omitempty"` + DetailHTTPRequest string `json:"detail_http_request,omitempty"` + DetailHTTPResponse string `json:"detail_http_response,omitempty"` + RawVulnJSON string `json:"raw_vuln_json,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// FunctionEvalRequest represents a request to evaluate a function +type FunctionEvalRequest struct { + Script string `json:"script"` + Target string `json:"target,omitempty"` + Params map[string]string `json:"params,omitempty"` +} + +// FunctionEvalResponse represents the response from evaluating a function +type FunctionEvalResponse struct { + Result interface{} `json:"result"` + RenderedScript string `json:"rendered_script,omitempty"` + Error bool `json:"error,omitempty"` + Message string `json:"message,omitempty"` +} + +// StepResultsResponse represents the response from listing step results +type StepResultsResponse struct { + Data []StepResult `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// StepResult represents a step execution result from the API +type StepResult struct { + ID string `json:"id"` + RunID int64 `json:"run_id"` + StepName string `json:"step_name"` + StepType string `json:"step_type"` + Status string `json:"status"` + Command string `json:"command,omitempty"` + Output string `json:"output,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + Exports map[string]interface{} `json:"exports,omitempty"` + DurationMs int64 `json:"duration_ms"` + LogFile string `json:"log_file,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// ArtifactsResponse represents the response from listing artifacts +type ArtifactsResponse struct { + Data []Artifact `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// Artifact represents an artifact from the API +type Artifact struct { + ID string `json:"id"` + RunID int64 `json:"run_id"` + Workspace string `json:"workspace"` + Name string `json:"name"` + ArtifactPath string `json:"artifact_path"` + ArtifactType string `json:"artifact_type,omitempty"` + ContentType string `json:"content_type,omitempty"` + SizeBytes int64 `json:"size_bytes"` + LineCount int `json:"line_count"` + Description string `json:"description,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// EventLogsResponse represents the response from listing event logs +type EventLogsResponse struct { + Data []EventLog `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// EventLog represents an event log from the API +type EventLog struct { + ID int64 `json:"id"` + Topic string `json:"topic"` + EventID string `json:"event_id,omitempty"` + Name string `json:"name,omitempty"` + Source string `json:"source,omitempty"` + DataType string `json:"data_type,omitempty"` + Data string `json:"data,omitempty"` + Workspace string `json:"workspace,omitempty"` + RunID string `json:"run_id,omitempty"` + WorkflowName string `json:"workflow_name,omitempty"` + Processed bool `json:"processed"` + ProcessedAt *time.Time `json:"processed_at,omitempty"` + Error string `json:"error,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// SchedulesResponse represents the response from listing schedules +type SchedulesResponse struct { + Data []Schedule `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// Schedule represents a schedule from the API +type Schedule struct { + ID string `json:"id"` + Name string `json:"name"` + WorkflowName string `json:"workflow_name"` + WorkflowPath string `json:"workflow_path"` + TriggerName string `json:"trigger_name"` + TriggerType string `json:"trigger_type"` + Schedule string `json:"schedule,omitempty"` + EventTopic string `json:"event_topic,omitempty"` + WatchPath string `json:"watch_path,omitempty"` + InputConfig map[string]interface{} `json:"input_config,omitempty"` + IsEnabled bool `json:"is_enabled"` + LastRun *time.Time `json:"last_run,omitempty"` + NextRun *time.Time `json:"next_run,omitempty"` + RunCount int `json:"run_count"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// AssetDiffsResponse represents the response from listing asset diffs +type AssetDiffsResponse struct { + Data []AssetDiff `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// AssetDiff represents an asset diff snapshot from the API +type AssetDiff struct { + ID int64 `json:"id"` + WorkspaceName string `json:"workspace_name"` + FromTime time.Time `json:"from_time"` + ToTime time.Time `json:"to_time"` + TotalAdded int `json:"total_added"` + TotalRemoved int `json:"total_removed"` + TotalChanged int `json:"total_changed"` + DiffData string `json:"diff_data,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// VulnDiffsResponse represents the response from listing vulnerability diffs +type VulnDiffsResponse struct { + Data []VulnDiff `json:"data"` + Pagination Pagination `json:"pagination"` +} + +// VulnDiff represents a vulnerability diff snapshot from the API +type VulnDiff struct { + ID int64 `json:"id"` + WorkspaceName string `json:"workspace_name"` + FromTime time.Time `json:"from_time"` + ToTime time.Time `json:"to_time"` + TotalAdded int `json:"total_added"` + TotalRemoved int `json:"total_removed"` + TotalChanged int `json:"total_changed"` + DiffData string `json:"diff_data,omitempty"` + CreatedAt time.Time `json:"created_at"` +} diff --git a/internal/console/capture.go b/internal/console/capture.go index 9277c81..a883806 100644 --- a/internal/console/capture.go +++ b/internal/console/capture.go @@ -21,15 +21,27 @@ type Capture struct { wg sync.WaitGroup } -// StartCapture begins capturing stdout/stderr to the specified file +// StartCapture begins capturing stdout/stderr to the specified file. +// This truncates the file if it exists. func StartCapture(filePath string) (*Capture, error) { + return startCaptureWithFlags(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC) +} + +// StartCaptureAppend begins capturing stdout/stderr, appending to file if it exists. +// Use this for subsequent captures in the same workflow (e.g., multiple modules in a flow). +func StartCaptureAppend(filePath string) (*Capture, error) { + return startCaptureWithFlags(filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND) +} + +// startCaptureWithFlags is the internal implementation that accepts file open flags. +func startCaptureWithFlags(filePath string, flags int) (*Capture, error) { // Ensure directory exists if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil { return nil, err } - // Open file for writing - file, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + // Open file for writing with specified flags + file, err := os.OpenFile(filePath, flags, 0644) if err != nil { return nil, err } diff --git a/internal/core/constants.go b/internal/core/constants.go index 3463413..2c4e020 100644 --- a/internal/core/constants.go +++ b/internal/core/constants.go @@ -3,7 +3,7 @@ package core // Project metadata constants const ( // VERSION of this project - VERSION = "v5.0.0-beta" + VERSION = "v5.0.0" // DESC description of the tool DESC = "A Modern Orchestration Engine for Security" // BINARY name of osmedeus diff --git a/internal/core/context.go b/internal/core/context.go index 13c3ec6..2428ec6 100644 --- a/internal/core/context.go +++ b/internal/core/context.go @@ -11,7 +11,7 @@ import ( type ExecutionContext struct { WorkflowName string WorkflowKind WorkflowKind - RunID string + RunUUID string Target string WorkspacePath string BaseFolder string @@ -43,11 +43,11 @@ type ExecutionContext struct { } // NewExecutionContext creates a new execution context -func NewExecutionContext(workflowName string, kind WorkflowKind, runID, target string) *ExecutionContext { +func NewExecutionContext(workflowName string, kind WorkflowKind, runUUID, target string) *ExecutionContext { return &ExecutionContext{ WorkflowName: workflowName, WorkflowKind: kind, - RunID: runID, + RunUUID: runUUID, Target: target, Params: make(map[string]interface{}), Exports: make(map[string]interface{}), @@ -151,7 +151,7 @@ func (c *ExecutionContext) Clone() *ExecutionContext { clone := &ExecutionContext{ WorkflowName: c.WorkflowName, WorkflowKind: c.WorkflowKind, - RunID: c.RunID, + RunUUID: c.RunUUID, Target: c.Target, WorkspacePath: c.WorkspacePath, BaseFolder: c.BaseFolder, @@ -193,7 +193,7 @@ func (c *ExecutionContext) CloneForLoop(loopVar string, loopValue interface{}, i clone := &ExecutionContext{ WorkflowName: c.WorkflowName, WorkflowKind: c.WorkflowKind, - RunID: c.RunID, + RunUUID: c.RunUUID, Target: c.Target, WorkspacePath: c.WorkspacePath, BaseFolder: c.BaseFolder, diff --git a/internal/core/dependency.go b/internal/core/dependency.go index 6508d16..321f13a 100644 --- a/internal/core/dependency.go +++ b/internal/core/dependency.go @@ -86,6 +86,8 @@ func MatchesVariableType(value string, varType VariableType) (bool, error) { case VarTypeCIDR: _, _, err := net.ParseCIDR(value) return err == nil, nil + case VarTypeIP: + return net.ParseIP(value) != nil, nil case VarTypeRepo: return isRepo(value), nil case VarTypePath, VarTypeFile, VarTypeFolder: @@ -159,6 +161,8 @@ func MatchesTargetType(target string, targetType TargetType) (bool, error) { return MatchesVariableType(target, VarTypeNumber) case TargetTypeString: return true, nil + case TargetTypeIP: + return MatchesVariableType(target, VarTypeIP) case TargetTypeFile: info, err := os.Stat(target) if err != nil { diff --git a/internal/core/dependency_test.go b/internal/core/dependency_test.go index 2512f74..477dbaa 100644 --- a/internal/core/dependency_test.go +++ b/internal/core/dependency_test.go @@ -86,6 +86,42 @@ func TestMatchesAnyVariableType(t *testing.T) { typeSpec: "number", want: false, }, + { + name: "ip type accepts IPv4", + value: "192.168.1.1", + typeSpec: VarTypeIP, + want: true, + }, + { + name: "ip type accepts IPv6", + value: "2001:db8::1", + typeSpec: VarTypeIP, + want: true, + }, + { + name: "ip type rejects domain", + value: "example.com", + typeSpec: VarTypeIP, + want: false, + }, + { + name: "ip type rejects invalid ip", + value: "999.999.999.999", + typeSpec: VarTypeIP, + want: false, + }, + { + name: "comma-separated ip,cidr accepts ip", + value: "162.13.44.21", + typeSpec: "ip,cidr", + want: true, + }, + { + name: "comma-separated ip,cidr accepts cidr", + value: "10.0.0.0/8", + typeSpec: "ip,cidr", + want: true, + }, } for _, tt := range tests { @@ -163,6 +199,36 @@ func TestMatchesAnyTargetType(t *testing.T) { typeSpec: "string", want: true, }, + { + name: "ip type accepts IPv4", + target: "162.13.44.21", + typeSpec: TargetTypeIP, + want: true, + }, + { + name: "ip type accepts IPv6", + target: "2001:db8::1", + typeSpec: TargetTypeIP, + want: true, + }, + { + name: "ip type rejects domain", + target: "example.com", + typeSpec: TargetTypeIP, + want: false, + }, + { + name: "comma-separated ip,cidr accepts ip", + target: "192.168.1.1", + typeSpec: "ip,cidr", + want: true, + }, + { + name: "comma-separated ip,cidr accepts cidr", + target: "10.0.0.0/8", + typeSpec: "ip,cidr", + want: true, + }, } for _, tt := range tests { diff --git a/internal/core/override.go b/internal/core/override.go index 792f3fd..c7a3615 100644 --- a/internal/core/override.go +++ b/internal/core/override.go @@ -1,5 +1,7 @@ package core +import "fmt" + // OverrideMode specifies how to merge steps or modules during inheritance type OverrideMode string @@ -56,6 +58,56 @@ type ParamOverride struct { Generator *string `yaml:"generator,omitempty"` } +// UnmarshalYAML supports shorthand syntax where a scalar value is treated as the default: +// +// param-a: "value" # shorthand: scalar becomes default +// param-b: false # shorthand: bool becomes default +// +// And verbose syntax with full struct: +// +// param-a: +// default: "value" +// type: "string" +// required: true +func (p *ParamOverride) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Try scalar first (shorthand syntax) + var scalar interface{} + if err := unmarshal(&scalar); err == nil { + switch v := scalar.(type) { + case string, bool, int, int64, float64: + p.Default = v + return nil + case nil: + // nil value is valid, means no default + return nil + case map[string]interface{}: + // Fall through to map handling below + default: + // Unknown type, try map handling + } + } + + // Try as map (verbose syntax) + var m map[string]interface{} + if err := unmarshal(&m); err != nil { + return fmt.Errorf("invalid param override: expected scalar or map") + } + + if v, ok := m["default"]; ok { + p.Default = v + } + if v, ok := m["type"].(string); ok { + p.Type = &v + } + if v, ok := m["required"].(bool); ok { + p.Required = &v + } + if v, ok := m["generator"].(string); ok { + p.Generator = &v + } + return nil +} + // StepsOverride specifies how to override steps in a module workflow type StepsOverride struct { // Mode specifies the merge strategy: replace, prepend, append, merge diff --git a/internal/core/override_test.go b/internal/core/override_test.go new file mode 100644 index 0000000..52e1fe6 --- /dev/null +++ b/internal/core/override_test.go @@ -0,0 +1,222 @@ +package core + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestParamOverrideUnmarshalYAML(t *testing.T) { + tests := []struct { + name string + input string + wantDefault any + wantType *string + wantReq *bool + wantGen *string + wantErr bool + }{ + { + name: "shorthand string", + input: `"hello"`, + wantDefault: "hello", + }, + { + name: "shorthand bool true", + input: `true`, + wantDefault: true, + }, + { + name: "shorthand bool false", + input: `false`, + wantDefault: false, + }, + { + name: "shorthand int", + input: `42`, + wantDefault: 42, + }, + { + name: "shorthand float", + input: `3.14`, + wantDefault: 3.14, + }, + { + name: "shorthand nil", + input: `null`, + wantDefault: nil, + }, + { + name: "verbose default only", + input: `default: "verbose-value"`, + wantDefault: "verbose-value", + }, + { + name: "verbose with type", + input: "default: \"value\"\ntype: \"string\"", + wantDefault: "value", + wantType: ptr("string"), + }, + { + name: "verbose with required", + input: "default: \"value\"\nrequired: true", + wantDefault: "value", + wantReq: ptr(true), + }, + { + name: "verbose with generator", + input: "default: \"value\"\ngenerator: \"uuid()\"", + wantDefault: "value", + wantGen: ptr("uuid()"), + }, + { + name: "verbose full struct", + input: "default: \"full\"\ntype: \"string\"\nrequired: true\ngenerator: \"gen()\"", + wantDefault: "full", + wantType: ptr("string"), + wantReq: ptr(true), + wantGen: ptr("gen()"), + }, + { + name: "verbose bool default", + input: `default: false`, + wantDefault: false, + }, + { + name: "verbose int default", + input: `default: 100`, + wantDefault: 100, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var p ParamOverride + err := yaml.Unmarshal([]byte(tt.input), &p) + + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.wantDefault, p.Default) + assert.Equal(t, tt.wantType, p.Type) + assert.Equal(t, tt.wantReq, p.Required) + assert.Equal(t, tt.wantGen, p.Generator) + }) + } +} + +func TestParamOverrideMapUnmarshalYAML(t *testing.T) { + tests := []struct { + name string + input string + want map[string]*ParamOverride + wantErr bool + }{ + { + name: "mixed shorthand and verbose", + input: ` +param-a: "shorthand-value" +param-b: + default: "verbose-value" + type: "string" +param-c: false +param-d: 42 +`, + want: map[string]*ParamOverride{ + "param-a": {Default: "shorthand-value"}, + "param-b": {Default: "verbose-value", Type: ptr("string")}, + "param-c": {Default: false}, + "param-d": {Default: 42}, + }, + }, + { + name: "all shorthand", + input: ` +name: "test" +enabled: true +count: 5 +`, + want: map[string]*ParamOverride{ + "name": {Default: "test"}, + "enabled": {Default: true}, + "count": {Default: 5}, + }, + }, + { + name: "all verbose", + input: ` +param-a: + default: "value-a" + required: true +param-b: + default: "value-b" + generator: "gen()" +`, + want: map[string]*ParamOverride{ + "param-a": {Default: "value-a", Required: ptr(true)}, + "param-b": {Default: "value-b", Generator: ptr("gen()")}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var result map[string]*ParamOverride + err := yaml.Unmarshal([]byte(tt.input), &result) + + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + require.Equal(t, len(tt.want), len(result)) + + for k, want := range tt.want { + got, ok := result[k] + require.True(t, ok, "missing key %s", k) + assert.Equal(t, want.Default, got.Default, "key %s: Default mismatch", k) + assert.Equal(t, want.Type, got.Type, "key %s: Type mismatch", k) + assert.Equal(t, want.Required, got.Required, "key %s: Required mismatch", k) + assert.Equal(t, want.Generator, got.Generator, "key %s: Generator mismatch", k) + } + }) + } +} + +func TestWorkflowOverrideParamsUnmarshalYAML(t *testing.T) { + input := ` +params: + param-a: "shorthand" + param-b: + default: "verbose" + type: "string" +steps: + mode: append +` + var override WorkflowOverride + err := yaml.Unmarshal([]byte(input), &override) + require.NoError(t, err) + + require.NotNil(t, override.Params) + require.Len(t, override.Params, 2) + + assert.Equal(t, "shorthand", override.Params["param-a"].Default) + + assert.Equal(t, "verbose", override.Params["param-b"].Default) + require.NotNil(t, override.Params["param-b"].Type) + assert.Equal(t, "string", *override.Params["param-b"].Type) + + require.NotNil(t, override.Steps) + assert.Equal(t, OverrideModeAppend, override.Steps.Mode) +} + +// ptr is a helper to create a pointer to a value +func ptr[T any](v T) *T { + return &v +} diff --git a/internal/core/types.go b/internal/core/types.go index fa8cde0..92f17c8 100644 --- a/internal/core/types.go +++ b/internal/core/types.go @@ -50,6 +50,7 @@ const ( VarTypeURL VariableType = "url" VarTypeCIDR VariableType = "cidr" VarTypeRepo VariableType = "repo" + VarTypeIP VariableType = "ip" ) type TargetType string @@ -65,6 +66,7 @@ const ( TargetTypeFolder TargetType = "folder" TargetTypeNumber TargetType = "number" TargetTypeString TargetType = "string" + TargetTypeIP TargetType = "ip" ) // ActionType for on_success/on_error handlers @@ -129,7 +131,7 @@ type StepResult struct { type WorkflowResult struct { WorkflowName string WorkflowKind WorkflowKind - RunID string + RunUUID string Target string Status RunStatus StartTime time.Time diff --git a/internal/database/batch.go b/internal/database/batch.go index 20e709a..b107647 100644 --- a/internal/database/batch.go +++ b/internal/database/batch.go @@ -28,12 +28,12 @@ func DefaultBatchConfig() *BatchConfig { type StepResultBuffer struct { mu sync.Mutex buffer []*StepResult - runID string + runID int64 config *BatchConfig } // NewStepResultBuffer creates a new step result buffer -func NewStepResultBuffer(runID string, cfg *BatchConfig) *StepResultBuffer { +func NewStepResultBuffer(runID int64, cfg *BatchConfig) *StepResultBuffer { if cfg == nil { cfg = DefaultBatchConfig() } @@ -205,8 +205,8 @@ func (pt *ProgressTracker) Stop() { } // BatchUpdateRunProgress performs a single update for multiple steps -func BatchUpdateRunProgress(ctx context.Context, runID string, steps int) error { - if db == nil || runID == "" || steps == 0 { +func BatchUpdateRunProgress(ctx context.Context, runUUID string, steps int) error { + if db == nil || runUUID == "" || steps == 0 { return nil } @@ -214,7 +214,7 @@ func BatchUpdateRunProgress(ctx context.Context, runID string, steps int) error Model((*Run)(nil)). Set("completed_steps = completed_steps + ?", steps). Set("updated_at = ?", time.Now()). - Where("id = ? OR run_id = ?", runID, runID). + Where("run_uuid = ?", runUUID). Exec(ctx) return err } diff --git a/internal/database/database.go b/internal/database/database.go index 2b9f2dc..9909c7a 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -170,6 +170,11 @@ func Migrate(ctx context.Context) error { } } + // Create indexes for Run table + if err := createRunIndexes(ctx); err != nil { + return err + } + // Create indexes for Asset table if err := createAssetIndexes(ctx); err != nil { return err @@ -208,6 +213,25 @@ func Migrate(ctx context.Context) error { return nil } +// createRunIndexes creates indexes for the runs table +func createRunIndexes(ctx context.Context) error { + indexes := []string{ + "CREATE INDEX IF NOT EXISTS idx_runs_run_uuid ON runs(run_uuid)", + "CREATE INDEX IF NOT EXISTS idx_runs_run_group_id ON runs(run_group_id)", + "CREATE INDEX IF NOT EXISTS idx_runs_status ON runs(status)", + "CREATE INDEX IF NOT EXISTS idx_runs_workflow_name ON runs(workflow_name)", + "CREATE INDEX IF NOT EXISTS idx_runs_target ON runs(target)", + } + + for _, idx := range indexes { + if _, err := db.ExecContext(ctx, idx); err != nil { + return fmt.Errorf("failed to create index: %w", err) + } + } + + return nil +} + // createAssetIndexes creates indexes for the assets table func createAssetIndexes(ctx context.Context) error { indexes := []string{ diff --git a/internal/database/distributed_hooks_test.go b/internal/database/distributed_hooks_test.go index e778214..c43a1b6 100644 --- a/internal/database/distributed_hooks_test.go +++ b/internal/database/distributed_hooks_test.go @@ -71,7 +71,7 @@ func TestTrySendRunToRedis(t *testing.T) { UnregisterDistributedHooks() ctx := context.Background() - run := &Run{ID: "test-run", RunID: "run-123"} + run := &Run{ID: 1, RunUUID: "run-123"} // Without hooks, should return false if trySendRunToRedis(ctx, run) { @@ -97,7 +97,7 @@ func TestTrySendRunToRedis(t *testing.T) { t.Error("SendRun was not called") } if sentRun.ID != run.ID { - t.Errorf("Wrong run sent: expected %s, got %s", run.ID, sentRun.ID) + t.Errorf("Wrong run sent: expected %d, got %d", run.ID, sentRun.ID) } // Cleanup @@ -244,7 +244,7 @@ func TestConcurrentHooksAccess(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - run := &Run{ID: "concurrent-test"} + run := &Run{ID: 1, RunUUID: "concurrent-test"} trySendRunToRedis(ctx, run) }() } diff --git a/internal/database/models.go b/internal/database/models.go index 52f80c1..dfe6569 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -10,14 +10,14 @@ import ( type Run struct { bun.BaseModel `bun:"table:runs,alias:r"` - ID string `bun:"id,pk,type:text" json:"id"` - RunID string `bun:"run_id,unique,notnull" json:"run_id"` + ID int64 `bun:"id,pk,autoincrement" json:"id"` + RunUUID string `bun:"run_uuid,unique,notnull" json:"run_uuid"` WorkflowName string `bun:"workflow_name,notnull" json:"workflow_name"` WorkflowKind string `bun:"workflow_kind,notnull" json:"workflow_kind"` Target string `bun:"target,notnull" json:"target"` Params map[string]interface{} `bun:"params,type:json" json:"params"` Status string `bun:"status,notnull" json:"status"` - WorkspacePath string `bun:"workspace_path" json:"workspace_path"` + Workspace string `bun:"workspace" json:"workspace"` StartedAt *time.Time `bun:"started_at" json:"started_at"` CompletedAt *time.Time `bun:"completed_at" json:"completed_at"` ErrorMessage string `bun:"error_message" json:"error_message,omitempty"` @@ -29,8 +29,8 @@ type Run struct { TriggerType string `bun:"trigger_type" json:"trigger_type,omitempty"` // manual, cron, event TriggerName string `bun:"trigger_name" json:"trigger_name,omitempty"` - // Job grouping - multiple targets from same request share a JobID - JobID string `bun:"job_id" json:"job_id,omitempty"` + // Run grouping - multiple targets from same request share a RunGroupID + RunGroupID string `bun:"run_group_id" json:"run_group_id,omitempty"` // Progress tracking TotalSteps int `bun:"total_steps" json:"total_steps"` @@ -39,7 +39,7 @@ type Run struct { // Relations Steps []*StepResult `bun:"rel:has-many,join:id=run_id" json:"steps,omitempty"` Artifacts []*Artifact `bun:"rel:has-many,join:id=run_id" json:"artifacts,omitempty"` - Events []*EventLog `bun:"rel:has-many,join:run_id=run_id" json:"events,omitempty"` + Events []*EventLog `bun:"rel:has-many,join:run_uuid=run_id" json:"events,omitempty"` } // StepResult represents a step execution result @@ -47,7 +47,7 @@ type StepResult struct { bun.BaseModel `bun:"table:step_results,alias:sr"` ID string `bun:"id,pk,type:text" json:"id"` - RunID string `bun:"run_id,notnull,type:text" json:"run_id"` + RunID int64 `bun:"run_id,notnull" json:"run_id"` StepName string `bun:"step_name,notnull" json:"step_name"` StepType string `bun:"step_type,notnull" json:"step_type"` Status string `bun:"status,notnull" json:"status"` @@ -109,7 +109,7 @@ type Artifact struct { bun.BaseModel `bun:"table:artifacts,alias:a"` ID string `bun:"id,pk,type:text" json:"id"` - RunID string `bun:"run_id,notnull,type:text" json:"run_id"` + RunID int64 `bun:"run_id,notnull" json:"run_id"` Workspace string `bun:"workspace,notnull" json:"workspace"` Name string `bun:"name,notnull" json:"name"` ArtifactPath string `bun:"artifact_path,notnull" json:"artifact_path"` diff --git a/internal/database/seed.go b/internal/database/seed.go index cc52ced..36c87f7 100644 --- a/internal/database/seed.go +++ b/internal/database/seed.go @@ -22,8 +22,8 @@ func SeedDatabase(ctx context.Context) error { scan4ID := uuid.New().String() // secondary.com - part of batch job scan5ID := uuid.New().String() // tertiary.io - part of batch job - // Job ID for batch scanning demonstration (scan1, scan4, scan5 share this) - job1ID := uuid.New().String() + // Run group ID for batch scanning demonstration (scan1, scan4, scan5 share this) + runGroup1ID := uuid.New().String()[:8] now := time.Now() oneHourAgo := now.Add(-1 * time.Hour) @@ -36,18 +36,17 @@ func SeedDatabase(ctx context.Context) error { twoWeeksAgo := now.Add(-14 * 24 * time.Hour) oneMonthAgo := now.Add(-30 * 24 * time.Hour) - // Seed Runs + // Seed Runs - ID is auto-generated, use RunUUID as unique identifier runs := []Run{ { - ID: scan1ID, - RunID: fmt.Sprintf("run-%s", scan1ID[:8]), - JobID: job1ID, // Part of batch job with scan4 and scan5 + RunUUID: scan1ID, + RunGroupID: runGroup1ID, // Part of batch run group with scan4 and scan5 WorkflowName: "subdomain-enum", WorkflowKind: "module", Target: "example.com", Params: map[string]interface{}{"threads": 10, "timeout": 300}, Status: "completed", - WorkspacePath: "/home/osmedeus/workspaces-osmedeus/example.com", + Workspace: "example.com", StartedAt: &twoHoursAgo, CompletedAt: &oneHourAgo, TriggerType: "manual", @@ -57,14 +56,13 @@ func SeedDatabase(ctx context.Context) error { UpdatedAt: oneHourAgo, }, { - ID: scan2ID, - RunID: fmt.Sprintf("run-%s", scan2ID[:8]), + RunUUID: scan2ID, WorkflowName: "port-scan", WorkflowKind: "module", Target: "api.example.com", Params: map[string]interface{}{"ports": "1-10000", "rate": 1000}, Status: "running", - WorkspacePath: "/home/osmedeus/workspaces-osmedeus/api.example.com", + Workspace: "api.example.com", StartedAt: &thirtyMinsAgo, TriggerType: "cron", TriggerName: "daily-recon", @@ -74,14 +72,13 @@ func SeedDatabase(ctx context.Context) error { UpdatedAt: now, }, { - ID: scan3ID, - RunID: fmt.Sprintf("run-%s", scan3ID[:8]), + RunUUID: scan3ID, WorkflowName: "vuln-scan", WorkflowKind: "flow", Target: "staging.test.local", Params: map[string]interface{}{"severity": "critical,high", "templates": "cves"}, Status: "failed", - WorkspacePath: "/home/osmedeus/workspaces-osmedeus/staging.test.local", + Workspace: "staging.test.local", StartedAt: &twoHoursAgo, CompletedAt: &oneHourAgo, ErrorMessage: "nuclei: template loading failed: connection timeout", @@ -91,17 +88,16 @@ func SeedDatabase(ctx context.Context) error { CreatedAt: twoHoursAgo, UpdatedAt: oneHourAgo, }, - // Batch job runs - scan4 and scan5 share job1ID with scan1 + // Batch run group - scan4 and scan5 share runGroup1ID with scan1 { - ID: scan4ID, - RunID: fmt.Sprintf("run-%s", scan4ID[:8]), - JobID: job1ID, // Part of batch job with scan1 and scan5 + RunUUID: scan4ID, + RunGroupID: runGroup1ID, // Part of batch run group with scan1 and scan5 WorkflowName: "subdomain-enum", WorkflowKind: "module", Target: "secondary.com", Params: map[string]interface{}{"threads": 10, "timeout": 300}, Status: "completed", - WorkspacePath: "/home/osmedeus/workspaces-osmedeus/secondary.com", + Workspace: "secondary.com", StartedAt: &twoHoursAgo, CompletedAt: &oneHourAgo, TriggerType: "manual", @@ -111,15 +107,14 @@ func SeedDatabase(ctx context.Context) error { UpdatedAt: oneHourAgo, }, { - ID: scan5ID, - RunID: fmt.Sprintf("run-%s", scan5ID[:8]), - JobID: job1ID, // Part of batch job with scan1 and scan4 + RunUUID: scan5ID, + RunGroupID: runGroup1ID, // Part of batch run group with scan1 and scan4 WorkflowName: "subdomain-enum", WorkflowKind: "module", Target: "tertiary.io", Params: map[string]interface{}{"threads": 10, "timeout": 300}, Status: "running", - WorkspacePath: "/home/osmedeus/workspaces-osmedeus/tertiary.io", + Workspace: "tertiary.io", StartedAt: &thirtyMinsAgo, TriggerType: "manual", TotalSteps: 5, @@ -129,10 +124,13 @@ func SeedDatabase(ctx context.Context) error { }, } - for _, run := range runs { - if _, err := db.NewInsert().Model(&run).Exec(ctx); err != nil { + // Insert runs and build a map of RunUUID -> ID for referencing in StepResults/Artifacts + runIDMap := make(map[string]int64) + for i := range runs { + if _, err := db.NewInsert().Model(&runs[i]).Exec(ctx); err != nil { return fmt.Errorf("failed to insert run: %w", err) } + runIDMap[runs[i].RunUUID] = runs[i].ID } // Seed StepResults @@ -140,7 +138,7 @@ func SeedDatabase(ctx context.Context) error { // Run 1 steps (subdomain-enum - completed) { ID: uuid.New().String(), - RunID: scan1ID, + RunID: runIDMap[scan1ID], StepName: "subfinder", StepType: "bash", Status: "completed", @@ -155,7 +153,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan1ID, + RunID: runIDMap[scan1ID], StepName: "amass", StepType: "bash", Status: "completed", @@ -170,7 +168,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan1ID, + RunID: runIDMap[scan1ID], StepName: "merge-subdomains", StepType: "function", Status: "completed", @@ -184,7 +182,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan1ID, + RunID: runIDMap[scan1ID], StepName: "httpx", StepType: "bash", Status: "completed", @@ -199,7 +197,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan1ID, + RunID: runIDMap[scan1ID], StepName: "screenshot", StepType: "bash", Status: "completed", @@ -214,7 +212,7 @@ func SeedDatabase(ctx context.Context) error { // Run 2 steps (port-scan - running) { ID: uuid.New().String(), - RunID: scan2ID, + RunID: runIDMap[scan2ID], StepName: "masscan", StepType: "bash", Status: "completed", @@ -228,7 +226,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan2ID, + RunID: runIDMap[scan2ID], StepName: "nmap-service-scan", StepType: "bash", Status: "running", @@ -241,7 +239,7 @@ func SeedDatabase(ctx context.Context) error { // Run 3 steps (vuln-scan - failed) { ID: uuid.New().String(), - RunID: scan3ID, + RunID: runIDMap[scan3ID], StepName: "prepare-targets", StepType: "function", Status: "completed", @@ -254,7 +252,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan3ID, + RunID: runIDMap[scan3ID], StepName: "nuclei", StepType: "bash", Status: "failed", @@ -269,7 +267,7 @@ func SeedDatabase(ctx context.Context) error { // Additional steps for scan2ID (port-scan) { ID: uuid.New().String(), - RunID: scan2ID, + RunID: runIDMap[scan2ID], StepName: "port-filter", StepType: "function", Status: "completed", @@ -283,7 +281,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan2ID, + RunID: runIDMap[scan2ID], StepName: "banner-grab", StepType: "bash", Status: "pending", @@ -295,7 +293,7 @@ func SeedDatabase(ctx context.Context) error { // Additional steps for scan3ID (vuln-scan - some completed before failure) { ID: uuid.New().String(), - RunID: scan3ID, + RunID: runIDMap[scan3ID], StepName: "validate-targets", StepType: "function", Status: "completed", @@ -309,7 +307,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan3ID, + RunID: runIDMap[scan3ID], StepName: "load-templates", StepType: "bash", Status: "completed", @@ -324,7 +322,7 @@ func SeedDatabase(ctx context.Context) error { // Steps for scan4ID (secondary.com - completed batch job run) { ID: uuid.New().String(), - RunID: scan4ID, + RunID: runIDMap[scan4ID], StepName: "subfinder", StepType: "bash", Status: "completed", @@ -339,7 +337,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan4ID, + RunID: runIDMap[scan4ID], StepName: "amass", StepType: "bash", Status: "completed", @@ -354,7 +352,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan4ID, + RunID: runIDMap[scan4ID], StepName: "merge-subdomains", StepType: "function", Status: "completed", @@ -368,7 +366,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan4ID, + RunID: runIDMap[scan4ID], StepName: "httpx", StepType: "bash", Status: "completed", @@ -383,7 +381,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan4ID, + RunID: runIDMap[scan4ID], StepName: "screenshot", StepType: "bash", Status: "completed", @@ -398,7 +396,7 @@ func SeedDatabase(ctx context.Context) error { // Steps for scan5ID (tertiary.io - running batch job run) { ID: uuid.New().String(), - RunID: scan5ID, + RunID: runIDMap[scan5ID], StepName: "subfinder", StepType: "bash", Status: "completed", @@ -413,7 +411,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan5ID, + RunID: runIDMap[scan5ID], StepName: "amass", StepType: "bash", Status: "completed", @@ -428,7 +426,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan5ID, + RunID: runIDMap[scan5ID], StepName: "merge-subdomains", StepType: "function", Status: "completed", @@ -442,7 +440,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan5ID, + RunID: runIDMap[scan5ID], StepName: "httpx", StepType: "bash", Status: "running", @@ -454,7 +452,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan5ID, + RunID: runIDMap[scan5ID], StepName: "screenshot", StepType: "bash", Status: "pending", @@ -475,7 +473,7 @@ func SeedDatabase(ctx context.Context) error { artifacts := []Artifact{ { ID: uuid.New().String(), - RunID: scan1ID, + RunID: runIDMap[scan1ID], Workspace: "example.com", Name: "final-subdomains.txt", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/example.com/subdomain/final-subdomains.txt", @@ -488,7 +486,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan1ID, + RunID: runIDMap[scan1ID], Workspace: "example.com", Name: "alive-hosts.txt", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/example.com/http/alive-hosts.txt", @@ -501,7 +499,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan1ID, + RunID: runIDMap[scan1ID], Workspace: "example.com", Name: "httpx-output.json", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/example.com/http/httpx-output.json", @@ -514,7 +512,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan2ID, + RunID: runIDMap[scan2ID], Workspace: "api.example.com", Name: "masscan.txt", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/api.example.com/ports/masscan.txt", @@ -527,7 +525,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan1ID, + RunID: runIDMap[scan1ID], Workspace: "example.com", Name: "screenshots", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/example.com/screenshots/", @@ -541,7 +539,7 @@ func SeedDatabase(ctx context.Context) error { // Additional artifacts for scan2ID (port-scan) { ID: uuid.New().String(), - RunID: scan2ID, + RunID: runIDMap[scan2ID], Workspace: "api.example.com", Name: "nmap-services.xml", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/api.example.com/ports/nmap-services.xml", @@ -554,7 +552,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan2ID, + RunID: runIDMap[scan2ID], Workspace: "api.example.com", Name: "port-summary.csv", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/api.example.com/ports/port-summary.csv", @@ -567,7 +565,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan2ID, + RunID: runIDMap[scan2ID], Workspace: "api.example.com", Name: "targets.txt", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/api.example.com/targets.txt", @@ -581,7 +579,7 @@ func SeedDatabase(ctx context.Context) error { // Artifacts for scan3ID (vuln-scan - failed but has some outputs) { ID: uuid.New().String(), - RunID: scan3ID, + RunID: runIDMap[scan3ID], Workspace: "staging.test.local", Name: "nuclei-partial.json", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/staging.test.local/vuln/nuclei-partial.json", @@ -594,7 +592,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan3ID, + RunID: runIDMap[scan3ID], Workspace: "staging.test.local", Name: "targets-prepared.txt", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/staging.test.local/targets-prepared.txt", @@ -608,7 +606,7 @@ func SeedDatabase(ctx context.Context) error { // Artifacts for scan4ID (secondary.com - completed batch job) { ID: uuid.New().String(), - RunID: scan4ID, + RunID: runIDMap[scan4ID], Workspace: "secondary.com", Name: "final-subdomains.txt", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/secondary.com/subdomain/final-subdomains.txt", @@ -621,7 +619,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan4ID, + RunID: runIDMap[scan4ID], Workspace: "secondary.com", Name: "alive-hosts.txt", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/secondary.com/http/alive-hosts.txt", @@ -634,7 +632,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan4ID, + RunID: runIDMap[scan4ID], Workspace: "secondary.com", Name: "httpx-output.json", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/secondary.com/http/httpx-output.json", @@ -647,7 +645,7 @@ func SeedDatabase(ctx context.Context) error { }, { ID: uuid.New().String(), - RunID: scan4ID, + RunID: runIDMap[scan4ID], Workspace: "secondary.com", Name: "screenshots", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/secondary.com/screenshots/", @@ -661,7 +659,7 @@ func SeedDatabase(ctx context.Context) error { // Artifacts for scan5ID (tertiary.io - running batch job) { ID: uuid.New().String(), - RunID: scan5ID, + RunID: runIDMap[scan5ID], Workspace: "tertiary.io", Name: "final-subdomains.txt", ArtifactPath: "/home/osmedeus/workspaces-osmedeus/tertiary.io/subdomain/final-subdomains.txt", @@ -1639,7 +1637,7 @@ func SeedDatabase(ctx context.Context) error { Name: "subdomain-enum started (batch job)", Source: "executor", DataType: "scan", - Data: fmt.Sprintf(`{"scan_id":"%s","target":"secondary.com","job_id":"%s"}`, scan4ID, job1ID), + Data: fmt.Sprintf(`{"scan_id":"%s","target":"secondary.com","run_group_id":"%s"}`, scan4ID, runGroup1ID), Workspace: "secondary.com", RunID: scan4ID, WorkflowName: "subdomain-enum", @@ -1653,7 +1651,7 @@ func SeedDatabase(ctx context.Context) error { Name: "subdomain-enum completed (batch job)", Source: "executor", DataType: "scan", - Data: fmt.Sprintf(`{"scan_id":"%s","target":"secondary.com","job_id":"%s","duration_ms":3200000}`, scan4ID, job1ID), + Data: fmt.Sprintf(`{"scan_id":"%s","target":"secondary.com","run_group_id":"%s","duration_ms":3200000}`, scan4ID, runGroup1ID), Workspace: "secondary.com", RunID: scan4ID, WorkflowName: "subdomain-enum", @@ -1682,7 +1680,7 @@ func SeedDatabase(ctx context.Context) error { Name: "subdomain-enum started (batch job)", Source: "executor", DataType: "scan", - Data: fmt.Sprintf(`{"scan_id":"%s","target":"tertiary.io","job_id":"%s"}`, scan5ID, job1ID), + Data: fmt.Sprintf(`{"scan_id":"%s","target":"tertiary.io","run_group_id":"%s"}`, scan5ID, runGroup1ID), Workspace: "tertiary.io", RunID: scan5ID, WorkflowName: "subdomain-enum", @@ -1697,7 +1695,7 @@ func SeedDatabase(ctx context.Context) error { Name: "Batch job started", Source: "executor", DataType: "job", - Data: fmt.Sprintf(`{"job_id":"%s","targets":["example.com","secondary.com","tertiary.io"],"total_targets":3}`, job1ID), + Data: fmt.Sprintf(`{"run_group_id":"%s","targets":["example.com","secondary.com","tertiary.io"],"total_targets":3}`, runGroup1ID), Processed: true, CreatedAt: twoHoursAgo, }, @@ -2583,7 +2581,7 @@ func ListTables(ctx context.Context) ([]TableInfo, error) { // tableSearchColumns defines which columns to search for each table var tableSearchColumns = map[string][]string{ - "runs": {"id", "run_id", "job_id", "workflow_name", "target", "status", "error_message"}, + "runs": {"id", "run_uuid", "run_group_id", "workflow_name", "target", "status", "error_message"}, "step_results": {"id", "run_id", "step_name", "step_type", "status", "command", "output", "error_message"}, "artifacts": {"id", "run_id", "name", "path", "type", "description"}, "assets": {"workspace", "asset_value", "url", "title", "host_ip", "source", "labels"}, @@ -2597,7 +2595,7 @@ var tableSearchColumns = map[string][]string{ // tableDisplayColumns defines which columns to display by default for each table (ordered) var tableDisplayColumns = map[string][]string{ - "runs": {"run_id", "job_id", "workflow_name", "target", "status", "started_at", "completed_at"}, + "runs": {"run_uuid", "workflow_name", "target", "trigger_type", "status", "completed_steps", "total_steps", "started_at"}, "step_results": {"step_name", "step_type", "status", "duration_ms", "command"}, "artifacts": {"name", "path", "type", "size_bytes", "line_count"}, "assets": {"asset_value", "host_ip", "title", "status_code", "last_seen_at", "url"}, @@ -2611,7 +2609,7 @@ var tableDisplayColumns = map[string][]string{ // tableAllColumns defines ALL columns for each table (ordered, matching model structs) var tableAllColumns = map[string][]string{ - "runs": {"id", "run_id", "job_id", "workflow_name", "workflow_kind", "target", "params", + "runs": {"id", "run_uuid", "run_group_id", "workflow_name", "workflow_kind", "target", "params", "status", "workspace_path", "started_at", "completed_at", "error_message", "schedule_id", "trigger_type", "trigger_name", "total_steps", "completed_steps", "created_at", "updated_at"}, @@ -3701,7 +3699,7 @@ func GetRunByID(ctx context.Context, id string, includeSteps, includeArtifacts b } var run Run - query := db.NewSelect().Model(&run).Where("id = ? OR run_id = ?", id, id) + query := db.NewSelect().Model(&run).Where("run_uuid = ?", id) if includeSteps { query = query.Relation("Steps") @@ -3718,23 +3716,23 @@ func GetRunByID(ctx context.Context, id string, includeSteps, includeArtifacts b return &run, nil } -// GetRunsByJobID returns all runs for a given job ID -func GetRunsByJobID(ctx context.Context, jobID string) ([]*Run, error) { +// GetRunsByRunGroupID returns all runs for a given run group ID +func GetRunsByRunGroupID(ctx context.Context, runGroupID string) ([]*Run, error) { if db == nil { return nil, fmt.Errorf("database not connected") } - if jobID == "" { - return nil, fmt.Errorf("job ID is required") + if runGroupID == "" { + return nil, fmt.Errorf("run group ID is required") } var runs []*Run err := db.NewSelect(). Model(&runs). - Where("job_id = ?", jobID). + Where("run_group_id = ?", runGroupID). Order("created_at ASC"). Scan(ctx) if err != nil { - return nil, fmt.Errorf("failed to get runs by job ID: %w", err) + return nil, fmt.Errorf("failed to get runs by run group ID: %w", err) } return runs, nil @@ -3768,7 +3766,7 @@ func CreateRun(ctx context.Context, run *Run) error { } // UpdateRunStatus updates the status of a run -func UpdateRunStatus(ctx context.Context, id, status, errorMessage string) error { +func UpdateRunStatus(ctx context.Context, runUUID, status, errorMessage string) error { if db == nil { return fmt.Errorf("database not connected") } @@ -3778,7 +3776,7 @@ func UpdateRunStatus(ctx context.Context, id, status, errorMessage string) error Model((*Run)(nil)). Set("status = ?", status). Set("updated_at = ?", now). - Where("id = ? OR run_id = ?", id, id) + Where("run_uuid = ?", runUUID) if errorMessage != "" { query = query.Set("error_message = ?", errorMessage) @@ -3807,7 +3805,7 @@ func UpdateRunStatus(ctx context.Context, id, status, errorMessage string) error } // IncrementRunCompletedSteps increments the completed_steps counter for a run -func IncrementRunCompletedSteps(ctx context.Context, runID string) error { +func IncrementRunCompletedSteps(ctx context.Context, runUUID string) error { if db == nil { return fmt.Errorf("database not connected") } @@ -3817,7 +3815,7 @@ func IncrementRunCompletedSteps(ctx context.Context, runID string) error { Model((*Run)(nil)). Set("completed_steps = completed_steps + 1"). Set("updated_at = ?", now). - Where("id = ? OR run_id = ?", runID, runID). + Where("run_uuid = ?", runUUID). Exec(ctx) if err != nil { return fmt.Errorf("failed to increment completed steps: %w", err) @@ -3878,3 +3876,162 @@ func GetRunArtifacts(ctx context.Context, runID string) ([]Artifact, error) { return artifacts, nil } + +// StepResultQuery holds query parameters for listing step results +type StepResultQuery struct { + Workspace string + Status string + StepType string + RunID int64 + Offset int + Limit int +} + +// StepResultResult holds paginated step result results +type StepResultResult struct { + Data []StepResult + TotalCount int + Offset int + Limit int +} + +// ListStepResults returns step results with pagination and filtering +func ListStepResults(ctx context.Context, query StepResultQuery) (*StepResultResult, error) { + if db == nil { + return nil, fmt.Errorf("database not connected") + } + + result := &StepResultResult{ + Offset: query.Offset, + Limit: query.Limit, + } + + applyFilters := func(q *bun.SelectQuery) *bun.SelectQuery { + if query.RunID > 0 { + q = q.Where("sr.run_id = ?", query.RunID) + } + if query.Status != "" { + q = q.Where("sr.status = ?", query.Status) + } + if query.StepType != "" { + q = q.Where("sr.step_type = ?", query.StepType) + } + if query.Workspace != "" { + // Join with runs table to filter by workspace + q = q.Join("JOIN runs r ON r.id = sr.run_id"). + Where("r.workspace = ?", query.Workspace) + } + return q + } + + countQuery := db.NewSelect().Model((*StepResult)(nil)) + countQuery = applyFilters(countQuery) + count, err := countQuery.Count(ctx) + if err != nil { + return nil, fmt.Errorf("failed to count step results: %w", err) + } + result.TotalCount = count + + baseQuery := db.NewSelect().Model(&result.Data) + baseQuery = applyFilters(baseQuery) + err = baseQuery. + Order("sr.created_at DESC"). + Offset(query.Offset). + Limit(query.Limit). + Scan(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch step results: %w", err) + } + + return result, nil +} + +// AssetDiffSnapshotResult holds paginated asset diff snapshot results +type AssetDiffSnapshotResult struct { + Data []AssetDiffSnapshot + TotalCount int + Offset int + Limit int +} + +// ListAssetDiffSnapshots returns asset diff snapshots with pagination +func ListAssetDiffSnapshots(ctx context.Context, workspace string, offset, limit int) (*AssetDiffSnapshotResult, error) { + if db == nil { + return nil, fmt.Errorf("database not connected") + } + + result := &AssetDiffSnapshotResult{ + Offset: offset, + Limit: limit, + } + + countQuery := db.NewSelect().Model((*AssetDiffSnapshot)(nil)) + if workspace != "" { + countQuery = countQuery.Where("workspace_name = ?", workspace) + } + count, err := countQuery.Count(ctx) + if err != nil { + return nil, fmt.Errorf("failed to count asset diff snapshots: %w", err) + } + result.TotalCount = count + + baseQuery := db.NewSelect().Model(&result.Data) + if workspace != "" { + baseQuery = baseQuery.Where("workspace_name = ?", workspace) + } + err = baseQuery. + Order("created_at DESC"). + Offset(offset). + Limit(limit). + Scan(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch asset diff snapshots: %w", err) + } + + return result, nil +} + +// VulnDiffSnapshotResult holds paginated vulnerability diff snapshot results +type VulnDiffSnapshotResult struct { + Data []VulnDiffSnapshot + TotalCount int + Offset int + Limit int +} + +// ListVulnDiffSnapshots returns vulnerability diff snapshots with pagination +func ListVulnDiffSnapshots(ctx context.Context, workspace string, offset, limit int) (*VulnDiffSnapshotResult, error) { + if db == nil { + return nil, fmt.Errorf("database not connected") + } + + result := &VulnDiffSnapshotResult{ + Offset: offset, + Limit: limit, + } + + countQuery := db.NewSelect().Model((*VulnDiffSnapshot)(nil)) + if workspace != "" { + countQuery = countQuery.Where("workspace_name = ?", workspace) + } + count, err := countQuery.Count(ctx) + if err != nil { + return nil, fmt.Errorf("failed to count vulnerability diff snapshots: %w", err) + } + result.TotalCount = count + + baseQuery := db.NewSelect().Model(&result.Data) + if workspace != "" { + baseQuery = baseQuery.Where("workspace_name = ?", workspace) + } + err = baseQuery. + Order("created_at DESC"). + Offset(offset). + Limit(limit). + Scan(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch vulnerability diff snapshots: %w", err) + } + + return result, nil +} diff --git a/internal/distributed/master.go b/internal/distributed/master.go index d5fc594..1aa29dc 100644 --- a/internal/distributed/master.go +++ b/internal/distributed/master.go @@ -479,22 +479,22 @@ func (m *Master) processRunData(ctx context.Context, envelope *DataEnvelope) { repo := repository.NewRunRepository(m.db) - // Check if run exists (by run_id) - existing, err := repo.GetByRunID(ctx, run.RunID) + // Check if run exists (by run_uuid) + existing, err := repo.GetByRunID(ctx, run.RunUUID) if err == nil && existing != nil { // Update existing run run.ID = existing.ID if err := repo.Update(ctx, &run); err != nil { - m.logger.Error("failed to update run", zap.Error(err), zap.String("run_id", run.RunID)) + m.logger.Error("failed to update run", zap.Error(err), zap.String("run_uuid", run.RunUUID)) } else { - m.logger.Debug("updated run from worker", zap.String("run_id", run.RunID)) + m.logger.Debug("updated run from worker", zap.String("run_uuid", run.RunUUID)) } } else { // Create new run if err := repo.Create(ctx, &run); err != nil { - m.logger.Error("failed to create run", zap.Error(err), zap.String("run_id", run.RunID)) + m.logger.Error("failed to create run", zap.Error(err), zap.String("run_uuid", run.RunUUID)) } else { - m.logger.Debug("created run from worker", zap.String("run_id", run.RunID)) + m.logger.Debug("created run from worker", zap.String("run_uuid", run.RunUUID)) } } } @@ -514,7 +514,7 @@ func (m *Master) processStepData(ctx context.Context, envelope *DataEnvelope) { } else { m.logger.Debug("created step result from worker", zap.String("step_name", step.StepName), - zap.String("run_id", step.RunID), + zap.Int64("run_id", step.RunID), ) } } diff --git a/internal/executor/artifact_export.go b/internal/executor/artifact_export.go index 498925d..ed56e6d 100644 --- a/internal/executor/artifact_export.go +++ b/internal/executor/artifact_export.go @@ -15,7 +15,8 @@ import ( ) // RegisterArtifacts registers workflow reports and state files as artifacts in the database -func RegisterArtifacts(workflow *core.Workflow, execCtx *core.ExecutionContext, logger *zap.Logger) error { +// runID is the integer Run.ID used as a foreign key for artifacts +func RegisterArtifacts(workflow *core.Workflow, execCtx *core.ExecutionContext, runID int64, logger *zap.Logger) error { db := database.GetDB() if db == nil { return nil @@ -32,9 +33,6 @@ func RegisterArtifacts(workflow *core.Workflow, execCtx *core.ExecutionContext, } outputStr, _ := outputPath.(string) - // Get run ID - try dbRunID first (from server mode), then execCtx.RunID - runID := execCtx.RunID - // Register workflow reports for _, report := range workflow.Reports { // Render the path template diff --git a/internal/executor/bash_executor.go b/internal/executor/bash_executor.go index 05d4f1b..b2ac171 100644 --- a/internal/executor/bash_executor.go +++ b/internal/executor/bash_executor.go @@ -160,11 +160,20 @@ func (e *BashExecutor) executeCommand(ctx context.Context, command string, timeo if e.runner != nil { result, err := e.runner.Execute(ctx, command) duration := time.Since(startTime).Seconds() - if err != nil { + + // Check for context cancellation first (Ctrl+C or timeout) + // This must be checked before other errors because runners return nil error + // but set result.ExitCode to -1 when context is cancelled + if ctx.Err() != nil { if ctx.Err() == context.DeadlineExceeded { metrics.RecordToolExecution(toolName, "timeout", duration) return result.Output, fmt.Errorf("command timed out after %s", timeout) } + metrics.RecordToolExecution(toolName, "cancelled", duration) + return result.Output, ctx.Err() + } + + if err != nil { metrics.RecordToolExecution(toolName, "error", duration) return result.Output, fmt.Errorf("command failed: %w", err) } @@ -192,11 +201,17 @@ func (e *BashExecutor) executeCommand(ctx context.Context, command string, timeo output += "\n" + stderr.String() } - if err != nil { + // Check for context cancellation first (Ctrl+C or timeout) + if ctx.Err() != nil { if ctx.Err() == context.DeadlineExceeded { metrics.RecordToolExecution(toolName, "timeout", duration) return output, fmt.Errorf("command timed out after %s", timeout) } + metrics.RecordToolExecution(toolName, "cancelled", duration) + return output, ctx.Err() + } + + if err != nil { metrics.RecordToolExecution(toolName, "error", duration) return output, fmt.Errorf("command failed: %w\nstderr: %s", err, stderr.String()) } diff --git a/internal/executor/executor.go b/internal/executor/executor.go index 56f3c93..2bf4dff 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -2,6 +2,7 @@ package executor import ( "context" + "errors" "fmt" "math/rand" "os" @@ -61,7 +62,8 @@ type Executor struct { progressBar *terminal.ProgressBar disableWorkflowState bool // disable writing workflow YAML to output directory skipValidation bool // skip target type validation from dependencies.variables - dbRunID string // database run ID for tracking progress + dbRunUUID string // database run UUID for tracking progress + dbRunID int64 // database run ID for step result foreign keys onStepCompleted StepCompletedCallback // callback after each step completes loader *parser.Loader // workflow loader for loading nested modules in flows consoleCapture *console.Capture // console output capture for run-console.log @@ -128,12 +130,18 @@ func (e *Executor) SetProgressBar(pb *terminal.ProgressBar) { e.progressBar = pb } -// SetDBRunID sets the database run ID for progress tracking -func (e *Executor) SetDBRunID(runID string) { +// SetDBRunUUID sets the database run UUID for progress tracking +func (e *Executor) SetDBRunUUID(runUUID string) { + e.dbRunUUID = runUUID + // Initialize progress tracker with RunUUID + e.progressTracker = database.NewProgressTracker(runUUID, nil) +} + +// SetDBRunID sets the database run ID for step result foreign keys +func (e *Executor) SetDBRunID(runID int64) { e.dbRunID = runID - // Initialize batch buffers for database operations + // Initialize step result buffer with Run.ID (foreign key) e.stepResultBuffer = database.NewStepResultBuffer(runID, nil) - e.progressTracker = database.NewProgressTracker(runID, nil) } // SetOnStepCompleted sets the callback for step completion @@ -272,7 +280,8 @@ func (e *Executor) injectBuiltinVariables(cfg *config.Config, params map[string] // Auto-generated variables execCtx.SetVariable("TaskDate", now.Format("2006-01-02")) - execCtx.SetVariable("TaskID", execCtx.RunID) + execCtx.SetVariable("RunUUID", execCtx.RunUUID) + execCtx.SetVariable("DBRunID", e.dbRunID) // Integer Run.ID for database foreign keys execCtx.SetVariable("TimeStamp", fmt.Sprintf("%d", now.Unix())) execCtx.SetVariable("CurrentTime", now.Format("2006-01-02T15:04:05")) execCtx.SetVariable("Today", now.Format("2006-01-02")) @@ -335,7 +344,7 @@ func (e *Executor) debugLogTargetVariables(execCtx *core.ExecutionContext) { execCtx.Logger.Debug("Target variables", zap.String("workflow", execCtx.WorkflowName), - zap.String("run_id", execCtx.RunID), + zap.String("run_id", execCtx.RunUUID), zap.String("Target", getStr("Target")), zap.String("TargetSpace", getStr("TargetSpace")), zap.String("Output", getStr("Output")), @@ -756,18 +765,23 @@ func (e *Executor) ExecuteModule(ctx context.Context, module *core.Workflow, par } // Setup console capture to {{Output}}/run-console.log - if !e.dryRun { + // Only create if not already set up by parent (flow) to avoid truncating previous module output + createdCapture := false + if !e.dryRun && e.consoleCapture == nil { if logPath, ok := execCtx.GetVariable("StateConsoleLog"); ok { if logStr, ok := logPath.(string); ok && logStr != "" { var err error e.consoleCapture, err = console.StartCapture(logStr) if err != nil { e.logger.Warn("Failed to setup console capture", zap.Error(err)) + } else { + createdCapture = true } } } } - if e.consoleCapture != nil { + // Only defer cleanup if WE created the capture (not if inherited from flow) + if createdCapture { defer func() { _ = e.consoleCapture.Stop() e.consoleCapture = nil @@ -785,7 +799,7 @@ func (e *Executor) ExecuteModule(ctx context.Context, module *core.Workflow, par result := &core.WorkflowResult{ WorkflowName: module.Name, WorkflowKind: core.KindModule, - RunID: runID, + RunUUID: runID, Target: params["target"], Status: core.RunStatusRunning, StartTime: time.Now(), @@ -938,8 +952,8 @@ func (e *Executor) ExecuteModule(ctx context.Context, module *core.Workflow, par } // Call step completed callback (for database progress tracking) - if e.onStepCompleted != nil && e.dbRunID != "" { - e.onStepCompleted(ctx, e.dbRunID) + if e.onStepCompleted != nil && e.dbRunUUID != "" { + e.onStepCompleted(ctx, e.dbRunUUID) } e.logger.Debug("Step execution result", @@ -1037,7 +1051,7 @@ func (e *Executor) ExecuteModule(ctx context.Context, module *core.Workflow, par // Register artifacts (reports from workflow + state files) if !e.dryRun { - if err := RegisterArtifacts(module, execCtx, execCtx.Logger); err != nil { + if err := RegisterArtifacts(module, execCtx, e.dbRunID, execCtx.Logger); err != nil { execCtx.Logger.Warn("Failed to register artifacts", zap.Error(err)) } } @@ -1189,8 +1203,8 @@ func (e *Executor) executeStepsDAG(ctx context.Context, steps []core.Step, execC } // Callback - if e.onStepCompleted != nil && e.dbRunID != "" { - e.onStepCompleted(ctx, e.dbRunID) + if e.onStepCompleted != nil && e.dbRunUUID != "" { + e.onStepCompleted(ctx, e.dbRunUUID) } if err != nil { @@ -1368,7 +1382,7 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params result := &core.WorkflowResult{ WorkflowName: flow.Name, WorkflowKind: core.KindFlow, - RunID: runID, + RunUUID: runID, Target: params["target"], Status: core.RunStatusRunning, StartTime: time.Now(), @@ -1471,6 +1485,20 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params // Process ready queue (Kahn's algorithm) for len(ready) > 0 { + // Check for context cancellation at start of each iteration + select { + case <-ctx.Done(): + if e.progressBar != nil { + e.progressBar.Abort() + } + result.Status = core.RunStatusCancelled + result.EndTime = time.Now() + execCtx.Logger.Warn("Flow execution cancelled") + metrics.RecordWorkflowEnd(flow.Name, string(core.KindFlow), string(result.Status), result.EndTime.Sub(result.StartTime).Seconds()) + return result, ctx.Err() + default: + } + // Pop from ready queue modName := ready[0] ready = ready[1:] @@ -1587,6 +1615,18 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params // Execute the module moduleResult, err := e.ExecuteModule(ctx, module, mergedParams, cfg) if err != nil { + // Check for context cancellation FIRST (interrupt/timeout) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + if e.progressBar != nil { + e.progressBar.Abort() + } + result.Status = core.RunStatusCancelled + result.EndTime = time.Now() + execCtx.Logger.Warn("Flow execution cancelled", zap.Error(err)) + metrics.RecordWorkflowEnd(flow.Name, string(core.KindFlow), string(result.Status), result.EndTime.Sub(result.StartTime).Seconds()) + return result, err + } + execCtx.Logger.Error("Module execution failed", zap.String("module", modRef.Name), zap.Error(err)) @@ -1650,6 +1690,15 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params } } + // Check if context was cancelled during execution + if ctx.Err() != nil { + result.Status = core.RunStatusCancelled + result.EndTime = time.Now() + execCtx.Logger.Warn("Flow execution cancelled") + metrics.RecordWorkflowEnd(flow.Name, string(core.KindFlow), string(result.Status), result.EndTime.Sub(result.StartTime).Seconds()) + return result, ctx.Err() + } + result.Status = core.RunStatusCompleted result.EndTime = time.Now() result.Exports = execCtx.Exports @@ -1690,7 +1739,7 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params // Register artifacts (reports from workflow + state files) if !e.dryRun { - if err := RegisterArtifacts(flow, execCtx, execCtx.Logger); err != nil { + if err := RegisterArtifacts(flow, execCtx, e.dbRunID, execCtx.Logger); err != nil { execCtx.Logger.Warn("Failed to register artifacts", zap.Error(err)) } } @@ -1710,7 +1759,7 @@ func (e *Executor) ExecuteFlow(ctx context.Context, flow *core.Workflow, params // executeStep executes a single step func (e *Executor) executeStep(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) { - stepLogger := logger.WithStep(execCtx.WorkflowName, execCtx.RunID, step.Name) + stepLogger := logger.WithStep(execCtx.WorkflowName, execCtx.RunUUID, step.Name) stepLogger.Debug("executeStep called", zap.String("step_name", step.Name), zap.String("type", string(step.Type)), diff --git a/internal/executor/executor_test.go b/internal/executor/executor_test.go index c23611a..18ef6d7 100644 --- a/internal/executor/executor_test.go +++ b/internal/executor/executor_test.go @@ -1168,7 +1168,7 @@ func TestExecutionContext_CloneForLoop(t *testing.T) { // Verify metadata is copied assert.Equal(t, "test-workflow", clone.WorkflowName) assert.Equal(t, core.KindModule, clone.WorkflowKind) - assert.Equal(t, "run-123", clone.RunID) + assert.Equal(t, "run-123", clone.RunUUID) assert.Equal(t, "example.com", clone.Target) // Verify Params reference is shared (not copied) diff --git a/internal/executor/state_export.go b/internal/executor/state_export.go index ed120dc..ed0c22c 100644 --- a/internal/executor/state_export.go +++ b/internal/executor/state_export.go @@ -17,7 +17,7 @@ func buildExportContext(result *core.WorkflowResult, execCtx *core.ExecutionCont // Populate from execCtx if execCtx != nil { - ctx.RunID = execCtx.RunID + ctx.RunUUID = execCtx.RunUUID ctx.WorkflowName = execCtx.WorkflowName ctx.WorkflowKind = string(execCtx.WorkflowKind) ctx.Target = execCtx.Target @@ -28,8 +28,8 @@ func buildExportContext(result *core.WorkflowResult, execCtx *core.ExecutionCont // Populate/override from result if result != nil { - if ctx.RunID == "" { - ctx.RunID = result.RunID + if ctx.RunUUID == "" { + ctx.RunUUID = result.RunUUID } if ctx.WorkflowName == "" { ctx.WorkflowName = result.WorkflowName diff --git a/internal/functions/constants.go b/internal/functions/constants.go index bb5e12b..ef568f9 100644 --- a/internal/functions/constants.go +++ b/internal/functions/constants.go @@ -265,6 +265,10 @@ const ( FnDBVulnDiff = "db_vuln_diff" // db_vuln_diff(workspace) -> string (JSONL) FnDBAssetDiffToFile = "db_asset_diff_to_file" // db_asset_diff_to_file(workspace, dest) -> bool FnDBVulnDiffToFile = "db_vuln_diff_to_file" // db_vuln_diff_to_file(workspace, dest) -> bool + + // Run status functions - query run records + FnDBSelectRuns = "run_status" // run_status(workspace, format) -> string + FnDBSelectRunByUUID = "run_status_by_uuid" // run_status_by_uuid(uuid, format) -> string ) // AllFunctions returns a list of all available function names @@ -471,6 +475,8 @@ func AllFunctions() []string { FnDBVulnDiff, FnDBAssetDiffToFile, FnDBVulnDiffToFile, + FnDBSelectRuns, + FnDBSelectRunByUUID, // Installer functions FnGoGetter, @@ -756,6 +762,8 @@ func FunctionRegistry() map[string][]FunctionInfo { {FnDBVulnDiff, "db_vuln_diff(workspace)", "Get vulnerability diff as JSONL string", "string", "db_vuln_diff('{{Workspace}}')"}, {FnDBAssetDiffToFile, "db_asset_diff_to_file(workspace, dest)", "Write asset diff to JSONL file", "bool", "db_asset_diff_to_file('{{Workspace}}', '{{Output}}/asset-diff.jsonl')"}, {FnDBVulnDiffToFile, "db_vuln_diff_to_file(workspace, dest)", "Write vulnerability diff to JSONL file", "bool", "db_vuln_diff_to_file('{{Workspace}}', '{{Output}}/vuln-diff.jsonl')"}, + {FnDBSelectRuns, "run_status(workspace, format)", "Query run records by workspace. Format: markdown or jsonl", "string", "run_status('{{Workspace}}', 'markdown')"}, + {FnDBSelectRunByUUID, "run_status_by_uuid(uuid, format)", "Query run record by UUID. Format: markdown or jsonl", "string", "run_status_by_uuid('abc-123', 'jsonl')"}, }, CategoryInstaller: { {FnGoGetter, "go_getter(url, dest)", "Download files/repos using go-getter", "bool", "go_getter('https://github.com/user/repo.git?ref=main', '{{Output}}/repo')"}, diff --git a/internal/functions/db_functions.go b/internal/functions/db_functions.go index de3294c..086a154 100644 --- a/internal/functions/db_functions.go +++ b/internal/functions/db_functions.go @@ -535,7 +535,7 @@ func (vf *vmFunc) runtimeExport(call goja.FunctionCall) goja.Value { } ctx := &state.ExportContext{ - RunID: vf.getContext().scanID, + RunUUID: vf.getContext().scanID, WorkspaceName: vf.getContext().workspaceName, WorkspacePath: vf.getContext().workspacePath, WorkflowName: vf.getContext().workflowName, @@ -585,6 +585,9 @@ var assetColumns = []string{"id", "workspace", "asset_value", "url", "status_cod // vulnerabilityColumns defines the columns for vulnerability markdown table var vulnerabilityColumns = []string{"id", "workspace", "severity", "vuln_title", "asset_value"} +// runColumns defines the columns for run markdown table +var runColumns = []string{"run_uuid", "workflow_name", "target", "workspace", "status", "total_steps", "completed_steps", "started_at"} + // formatAsMarkdownTable converts a slice of maps to a markdown table func formatAsMarkdownTable(rows []map[string]interface{}, columns []string) string { if len(rows) == 0 { @@ -711,6 +714,31 @@ func vulnerabilityToMap(vuln *database.Vulnerability) map[string]interface{} { } } +// runToMap converts a Run struct to a map for formatting +func runToMap(run *database.Run) map[string]interface{} { + result := map[string]interface{}{ + "run_uuid": run.RunUUID, + "workflow_name": run.WorkflowName, + "workflow_kind": run.WorkflowKind, + "target": run.Target, + "workspace": run.Workspace, + "status": run.Status, + "total_steps": run.TotalSteps, + "completed_steps": run.CompletedSteps, + "trigger_type": run.TriggerType, + } + if run.StartedAt != nil { + result["started_at"] = run.StartedAt.Format(time.RFC3339) + } + if run.CompletedAt != nil { + result["completed_at"] = run.CompletedAt.Format(time.RFC3339) + } + if run.ErrorMessage != "" { + result["error_message"] = run.ErrorMessage + } + return result +} + // Forbidden SQL keywords for security var forbiddenSQLKeywords = []string{ "INSERT", "UPDATE", "DELETE", "DROP", "CREATE", "ALTER", @@ -1166,10 +1194,10 @@ func (vf *vmFunc) dbRegisterArtifact(call goja.FunctionCall) goja.Value { return vf.errorValue(fmt.Sprintf("file not found: %s", filePath)) } - // Get scan ID from context - scanID := vf.getContext().scanID - if scanID == "" { - return vf.errorValue("scan ID not set in context") + // Get run ID from context (integer for database foreign key) + runID := vf.getContext().runID + if runID == 0 { + return vf.errorValue("run ID not set in context") } // Count lines in file @@ -1188,7 +1216,7 @@ func (vf *vmFunc) dbRegisterArtifact(call goja.FunctionCall) goja.Value { // Create artifact record artifact := database.Artifact{ ID: artifactID, - RunID: scanID, + RunID: runID, Name: filepath.Base(filePath), ArtifactPath: filePath, ArtifactType: artifactType, @@ -1230,9 +1258,9 @@ func (vf *vmFunc) storeArtifact(call goja.FunctionCall) goja.Value { return vf.errorValue("workspace not set in context") } - runID := vf.getContext().scanID - if runID == "" { - return vf.errorValue("scan ID not set in context") + runID := vf.getContext().runID + if runID == 0 { + return vf.errorValue("run ID not set in context") } info, err := os.Stat(filePath) @@ -2522,3 +2550,103 @@ func fieldChangeSliceToMaps(changes []database.FieldChange) []map[string]interfa } return result } + +// dbSelectRuns queries run records by workspace +// Usage: run_status(workspace, format) -> string +func (vf *vmFunc) dbSelectRuns(call goja.FunctionCall) goja.Value { + logger.Get().Debug("Calling " + terminal.HiGreen(FnDBSelectRuns)) + + if len(call.Arguments) < 2 { + return vf.errorValue(FnDBSelectRuns + " requires 2 arguments: workspace, format") + } + + workspace := call.Argument(0).String() + format := call.Argument(1).String() + + if workspace == "undefined" { + workspace = "" + } + + if err := validateOutputFormat(format); err != nil { + return vf.errorValue(err.Error()) + } + + db := database.GetDB() + if db == nil { + return vf.errorValue("database not connected") + } + + ctx := context.Background() + var runs []*database.Run + query := db.NewSelect().Model(&runs).Order("started_at DESC") + + if workspace != "" { + query = query.Where("workspace = ?", workspace) + } + + query = query.Limit(1000) + + if err := query.Scan(ctx); err != nil { + return vf.errorValue(fmt.Sprintf("query failed: %v", err)) + } + + rows := make([]map[string]interface{}, len(runs)) + for i, run := range runs { + rows[i] = runToMap(run) + } + + var output string + switch format { + case "markdown": + output = formatAsMarkdownTable(rows, runColumns) + case "jsonl": + output = formatAsJSONL(rows) + } + + return vf.vm.ToValue(output) +} + +// dbSelectRunByUUID queries a run record by UUID +// Usage: run_status_by_uuid(uuid, format) -> string +func (vf *vmFunc) dbSelectRunByUUID(call goja.FunctionCall) goja.Value { + logger.Get().Debug("Calling " + terminal.HiGreen(FnDBSelectRunByUUID)) + + if len(call.Arguments) < 2 { + return vf.errorValue(FnDBSelectRunByUUID + " requires 2 arguments: uuid, format") + } + + runUUID := call.Argument(0).String() + format := call.Argument(1).String() + + if runUUID == "" || runUUID == "undefined" { + return vf.errorValue("uuid argument is required") + } + + if err := validateOutputFormat(format); err != nil { + return vf.errorValue(err.Error()) + } + + db := database.GetDB() + if db == nil { + return vf.errorValue("database not connected") + } + + ctx := context.Background() + var run database.Run + err := db.NewSelect().Model(&run).Where("run_uuid = ?", runUUID).Scan(ctx) + if err != nil { + return vf.errorValue(fmt.Sprintf("run not found: %v", err)) + } + + rows := []map[string]interface{}{runToMap(&run)} + + var output string + switch format { + case "markdown": + output = formatAsMarkdownTable(rows, runColumns) + case "jsonl": + output = formatAsJSONL(rows) + } + + return vf.vm.ToValue(output) +} diff --git a/internal/functions/goja_pool.go b/internal/functions/goja_pool.go index 7974702..878efd4 100644 --- a/internal/functions/goja_pool.go +++ b/internal/functions/goja_pool.go @@ -80,7 +80,8 @@ type VMContext struct { // Context fields for function execution (previously on GojaRuntime) workspaceName string stateFile string - scanID string + scanID string // RunUUID (string identifier) + runID int64 // Run.ID (integer for database foreign keys) workflowName string workflowKind string target string @@ -136,6 +137,7 @@ func (p *VMPool) Put(ctx *VMContext) { ctx.workspaceName = "" ctx.stateFile = "" ctx.scanID = "" + ctx.runID = 0 ctx.workflowName = "" ctx.workflowKind = "" ctx.target = "" @@ -171,9 +173,16 @@ func (v *VMContext) SetContext(ctx map[string]interface{}) { v.stateFile = sf } - // Extract scan ID - if sid, ok := ctx["TaskID"].(string); ok { + // Extract scan ID (RunUUID) - check both new and legacy variable names + if sid, ok := ctx["RunUUID"].(string); ok { v.scanID = sid + } else if sid, ok := ctx["TaskID"].(string); ok { + v.scanID = sid + } + + // Extract database Run.ID (integer for foreign keys) + if rid, ok := ctx["DBRunID"].(int64); ok { + v.runID = rid } // Extract workflow name and kind diff --git a/internal/functions/goja_runtime.go b/internal/functions/goja_runtime.go index 7d5c928..ce739ca 100644 --- a/internal/functions/goja_runtime.go +++ b/internal/functions/goja_runtime.go @@ -249,6 +249,8 @@ func (r *GojaRuntime) registerFunctionsOnVM(vm *goja.Runtime) { _ = vm.Set(FnDBVulnDiff, vf.dbVulnDiff) _ = vm.Set(FnDBAssetDiffToFile, vf.dbAssetDiffToFile) _ = vm.Set(FnDBVulnDiffToFile, vf.dbVulnDiffToFile) + _ = vm.Set(FnDBSelectRuns, vf.dbSelectRuns) + _ = vm.Set(FnDBSelectRunByUUID, vf.dbSelectRunByUUID) // Installer functions _ = vm.Set(FnGoGetter, vf.goGetter) diff --git a/internal/functions/markdown_functions.go b/internal/functions/markdown_functions.go index 0a56035..88ec86b 100644 --- a/internal/functions/markdown_functions.go +++ b/internal/functions/markdown_functions.go @@ -916,7 +916,7 @@ func (vf *vmFunc) getContextVariables() map[string]interface{} { varNames := []string{ "Target", "TargetSpace", "Workspace", "Output", "BaseFolder", "Binaries", "Data", "ExternalConfigs", "Workflows", "Workspaces", - "TaskID", "TaskDate", "Version", "DefaultUA", + "RunUUID", "TaskDate", "Version", "DefaultUA", } for _, name := range varNames { @@ -937,7 +937,7 @@ func (vf *vmFunc) getContextVariables() map[string]interface{} { ctx["TargetSpace"] = vmCtx.workspaceName } if vmCtx.scanID != "" { - ctx["TaskID"] = vmCtx.scanID + ctx["RunUUID"] = vmCtx.scanID } } @@ -1125,10 +1125,10 @@ func (vf *vmFunc) registerReportArtifact(filePath, name, description string) err } vmCtx := vf.getContext() - runID := "" + var runID int64 workspace := "" if vmCtx != nil { - runID = vmCtx.scanID + runID = vmCtx.runID workspace = vmCtx.workspaceName } diff --git a/internal/functions/registry_test.go b/internal/functions/registry_test.go index 246e0f2..5ff69cb 100644 --- a/internal/functions/registry_test.go +++ b/internal/functions/registry_test.go @@ -323,7 +323,8 @@ func TestRegistry_StoreArtifact(t *testing.T) { `store_artifact("`+filePath+`")`, map[string]interface{}{ "Workspace": "w1", - "TaskID": "r1", + "RunUUID": "r1", + "DBRunID": int64(1), }, ) require.NoError(t, err) @@ -334,7 +335,7 @@ func TestRegistry_StoreArtifact(t *testing.T) { var artifacts []database.Artifact err = db.NewSelect().Model(&artifacts). - Where("run_id = ?", "r1"). + Where("run_id = ?", int64(1)). Where("workspace = ?", "w1"). Scan(ctx) require.NoError(t, err) diff --git a/internal/heuristics/heuristics_test.go b/internal/heuristics/heuristics_test.go index d94529a..fd6d708 100644 --- a/internal/heuristics/heuristics_test.go +++ b/internal/heuristics/heuristics_test.go @@ -11,9 +11,9 @@ func TestDetectType_File(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.Remove(f.Name()) - f.Close() - + defer func() { _ = os.Remove(f.Name()) }() + _ = f.Close() + got := DetectType(f.Name()) if got != TargetTypeFile { t.Errorf("DetectType() = %v, want %v", got, TargetTypeFile) @@ -83,9 +83,9 @@ func TestAnalyze_FileTarget(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.Remove(f.Name()) - f.Close() - + defer func() { _ = os.Remove(f.Name()) }() + _ = f.Close() + info, err := Analyze(f.Name(), "basic") if err != nil { t.Errorf("Analyze() error = %v", err) diff --git a/internal/linter/rules.go b/internal/linter/rules.go index e02218e..f36ad53 100644 --- a/internal/linter/rules.go +++ b/internal/linter/rules.go @@ -48,7 +48,7 @@ var builtInVariables = map[string]bool{ // Metadata Variables "Version": true, "TaskDate": true, - "TaskID": true, + "RunUUID": true, "TimeStamp": true, "CurrentTime": true, "Today": true, diff --git a/internal/state/export.go b/internal/state/export.go index 7c1c573..13f91a6 100644 --- a/internal/state/export.go +++ b/internal/state/export.go @@ -31,21 +31,22 @@ func Export(stateFile string, ctx *ExportContext) error { UpdatedAt: time.Now(), } - // Try to load run from database first - runLoaded := false - if ctx != nil && ctx.RunID != "" && db != nil { - var run database.Run - err := db.NewSelect().Model(&run). - Where("id = ? OR run_id = ?", ctx.RunID, ctx.RunID).Scan(dbCtx) - if err == nil { - export.Run = runInfoFromDB(&run) - runLoaded = true - } + // Always use context data first for run info (has correct in-memory status) + // The DB read happens before status is updated, so it returns stale data. + // Context data comes from the in-memory result which has the correct status. + if ctx != nil { + export.Run = runInfoFromContext(ctx) } - // Fallback: populate run info from context - if !runLoaded && ctx != nil { - export.Run = runInfoFromContext(ctx) + // Optionally enrich with DB data for fields not available in context + // (currently all fields are available in context, so this is just for future-proofing) + if export.Run == nil && ctx != nil && ctx.RunUUID != "" && db != nil { + var run database.Run + err := db.NewSelect().Model(&run). + Where("run_uuid = ?", ctx.RunUUID).Scan(dbCtx) + if err == nil { + export.Run = runInfoFromDB(&run) + } } // Try to load workspace from database first @@ -86,13 +87,13 @@ func Export(stateFile string, ctx *ExportContext) error { func runInfoFromDB(run *database.Run) *RunInfo { return &RunInfo{ - RunID: run.RunID, + RunUUID: run.RunUUID, WorkflowName: run.WorkflowName, WorkflowKind: run.WorkflowKind, Target: run.Target, Params: run.Params, Status: run.Status, - WorkspacePath: run.WorkspacePath, + Workspace: run.Workspace, StartedAt: run.StartedAt, CompletedAt: run.CompletedAt, ErrorMessage: run.ErrorMessage, @@ -102,17 +103,17 @@ func runInfoFromDB(run *database.Run) *RunInfo { } func runInfoFromContext(ctx *ExportContext) *RunInfo { - if ctx.RunID == "" && ctx.WorkflowName == "" { + if ctx.RunUUID == "" && ctx.WorkflowName == "" { return nil } return &RunInfo{ - RunID: ctx.RunID, + RunUUID: ctx.RunUUID, WorkflowName: ctx.WorkflowName, WorkflowKind: ctx.WorkflowKind, Target: ctx.Target, Params: ctx.Params, Status: ctx.Status, - WorkspacePath: ctx.WorkspacePath, + Workspace: ctx.WorkspaceName, StartedAt: ctx.StartedAt, CompletedAt: ctx.CompletedAt, ErrorMessage: ctx.ErrorMessage, diff --git a/internal/state/export_test.go b/internal/state/export_test.go index f7b4873..88a2714 100644 --- a/internal/state/export_test.go +++ b/internal/state/export_test.go @@ -21,7 +21,7 @@ func TestExport_WithoutDatabase(t *testing.T) { // Create context without database now := time.Now() ctx := &ExportContext{ - RunID: "test-run-123", + RunUUID: "test-run-123", WorkflowName: "test-workflow", WorkflowKind: "module", Target: "example.com", @@ -55,8 +55,8 @@ func TestExport_WithoutDatabase(t *testing.T) { if export.Run == nil { t.Fatal("Expected Run to be populated") } - if export.Run.RunID != "test-run-123" { - t.Errorf("Expected RunID 'test-run-123', got '%s'", export.Run.RunID) + if export.Run.RunUUID != "test-run-123" { + t.Errorf("Expected RunID 'test-run-123', got '%s'", export.Run.RunUUID) } if export.Run.WorkflowName != "test-workflow" { t.Errorf("Expected WorkflowName 'test-workflow', got '%s'", export.Run.WorkflowName) @@ -111,7 +111,7 @@ func TestExport_MinimalContext(t *testing.T) { // Create minimal context ctx := &ExportContext{ - RunID: "minimal-run", + RunUUID: "minimal-run", WorkspaceName: "minimal-workspace", } @@ -136,8 +136,8 @@ func TestExport_MinimalContext(t *testing.T) { if export.Run == nil { t.Fatal("Expected Run to be populated") } - if export.Run.RunID != "minimal-run" { - t.Errorf("Expected RunID 'minimal-run', got '%s'", export.Run.RunID) + if export.Run.RunUUID != "minimal-run" { + t.Errorf("Expected RunID 'minimal-run', got '%s'", export.Run.RunUUID) } // Verify workspace info is present @@ -151,7 +151,7 @@ func TestExport_MinimalContext(t *testing.T) { func TestExport_EmptyStateFilePath(t *testing.T) { ctx := &ExportContext{ - RunID: "test", + RunUUID: "test", } err := Export("", ctx) diff --git a/internal/state/types.go b/internal/state/types.go index c4989d7..8007dab 100644 --- a/internal/state/types.go +++ b/internal/state/types.go @@ -12,13 +12,13 @@ type StateExport struct { // RunInfo contains run information for export (mirrors database.Run fields) type RunInfo struct { - RunID string `json:"run_id"` + RunUUID string `json:"run_uuid"` WorkflowName string `json:"workflow_name"` WorkflowKind string `json:"workflow_kind"` Target string `json:"target"` Params map[string]any `json:"params,omitempty"` Status string `json:"status"` - WorkspacePath string `json:"workspace_path"` + Workspace string `json:"workspace"` StartedAt *time.Time `json:"started_at,omitempty"` CompletedAt *time.Time `json:"completed_at,omitempty"` ErrorMessage string `json:"error_message,omitempty"` @@ -48,7 +48,7 @@ type WorkspaceInfo struct { // ExportContext provides the context needed for state export // This allows callers to provide whatever information they have available type ExportContext struct { - RunID string + RunUUID string WorkflowName string WorkflowKind string Target string diff --git a/internal/template/context.go b/internal/template/context.go index 192647c..6f8162a 100644 --- a/internal/template/context.go +++ b/internal/template/context.go @@ -19,7 +19,7 @@ func BuildContext(execCtx *core.ExecutionContext) map[string]interface{} { // Add standard variables ctx["workflow"] = execCtx.WorkflowName - ctx["run_id"] = execCtx.RunID + ctx["run_id"] = execCtx.RunUUID ctx["target"] = execCtx.Target ctx["workspace"] = execCtx.WorkspacePath ctx["base_folder"] = execCtx.BaseFolder diff --git a/internal/terminal/colors.go b/internal/terminal/colors.go index c71a769..ef6c3f0 100644 --- a/internal/terminal/colors.go +++ b/internal/terminal/colors.go @@ -2,6 +2,7 @@ package terminal import ( "os" + "strings" "golang.org/x/term" ) @@ -186,3 +187,21 @@ func HiBlue(s string) string { func Teal(s string) string { return colorize(colorTeal, s) } + +// ColorizeStatus applies ANSI color codes to status values for table display +func ColorizeStatus(status string) string { + switch strings.ToLower(status) { + case "running", "in_progress", "active": + return Blue(status) + case "failed", "error": + return Red(status) + case "completed", "success", "done": + return Green(status) + case "cancelled", "canceled": + return Yellow(status) + case "pending", "waiting", "queued": + return Gray(status) + default: + return status + } +} diff --git a/internal/workspace/workspace.go b/internal/workspace/workspace.go index ee44497..85369a3 100644 --- a/internal/workspace/workspace.go +++ b/internal/workspace/workspace.go @@ -16,7 +16,7 @@ type Workspace struct { LogDir string OutputDir string ReportDir string - RunID string + RunUUID string Target string } @@ -35,7 +35,7 @@ func NewWorkspace(baseDir, workflowName, runID, target string) (*Workspace, erro LogDir: filepath.Join(workflowDir, "logs"), OutputDir: filepath.Join(workflowDir, "output"), ReportDir: filepath.Join(workflowDir, "reports"), - RunID: runID, + RunUUID: runID, Target: target, } @@ -132,7 +132,7 @@ func (w *Workspace) GetVariables() map[string]interface{} { "output_dir": w.OutputDir, "log_dir": w.LogDir, "report_dir": w.ReportDir, - "run_id": w.RunID, + "run_id": w.RunUUID, "target": w.Target, } } diff --git a/lib/result.go b/lib/result.go index 050ee14..13caf8b 100644 --- a/lib/result.go +++ b/lib/result.go @@ -192,7 +192,7 @@ func fromWorkflowResult(result *core.WorkflowResult, outputPath string) *RunResu runResult := &RunResult{ WorkflowName: result.WorkflowName, - RunID: result.RunID, + RunID: result.RunUUID, Target: result.Target, Status: string(result.Status), StartTime: result.StartTime, diff --git a/pkg/cli/client.go b/pkg/cli/client.go new file mode 100644 index 0000000..64eef76 --- /dev/null +++ b/pkg/cli/client.go @@ -0,0 +1,698 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "os/signal" + "sort" + "strconv" + "strings" + "syscall" + "time" + + "github.com/charmbracelet/glamour" + "github.com/j3ssie/osmedeus/v5/internal/client" + "github.com/j3ssie/osmedeus/v5/internal/terminal" + "github.com/spf13/cobra" +) + +var ( + // Client connection flags + clientRemoteURL string + clientAuthKey string + + // Fetch subcommand flags + clientFetchTable string + clientFetchWorkspace string + clientFetchSeverity string + clientFetchStatus string + clientFetchLimit int + clientFetchOffset int + clientFetchRefresh string + + // Run subcommand flags + clientRunFlow string + clientRunModule string + clientRunTarget string + clientRunCancel string + + // Exec subcommand flags + clientExecScript string + clientExecTarget string +) + +// clientCmd is the parent command for interacting with a remote osmedeus server +var clientCmd = &cobra.Command{ + Use: "client", + Short: "Interact with a remote osmedeus server", + Long: UsageClient(), +} + +// clientFetchCmd fetches data from the server +var clientFetchCmd = &cobra.Command{ + Use: "fetch", + Short: "Fetch data from remote server (assets, runs, vulns, etc.)", + Long: UsageClientFetch(), + RunE: runClientFetch, +} + +// clientRunCmd creates or cancels runs on the server +var clientRunCmd = &cobra.Command{ + Use: "run", + Short: "Create or cancel a run on remote server", + Long: UsageClientRun(), + RunE: runClientRun, +} + +// clientExecCmd executes a function remotely +var clientExecCmd = &cobra.Command{ + Use: "exec [script]", + Short: "Execute a function on remote server", + Long: UsageClientExec(), + Args: cobra.MaximumNArgs(1), + RunE: runClientExec, +} + +func init() { + // Add persistent flags for client connection (available to all subcommands) + clientCmd.PersistentFlags().StringVar(&clientRemoteURL, "remote-url", "", "Remote server URL (default: $OSM_REMOTE_URL)") + clientCmd.PersistentFlags().StringVar(&clientAuthKey, "auth-key", "", "API authentication key (default: $OSM_REMOTE_AUTH_KEY)") + + // Fetch subcommand flags + clientFetchCmd.Flags().StringVarP(&clientFetchTable, "table", "t", "assets", "Table to fetch data from: runs, step_results, artifacts, assets, event_logs, schedules, workspaces, vulnerabilities, asset_diffs, vuln_diffs") + clientFetchCmd.Flags().StringVarP(&clientFetchWorkspace, "workspace", "w", "", "Filter by workspace name") + clientFetchCmd.Flags().StringVar(&clientFetchSeverity, "severity", "", "Filter by severity (for vulnerabilities only)") + clientFetchCmd.Flags().StringVar(&clientFetchStatus, "status", "", "Filter by status (for runs only)") + clientFetchCmd.Flags().IntVar(&clientFetchLimit, "limit", 20, "Maximum records to return") + clientFetchCmd.Flags().IntVar(&clientFetchOffset, "offset", 0, "Number of records to skip (pagination)") + clientFetchCmd.Flags().StringVar(&clientFetchRefresh, "refresh", "", "auto-refresh interval (e.g., 5s, 1m, 30s)") + + // Run subcommand flags + clientRunCmd.Flags().StringVarP(&clientRunFlow, "flow", "f", "", "Flow workflow name") + clientRunCmd.Flags().StringVarP(&clientRunModule, "module", "m", "", "Module workflow name") + clientRunCmd.Flags().StringVarP(&clientRunTarget, "target", "T", "", "Target to run against") + clientRunCmd.Flags().StringVar(&clientRunCancel, "cancel", "", "Run ID to cancel (switches to cancel mode)") + + // Exec subcommand flags + clientExecCmd.Flags().StringVarP(&clientExecScript, "script", "s", "", "Script to execute (alternative to positional arg)") + clientExecCmd.Flags().StringVarP(&clientExecTarget, "target", "t", "", "Target variable for script") + + // Build command hierarchy + clientCmd.AddCommand(clientFetchCmd) + clientCmd.AddCommand(clientRunCmd) + clientCmd.AddCommand(clientExecCmd) +} + +// createClient creates a new API client with the configured URL and auth key +func createClient() (*client.Client, error) { + return client.NewClient(clientRemoteURL, clientAuthKey) +} + +// outputJSON outputs data as JSON to stdout +func outputJSON(data interface{}) error { + output, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal JSON: %w", err) + } + fmt.Println(string(output)) + return nil +} + +// printClientTable prints data in a styled markdown table format +func printClientTable(tableName string, data interface{}, columns []string, pagination client.Pagination, recordCount int) { + printer := terminal.NewPrinter() + printer.Info("Table: %s", tableName) + + // Calculate pagination info + startRecord := pagination.Offset + 1 + endRecord := pagination.Offset + recordCount + if pagination.Total == 0 { + startRecord = 0 + } + fmt.Printf("Showing records %d-%d of %d\n\n", startRecord, endRecord, pagination.Total) + + // Format as markdown table + tableStr := formatClientMarkdownTable(data, columns, globalWidth) + + // Render with glamour + renderer, err := glamour.NewTermRenderer( + glamour.WithAutoStyle(), + glamour.WithWordWrap(0), + ) + if err == nil { + rendered, renderErr := renderer.Render(tableStr) + if renderErr == nil { + fmt.Print(rendered) + } else { + fmt.Println(tableStr) + } + } else { + fmt.Println(tableStr) + } +} + +// formatClientMarkdownTable formats records as a markdown table +func formatClientMarkdownTable(records interface{}, columns []string, maxWidth int) string { + // Convert records to []map[string]interface{} + jsonBytes, _ := json.Marshal(records) + var data []map[string]interface{} + if err := json.Unmarshal(jsonBytes, &data); err != nil { + return "No records found." + } + + if len(data) == 0 { + return "No records found." + } + + // Use specified columns or extract from first record + var headers []string + if len(columns) > 0 { + headers = columns + } else { + for key := range data[0] { + headers = append(headers, key) + } + sort.Strings(headers) + } + + // Build markdown table + var sb strings.Builder + + // Header row + sb.WriteString("| ") + sb.WriteString(strings.Join(headers, " | ")) + sb.WriteString(" |\n") + + // Separator row + sb.WriteString("|") + for range headers { + sb.WriteString(" --- |") + } + sb.WriteString("\n") + + // Data rows + for _, row := range data { + sb.WriteString("| ") + for i, h := range headers { + val := formatClientTableValue(row[h], maxWidth, h) + if i > 0 { + sb.WriteString(" | ") + } + sb.WriteString(val) + } + sb.WriteString(" |\n") + } + + return sb.String() +} + +// formatClientTableValue converts a value to string for markdown display +func formatClientTableValue(v interface{}, maxWidth int, columnName string) string { + if v == nil { + return "" + } + + var s string + switch val := v.(type) { + case string: + // Escape pipe characters and newlines + s = strings.ReplaceAll(val, "|", "\\|") + s = strings.ReplaceAll(s, "\n", " ") + case map[string]interface{}, []interface{}: + // Compact JSON for complex types + b, _ := json.Marshal(val) + s = string(b) + default: + s = fmt.Sprintf("%v", val) + } + + // Apply width limit + if maxWidth > 0 && len(s) > maxWidth { + if maxWidth > 3 { + s = s[:maxWidth-3] + "..." + } else { + s = s[:maxWidth] + } + } + + // Colorize status column + if columnName == "status" { + return terminal.ColorizeStatus(s) + } + return s +} + +// validFetchTables lists all valid table names for the fetch command +var validFetchTables = []string{ + "runs", "step_results", "artifacts", "assets", "event_logs", + "schedules", "workspaces", "vulnerabilities", "asset_diffs", "vuln_diffs", +} + +// clientTableDefaultColumns defines default columns for each table in client fetch output +var clientTableDefaultColumns = map[string][]string{ + "runs": {"run_uuid", "workflow_name", "target", "status", "completed_steps", "total_steps"}, + "step_results": {"step_name", "step_type", "status", "duration_ms", "command"}, + "artifacts": {"name", "artifact_path", "artifact_type", "size_bytes"}, + "assets": {"asset_value", "host_ip", "title", "status_code"}, + "event_logs": {"topic", "source", "processed", "workspace"}, + "schedules": {"name", "workflow_name", "trigger_type", "schedule", "is_enabled"}, + "workspaces": {"name", "data_source", "total_assets"}, + "vulnerabilities": {"vuln_title", "severity", "asset_value", "workspace"}, + "asset_diffs": {"workspace_name", "from_time", "to_time", "total_added", "total_removed"}, + "vuln_diffs": {"workspace_name", "from_time", "to_time", "total_added", "total_removed"}, +} + +// runClientFetchRefreshLoop continuously refreshes the fetch display at the specified interval +func runClientFetchRefreshLoop(cmd *cobra.Command, args []string, interval time.Duration) error { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sigChan) + + for { + fmt.Print("\033[2J\033[H") // Clear screen + if err := runClientFetchOnce(cmd, args); err != nil { + terminal.NewPrinter().Error("Fetch failed: %s", err) + } + fmt.Printf("\n%s Refreshing every %s. Press Ctrl+C to stop.\n", terminal.Gray("⟳"), interval) + + select { + case <-ticker.C: + continue + case <-sigChan: + fmt.Print("\033[2J\033[H") + terminal.NewPrinter().Info("Refresh stopped") + return nil + } + } +} + +// runClientFetch handles the fetch subcommand +func runClientFetch(cmd *cobra.Command, args []string) error { + // Check if refresh mode is enabled + if clientFetchRefresh != "" { + interval, err := time.ParseDuration(clientFetchRefresh) + if err != nil { + return fmt.Errorf("invalid refresh interval: %w", err) + } + if interval < time.Second { + return fmt.Errorf("refresh interval must be at least 1s") + } + return runClientFetchRefreshLoop(cmd, args, interval) + } + return runClientFetchOnce(cmd, args) +} + +// runClientFetchOnce performs a single fetch and displays the results +func runClientFetchOnce(cmd *cobra.Command, args []string) error { + c, err := createClient() + if err != nil { + return err + } + + ctx := context.Background() + query := url.Values{} + query.Set("limit", strconv.Itoa(clientFetchLimit)) + query.Set("offset", strconv.Itoa(clientFetchOffset)) + + var result interface{} + var endpoint string + + // Validate table name + validTable := false + for _, t := range validFetchTables { + if clientFetchTable == t { + validTable = true + break + } + } + if !validTable { + return fmt.Errorf("invalid table: %s (valid tables: %s)", clientFetchTable, strings.Join(validFetchTables, ", ")) + } + + switch clientFetchTable { + case "runs": + endpoint = "/runs" + if clientFetchStatus != "" { + query.Set("status", clientFetchStatus) + } + var resp client.RunsResponse + if err := c.Get(ctx, endpoint, query, &resp); err != nil { + return err + } + result = resp + if !globalJSON { + printRunsTable(resp) + return nil + } + + case "step_results": + endpoint = "/step-results" + if clientFetchWorkspace != "" { + query.Set("workspace", clientFetchWorkspace) + } + if clientFetchStatus != "" { + query.Set("status", clientFetchStatus) + } + var resp client.StepResultsResponse + if err := c.Get(ctx, endpoint, query, &resp); err != nil { + return err + } + result = resp + if !globalJSON { + printStepResultsTable(resp) + return nil + } + + case "artifacts": + endpoint = "/artifacts" + if clientFetchWorkspace != "" { + query.Set("workspace", clientFetchWorkspace) + } + var resp client.ArtifactsResponse + if err := c.Get(ctx, endpoint, query, &resp); err != nil { + return err + } + result = resp + if !globalJSON { + printArtifactsTable(resp) + return nil + } + + case "assets": + endpoint = "/assets" + if clientFetchWorkspace != "" { + query.Set("workspace", clientFetchWorkspace) + } + var resp client.AssetsResponse + if err := c.Get(ctx, endpoint, query, &resp); err != nil { + return err + } + result = resp + if !globalJSON { + printAssetsTable(resp) + return nil + } + + case "event_logs": + endpoint = "/event-logs" + if clientFetchWorkspace != "" { + query.Set("workspace", clientFetchWorkspace) + } + var resp client.EventLogsResponse + if err := c.Get(ctx, endpoint, query, &resp); err != nil { + return err + } + result = resp + if !globalJSON { + printEventLogsTable(resp) + return nil + } + + case "schedules": + endpoint = "/schedules" + var resp client.SchedulesResponse + if err := c.Get(ctx, endpoint, query, &resp); err != nil { + return err + } + result = resp + if !globalJSON { + printSchedulesTable(resp) + return nil + } + + case "workspaces": + endpoint = "/workspaces" + var resp client.WorkspacesResponse + if err := c.Get(ctx, endpoint, query, &resp); err != nil { + return err + } + result = resp + if !globalJSON { + printWorkspacesTable(resp) + return nil + } + + case "vulnerabilities": + endpoint = "/vulnerabilities" + if clientFetchWorkspace != "" { + query.Set("workspace", clientFetchWorkspace) + } + if clientFetchSeverity != "" { + query.Set("severity", clientFetchSeverity) + } + var resp client.VulnerabilitiesResponse + if err := c.Get(ctx, endpoint, query, &resp); err != nil { + return err + } + result = resp + if !globalJSON { + printVulnerabilitiesTable(resp) + return nil + } + + case "asset_diffs": + endpoint = "/assets/diffs" + if clientFetchWorkspace != "" { + query.Set("workspace", clientFetchWorkspace) + } + var resp client.AssetDiffsResponse + if err := c.Get(ctx, endpoint, query, &resp); err != nil { + return err + } + result = resp + if !globalJSON { + printAssetDiffsTable(resp) + return nil + } + + case "vuln_diffs": + endpoint = "/vulnerabilities/diffs" + if clientFetchWorkspace != "" { + query.Set("workspace", clientFetchWorkspace) + } + var resp client.VulnDiffsResponse + if err := c.Get(ctx, endpoint, query, &resp); err != nil { + return err + } + result = resp + if !globalJSON { + printVulnDiffsTable(resp) + return nil + } + } + + return outputJSON(result) +} + +// runClientRun handles the run subcommand (create or cancel) +func runClientRun(cmd *cobra.Command, args []string) error { + c, err := createClient() + if err != nil { + return err + } + + ctx := context.Background() + + // Cancel mode: --cancel flag is set + if clientRunCancel != "" { + var resp client.CancelRunResponse + if err := c.Delete(ctx, "/runs/"+clientRunCancel, &resp); err != nil { + return err + } + + if globalJSON { + return outputJSON(resp) + } + + printer := terminal.NewPrinter() + printer.Success("Run cancelled: %s", resp.RunID) + return nil + } + + // Create mode: create a new run + if clientRunFlow == "" && clientRunModule == "" { + return fmt.Errorf("either --flow or --module is required (or use --cancel to cancel a run)") + } + if clientRunFlow != "" && clientRunModule != "" { + return fmt.Errorf("only one of --flow or --module can be specified") + } + if clientRunTarget == "" { + return fmt.Errorf("--target is required") + } + + req := client.CreateRunRequest{ + Flow: clientRunFlow, + Module: clientRunModule, + Target: clientRunTarget, + } + + var resp client.CreateRunResponse + if err := c.Post(ctx, "/runs", req, &resp); err != nil { + return err + } + + if globalJSON { + return outputJSON(resp) + } + + // Print human-readable output + printer := terminal.NewPrinter() + printer.Success("Run created successfully") + fmt.Printf(" Workflow: %s (%s)\n", resp.Workflow, resp.Kind) + fmt.Printf(" Target: %s\n", resp.Target) + fmt.Printf(" Job ID: %s\n", resp.JobID) + if resp.RunID != "" { + fmt.Printf(" Run ID: %s\n", resp.RunID) + } + fmt.Printf(" Status: %s\n", resp.Status) + fmt.Printf(" Poll URL: %s\n", resp.PollURL) + + return nil +} + +// runClientExec handles the exec subcommand +func runClientExec(cmd *cobra.Command, args []string) error { + // Get script from args or flag + script := clientExecScript + if len(args) > 0 { + script = args[0] + } + if script == "" { + return fmt.Errorf("script is required (as argument or via --script)") + } + + c, err := createClient() + if err != nil { + return err + } + + ctx := context.Background() + req := client.FunctionEvalRequest{ + Script: script, + Target: clientExecTarget, + } + + var resp client.FunctionEvalResponse + if err := c.Post(ctx, "/functions/eval", req, &resp); err != nil { + return err + } + + if globalJSON { + return outputJSON(resp) + } + + // Print human-readable output + if resp.Error { + return fmt.Errorf("execution failed: %s", resp.Message) + } + + // Print result + switch v := resp.Result.(type) { + case string: + fmt.Println(v) + case nil: + // No output + default: + output, _ := json.MarshalIndent(v, "", " ") + fmt.Println(string(output)) + } + + return nil +} + +// printAssetsTable prints assets in a table format +func printAssetsTable(resp client.AssetsResponse) { + if len(resp.Data) == 0 { + terminal.NewPrinter().Info("No assets found") + return + } + printClientTable("assets", resp.Data, clientTableDefaultColumns["assets"], resp.Pagination, len(resp.Data)) +} + +// printWorkspacesTable prints workspaces in a table format +func printWorkspacesTable(resp client.WorkspacesResponse) { + if len(resp.Data) == 0 { + terminal.NewPrinter().Info("No workspaces found") + return + } + printClientTable("workspaces", resp.Data, clientTableDefaultColumns["workspaces"], resp.Pagination, len(resp.Data)) +} + +// printVulnerabilitiesTable prints vulnerabilities in a table format +func printVulnerabilitiesTable(resp client.VulnerabilitiesResponse) { + if len(resp.Data) == 0 { + terminal.NewPrinter().Info("No vulnerabilities found") + return + } + printClientTable("vulnerabilities", resp.Data, clientTableDefaultColumns["vulnerabilities"], resp.Pagination, len(resp.Data)) +} + +// printRunsTable prints runs in a table format +func printRunsTable(resp client.RunsResponse) { + if len(resp.Data) == 0 { + terminal.NewPrinter().Info("No runs found") + return + } + printClientTable("runs", resp.Data, clientTableDefaultColumns["runs"], resp.Pagination, len(resp.Data)) +} + +// printStepResultsTable prints step results in a table format +func printStepResultsTable(resp client.StepResultsResponse) { + if len(resp.Data) == 0 { + terminal.NewPrinter().Info("No step results found") + return + } + printClientTable("step_results", resp.Data, clientTableDefaultColumns["step_results"], resp.Pagination, len(resp.Data)) +} + +// printArtifactsTable prints artifacts in a table format +func printArtifactsTable(resp client.ArtifactsResponse) { + if len(resp.Data) == 0 { + terminal.NewPrinter().Info("No artifacts found") + return + } + printClientTable("artifacts", resp.Data, clientTableDefaultColumns["artifacts"], resp.Pagination, len(resp.Data)) +} + +// printEventLogsTable prints event logs in a table format +func printEventLogsTable(resp client.EventLogsResponse) { + if len(resp.Data) == 0 { + terminal.NewPrinter().Info("No event logs found") + return + } + printClientTable("event_logs", resp.Data, clientTableDefaultColumns["event_logs"], resp.Pagination, len(resp.Data)) +} + +// printSchedulesTable prints schedules in a table format +func printSchedulesTable(resp client.SchedulesResponse) { + if len(resp.Data) == 0 { + terminal.NewPrinter().Info("No schedules found") + return + } + printClientTable("schedules", resp.Data, clientTableDefaultColumns["schedules"], resp.Pagination, len(resp.Data)) +} + +// printAssetDiffsTable prints asset diffs in a table format +func printAssetDiffsTable(resp client.AssetDiffsResponse) { + if len(resp.Data) == 0 { + terminal.NewPrinter().Info("No asset diffs found") + return + } + printClientTable("asset_diffs", resp.Data, clientTableDefaultColumns["asset_diffs"], resp.Pagination, len(resp.Data)) +} + +// printVulnDiffsTable prints vulnerability diffs in a table format +func printVulnDiffsTable(resp client.VulnDiffsResponse) { + if len(resp.Data) == 0 { + terminal.NewPrinter().Info("No vulnerability diffs found") + return + } + printClientTable("vuln_diffs", resp.Data, clientTableDefaultColumns["vuln_diffs"], resp.Pagination, len(resp.Data)) +} + diff --git a/pkg/cli/db.go b/pkg/cli/db.go index bb19d02..2cf3435 100644 --- a/pkg/cli/db.go +++ b/pkg/cli/db.go @@ -5,8 +5,11 @@ import ( "encoding/json" "fmt" "os" + "os/signal" "sort" "strings" + "syscall" + "time" "github.com/charmbracelet/glamour" "github.com/j3ssie/osmedeus/v5/internal/config" @@ -16,20 +19,18 @@ import ( ) var ( - dbForce bool dbTable string dbOffset int dbLimit int - dbJSON bool dbNoTUI bool dbWhere []string dbColumns string dbSearch string - dbWidth int dbAll bool dbIndexForce bool dbListColumns bool dbExcludeColumns string + dbRefresh string ) // defaultHiddenColumns are columns hidden by default for all tables @@ -37,7 +38,7 @@ var defaultHiddenColumns = []string{"id", "created_at", "updated_at", "completed // tableDefaultColumns defines default columns for specific tables var tableDefaultColumns = map[string][]string{ - "runs": {"run_id", "job_id", "workflow_name", "target", "status", "started_at"}, + "runs": {"run_uuid", "workflow_name", "target", "workspace", "trigger_type", "status", "completed_steps", "total_steps"}, "step_results": {"step_name", "step_type", "status", "duration_ms", "command"}, "artifacts": {"name", "path", "type", "size_bytes", "line_count"}, "assets": {"asset_value", "host_ip", "title", "status_code", "last_seen_at", "technologies"}, @@ -52,6 +53,7 @@ var dbCmd = &cobra.Command{ Use: "db", Short: "Database management commands", Long: UsageDB(), + RunE: runDBList, } // dbSeedCmd - seed database with sample data @@ -111,20 +113,21 @@ Use --force to re-index all workflows regardless of checksum.`, } func init() { - dbCleanCmd.Flags().BoolVar(&dbForce, "force", false, "skip confirmation prompt") + // Note: --force flag is now global (defined in root.go) - dbListCmd.Flags().StringVarP(&dbTable, "table", "t", "", "table name to list records from (runs, step_results, artifacts, assets, event_logs, schedules, workspaces)") - dbListCmd.Flags().IntVar(&dbOffset, "offset", 0, "number of records to skip (for pagination)") - dbListCmd.Flags().IntVar(&dbLimit, "limit", 50, "maximum number of records to return") - dbListCmd.Flags().BoolVar(&dbJSON, "json", false, "output records as JSON only (no extra output, bypasses TUI)") - dbListCmd.Flags().BoolVar(&dbNoTUI, "no-tui", false, "disable interactive TUI mode, use plain text output") - dbListCmd.Flags().StringArrayVar(&dbWhere, "where", nil, "filter records (key=value format, can be repeated) - only with --no-tui") - dbListCmd.Flags().StringVar(&dbColumns, "columns", "", "comma-separated columns to display (default: all) - only with --no-tui") - dbListCmd.Flags().StringVar(&dbSearch, "search", "", "search all columns for substring (case-insensitive) - only with --no-tui") - dbListCmd.Flags().IntVar(&dbWidth, "width", 30, "max column width for table display (0 = no limit) - only with --no-tui") - dbListCmd.Flags().BoolVar(&dbAll, "all", false, "show all columns including hidden ones (id, timestamps) - only with --no-tui") - dbListCmd.Flags().BoolVar(&dbListColumns, "list-columns", false, "list all available columns for the specified table") - dbListCmd.Flags().StringVar(&dbExcludeColumns, "exclude-columns", "", "comma-separated column names to exclude from output") + // Use persistent flags on dbCmd so they work on both `db` and `db ls` + dbCmd.PersistentFlags().StringVarP(&dbTable, "table", "t", "", "table name to list records from (runs, step_results, artifacts, assets, event_logs, schedules, workspaces)") + dbCmd.PersistentFlags().IntVar(&dbOffset, "offset", 0, "number of records to skip (for pagination)") + dbCmd.PersistentFlags().IntVar(&dbLimit, "limit", 50, "maximum number of records to return") + // Note: --json and --width flags are now global (defined in root.go) + dbCmd.PersistentFlags().BoolVar(&dbNoTUI, "no-tui", false, "disable interactive TUI mode, use plain text output") + dbCmd.PersistentFlags().StringArrayVar(&dbWhere, "where", nil, "filter records (key=value format, can be repeated) - only with --no-tui") + dbCmd.PersistentFlags().StringVar(&dbColumns, "columns", "", "comma-separated columns to display (default: all) - only with --no-tui") + dbCmd.PersistentFlags().StringVar(&dbSearch, "search", "", "search all columns for substring (case-insensitive) - only with --no-tui") + dbCmd.PersistentFlags().BoolVar(&dbAll, "all", false, "show all columns including hidden ones (id, timestamps) - only with --no-tui") + dbCmd.PersistentFlags().BoolVar(&dbListColumns, "list-columns", false, "list all available columns for the specified table") + dbCmd.PersistentFlags().StringVar(&dbExcludeColumns, "exclude-columns", "", "comma-separated column names to exclude from output") + dbCmd.PersistentFlags().StringVar(&dbRefresh, "refresh", "", "auto-refresh interval (e.g., 5s, 1m, 30s)") dbIndexWorkflowCmd.Flags().BoolVar(&dbIndexForce, "force", false, "force re-index all workflows regardless of checksum") @@ -191,7 +194,7 @@ func runDBClean(cmd *cobra.Command, args []string) error { return fmt.Errorf("configuration not loaded") } - if !dbForce { + if !globalForce { printer.Warning("This will delete ALL data from the database!") printer.Warning("Use --force to skip this confirmation") return fmt.Errorf("operation aborted: use --force to confirm") @@ -331,7 +334,7 @@ func runDBList(cmd *cobra.Command, args []string) error { } // JSON mode bypasses TUI entirely - if dbJSON { + if globalJSON { if dbTable != "" { return listTableRecordsJSON(ctx) } @@ -458,8 +461,53 @@ func listAllTables(ctx context.Context, cfg *config.Config, printer *terminal.Pr return nil } +// runDBRefreshLoop continuously refreshes the table display at the specified interval +func runDBRefreshLoop(ctx context.Context, cfg *config.Config, printer *terminal.Printer, interval time.Duration) error { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sigChan) + + for { + fmt.Print("\033[2J\033[H") // Clear screen + if err := listTableRecordsOnce(ctx, cfg, printer); err != nil { + printer.Error("Query failed: %s", err) + } + fmt.Printf("\n%s Refreshing every %s. Press Ctrl+C to stop.\n", terminal.Gray("⟳"), interval) + + select { + case <-ticker.C: + continue + case <-sigChan: + fmt.Print("\033[2J\033[H") + printer.Info("Refresh stopped") + return nil + case <-ctx.Done(): + return ctx.Err() + } + } +} + // listTableRecords lists records from a specific table with pagination func listTableRecords(ctx context.Context, cfg *config.Config, printer *terminal.Printer) error { + // Check if refresh mode is enabled + if dbRefresh != "" { + interval, err := time.ParseDuration(dbRefresh) + if err != nil { + return fmt.Errorf("invalid refresh interval: %w", err) + } + if interval < time.Second { + return fmt.Errorf("refresh interval must be at least 1s") + } + return runDBRefreshLoop(ctx, cfg, printer, interval) + } + return listTableRecordsOnce(ctx, cfg, printer) +} + +// listTableRecordsOnce performs a single query and displays the results +func listTableRecordsOnce(ctx context.Context, cfg *config.Config, printer *terminal.Printer) error { // Validate limit if dbLimit <= 0 { dbLimit = 50 @@ -488,7 +536,7 @@ func listTableRecords(ctx context.Context, cfg *config.Config, printer *terminal } // JSON-only output mode - if dbJSON { + if globalJSON { jsonBytes, err := json.Marshal(records.Records) if err != nil { return fmt.Errorf("failed to format records: %w", err) @@ -510,15 +558,17 @@ func listTableRecords(ctx context.Context, cfg *config.Config, printer *terminal printer.Info("Table: %s", records.Table) fmt.Printf("Showing records %d-%d of %d\n\n", startRecord, endRecord, records.TotalCount) - // Output as markdown table with glamour rendering - tableStr := formatAsMarkdownTable(records.Records, columns, dbWidth, hideDefaultColumns, excludeColumns) + // Output as markdown table + tableStr := formatAsMarkdownTable(records.Records, columns, globalWidth, hideDefaultColumns, excludeColumns) + + // Render with glamour for styled markdown table renderer, err := glamour.NewTermRenderer( glamour.WithAutoStyle(), - glamour.WithWordWrap(0), // No word wrap for tables + glamour.WithWordWrap(0), ) if err == nil { - rendered, err := renderer.Render(tableStr) - if err == nil { + rendered, renderErr := renderer.Render(tableStr) + if renderErr == nil { fmt.Print(rendered) } else { fmt.Println(tableStr) @@ -695,10 +745,12 @@ func formatTableValue(v interface{}, maxWidth int) string { // Apply width limit if maxWidth > 0 && len(s) > maxWidth { if maxWidth > 3 { - return s[:maxWidth-3] + "..." + s = s[:maxWidth-3] + "..." + } else { + s = s[:maxWidth] } - return s[:maxWidth] } + return s } diff --git a/pkg/cli/function.go b/pkg/cli/function.go index 4a1ae60..1e50046 100644 --- a/pkg/cli/function.go +++ b/pkg/cli/function.go @@ -5,8 +5,11 @@ import ( "fmt" "io" "os" + "os/signal" "strings" "sync" + "syscall" + "time" "github.com/j3ssie/osmedeus/v5/internal/config" "github.com/j3ssie/osmedeus/v5/internal/database" @@ -23,13 +26,16 @@ var ( evalStdin bool evalFunctionName string funcSearchFilter string - funcColumnWidth int funcShowExample bool // Bulk processing flags funcTargetsFile string funcFunctionFile string funcConcurrency int + + // Repeat flags + funcRepeat bool + funcRepeatWaitTime string ) // functionCmd is the parent command for function operations @@ -78,8 +84,12 @@ func init() { functionEvalCmd.Flags().StringVar(&funcFunctionFile, "function-file", "", "file containing the function/script to execute") functionEvalCmd.Flags().IntVarP(&funcConcurrency, "concurrency", "c", 1, "number of concurrent executions") + // Repeat flags + functionEvalCmd.Flags().BoolVar(&funcRepeat, "repeat", false, "repeat run after completion") + functionEvalCmd.Flags().StringVar(&funcRepeatWaitTime, "repeat-wait-time", "5s", "wait time between repeats (e.g., 30s, 20m, 10h, 1d)") + functionListCmd.Flags().StringVarP(&funcSearchFilter, "search", "s", "", "filter functions by name or description") - functionListCmd.Flags().IntVar(&funcColumnWidth, "width", 60, "max column width (wraps lines instead of truncating)") + // Note: --width flag is now global (defined in root.go) functionListCmd.Flags().BoolVar(&funcShowExample, "example", false, "show example usage below each function description") // evalCmd flags (same as functionEvalCmd - it's a shorthand) @@ -91,6 +101,8 @@ func init() { evalCmd.Flags().StringVarP(&funcTargetsFile, "targets", "T", "", "file containing targets (one per line)") evalCmd.Flags().StringVar(&funcFunctionFile, "function-file", "", "file containing the function/script to execute") evalCmd.Flags().IntVarP(&funcConcurrency, "concurrency", "c", 1, "number of concurrent executions") + evalCmd.Flags().BoolVar(&funcRepeat, "repeat", false, "repeat run after completion") + evalCmd.Flags().StringVar(&funcRepeatWaitTime, "repeat-wait-time", "5s", "wait time between repeats (e.g., 30s, 20m, 10h, 1d)") functionCmd.AddCommand(functionEvalCmd) functionCmd.AddCommand(functionListCmd) @@ -99,6 +111,24 @@ func init() { func runFunctionEval(cmd *cobra.Command, args []string) error { printer := terminal.NewPrinter() + // Parse repeat wait time + var waitDuration time.Duration + if funcRepeat { + var err error + waitDuration, err = parseRunDuration(funcRepeatWaitTime) + if err != nil { + return fmt.Errorf("invalid repeat-wait-time: %w", err) + } + printer.Info("Repeat mode enabled, wait time: %s", funcRepeatWaitTime) + } + + // Setup signal handling for repeat mode + sigChan := make(chan os.Signal, 1) + if funcRepeat { + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sigChan) + } + // Connect to database for db_* functions (skip if --disable-db is set) if !disableDB { cfg := config.Get() @@ -160,13 +190,38 @@ func runFunctionEval(cmd *cobra.Command, args []string) error { return fmt.Errorf("no script provided: use positional argument, -e flag, --function-file, or --stdin") } - // Bulk processing mode: process multiple targets from file - if funcTargetsFile != "" { - return runBulkFunctionEval(printer, script) - } + // Main execution loop + iteration := 0 + for { + iteration++ + if funcRepeat && iteration > 1 { + printer.Section(fmt.Sprintf("Repeat Iteration %d", iteration)) + } - // Single target execution (existing behavior) - return executeFunctionForTarget(printer, script, evalTarget) + var lastErr error + // Bulk processing mode: process multiple targets from file + if funcTargetsFile != "" { + lastErr = runBulkFunctionEval(printer, script) + } else { + // Single target execution (existing behavior) + lastErr = executeFunctionForTarget(printer, script, evalTarget) + } + + if !funcRepeat { + return lastErr + } + + printer.Info("Iteration %d completed. Waiting %s before next iteration...", iteration, funcRepeatWaitTime) + printer.Info("Press Ctrl+C to stop repeat mode") + + select { + case <-time.After(waitDuration): + // Continue to next iteration + case <-sigChan: + printer.Info("Interrupt received, stopping repeat mode") + return nil + } + } } // runBulkFunctionEval processes the script for multiple targets from a file @@ -371,8 +426,8 @@ func runFunctionList(cmd *cobra.Command, args []string) error { } headers := []string{"Category", "Function", "Description", "Returns"} - if funcColumnWidth > 0 { - printMarkdownTableWithWidth(headers, rows, funcColumnWidth) + if globalWidth > 0 { + printMarkdownTableWithWidth(headers, rows, globalWidth) } else { printMarkdownTable(headers, rows) } diff --git a/pkg/cli/health.go b/pkg/cli/health.go index 2cac20d..e59423a 100644 --- a/pkg/cli/health.go +++ b/pkg/cli/health.go @@ -220,8 +220,8 @@ func checkWorkflows(printer *terminal.Printer, cfg *config.Config) bool { } if len(workflowFiles) == 0 { - printer.Warning(" No workflows found in %s", terminal.White(cfg.WorkflowsPath)) - return false + printer.Error(" Workflows folder is empty: %s", terminal.White(cfg.WorkflowsPath)) + return true } p := parser.NewParser() diff --git a/pkg/cli/install.go b/pkg/cli/install.go index a65c89b..63f40a2 100644 --- a/pkg/cli/install.go +++ b/pkg/cli/install.go @@ -28,7 +28,6 @@ var ( nixInstallation bool nixPkgs []string installOptional bool - descMaxWidth int hideBinaryTags bool baseSample bool basePreset bool @@ -156,9 +155,24 @@ func runInstallWorkflow(cmd *cobra.Command, args []string) error { headers, ) - if workflowPreset { - printer := terminal.NewPrinter() + printer := terminal.NewPrinter() + // Check if workflow folder exists and prompt for confirmation + if _, err := os.Stat(inst.WorkflowFolder); err == nil { + if !globalForce { + printer.Warning("Existing workflow folder detected!") + printer.Warning("Path: %s", inst.WorkflowFolder) + printer.Warning("This operation will REMOVE the existing workflow folder and all its contents.") + fmt.Println() + + if !confirmPrompt("Do you want to continue?") { + printer.Info("Operation cancelled. Use --force to skip this confirmation.") + return nil + } + } + } + + if workflowPreset { workflowURL := os.Getenv("OSM_WORKFLOW_URL") if workflowURL != "" { printer.Info("Using workflow URL from OSM_WORKFLOW_URL environment variable") @@ -355,13 +369,13 @@ func runInstallBinary(cmd *cobra.Command, args []string) error { // Handle --list-registry-nix-build flag if listRegistryNixBuild { - return printNixBinaries(printer, descMaxWidth, !hideBinaryTags) + return printNixBinaries(printer, globalWidth, !hideBinaryTags) } // Handle --list-registry-direct-fetch flag if listRegistryDirectFetch { headers := parseCustomHeaders(customHeaders) - return printRegistryBinaries(registryPath, headers, printer, descMaxWidth, !hideBinaryTags) + return printRegistryBinaries(registryPath, headers, printer, globalWidth, !hideBinaryTags) } // Handle --go-getter flag (download/clone via go-getter) @@ -1519,6 +1533,7 @@ func init() { installBaseCmd.Flags().BoolVar(&baseSample, "sample", false, "initialize base folder from embedded sample (replaces existing base folder)") installBaseCmd.Flags().BoolVar(&basePreset, "preset", false, "install from OSM_PRESET_URL environment variable (default: DEFAULT_BASE_REPO)") installWorkflowCmd.Flags().BoolVar(&workflowPreset, "preset", false, "install from OSM_WORKFLOW_URL environment variable (default: DEFAULT_WORKFLOW_REPO)") + // Note: --force flag is now global (defined in root.go) // Binary command flags installBinaryCmd.Flags().StringSliceVarP(&binaryNames, "name", "n", []string{}, "binary name(s) to install (can be repeated)") @@ -1529,7 +1544,7 @@ func init() { installBinaryCmd.Flags().BoolVar(&nixBuildInstall, "nix-build-install", false, "use Nix to install binaries instead of direct downloads") installBinaryCmd.Flags().BoolVar(&nixInstallation, "nix-installation", false, "install Nix package manager (Determinate Systems installer)") installBinaryCmd.Flags().BoolVar(&installOptional, "install-optional", false, "include optional binaries in installation") - installBinaryCmd.Flags().IntVar(&descMaxWidth, "max-width", 80, "max width of description column in list output") + // Note: --width flag is now global (defined in root.go), --max-width removed in favor of --width installBinaryCmd.Flags().BoolVar(&hideBinaryTags, "disable-tags", false, "hide tags column in list output") installBinaryCmd.Flags().StringSliceVar(&goGetterSources, "go-getter", []string{}, "source URL(s) to download via go-getter (supports git repos, archives, etc.)") installBinaryCmd.Flags().StringVar(&goGetterDest, "go-getter-dest", "", "destination directory for go-getter downloads (default: $HOME)") diff --git a/pkg/cli/root.go b/pkg/cli/root.go index 1f81d39..336f57c 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -39,6 +39,11 @@ var ( ciOutputFormat bool skipAutoSetup bool + // Global flags available to all subcommands + globalForce bool + globalJSON bool + globalWidth int + // Build info - set via SetBuildInfo from main.go buildTime = "unknown" commitHash = "unknown" @@ -269,6 +274,11 @@ func init() { rootCmd.PersistentFlags().BoolVar(&ciOutputFormat, "ci-output-format", false, "output results in JSON format for CI pipelines") rootCmd.PersistentFlags().BoolVar(&skipAutoSetup, "skip-auto-setup", false, "skip automatic first-time setup") + // Global flags available to all subcommands + rootCmd.PersistentFlags().BoolVar(&globalForce, "force", false, "skip confirmation prompts and force operations") + rootCmd.PersistentFlags().BoolVar(&globalJSON, "json", false, "output in JSON format") + rootCmd.PersistentFlags().IntVar(&globalWidth, "width", 80, "max column width for table display (0 = no limit)") + // Suppress usage display and default error output (we handle errors in Execute()) rootCmd.SilenceUsage = true rootCmd.SilenceErrors = true @@ -297,6 +307,7 @@ func init() { rootCmd.AddCommand(versionCmd) rootCmd.AddCommand(updateCmd) rootCmd.AddCommand(evalCmd) + rootCmd.AddCommand(clientCmd) } // installRequiredBinaries installs all required binaries from the registry. @@ -482,6 +493,7 @@ func shouldSkipAutoSetup(cmd *cobra.Command) bool { "help": true, "update": true, "completion": true, + "client": true, } // Check command and all parent commands for c := cmd; c != nil; c = c.Parent() { diff --git a/pkg/cli/run.go b/pkg/cli/run.go index 5a5b5f6..047858f 100644 --- a/pkg/cli/run.go +++ b/pkg/cli/run.go @@ -26,6 +26,7 @@ import ( "github.com/j3ssie/osmedeus/v5/internal/database" "github.com/j3ssie/osmedeus/v5/internal/distributed" "github.com/j3ssie/osmedeus/v5/internal/executor" + "github.com/j3ssie/osmedeus/v5/internal/heuristics" "github.com/j3ssie/osmedeus/v5/internal/logger" "github.com/j3ssie/osmedeus/v5/internal/parser" "github.com/j3ssie/osmedeus/v5/internal/terminal" @@ -105,7 +106,7 @@ func init() { runCmd.Flags().BoolVarP(&distributedRun, "distributed-run", "D", false, "submit run to distributed worker queue (requires Redis)") runCmd.Flags().StringVar(&redisURLRun, "redis-url", "", "Redis connection URL for distributed mode (overrides settings)") runCmd.Flags().BoolVar(&repeatRun, "repeat", false, "repeat run after completion") - runCmd.Flags().StringVar(&repeatWaitTime, "repeat-wait-time", "1h", "wait time between repeats (e.g., 30s, 20m, 10h, 1d)") + runCmd.Flags().StringVar(&repeatWaitTime, "repeat-wait-time", "1m", "wait time between repeats (e.g., 30s, 20m, 10h, 1d)") runCmd.Flags().StringVar(&runTimeout, "timeout", "", "run timeout (e.g., 2h, 3h, 1d)") runCmd.Flags().BoolVar(&stdModule, "std-module", false, "read module YAML from stdin") runCmd.Flags().StringVar(&moduleURL, "module-url", "", "URL to fetch module YAML from (supports GitHub private repos)") @@ -710,9 +711,10 @@ func executeRunForTargetWithContext(ctx context.Context, workflow *core.Workflow } // Create run record in database (skip for dry-run) - var runID string + var runUUID string + var runID int64 if !dryRun { - runID = createCLIRunRecord(ctx, cfg, workflow, target, params) + runUUID, runID = createCLIRunRecord(ctx, cfg, workflow, target, params, loader) } // Create executor @@ -734,10 +736,11 @@ func executeRunForTargetWithContext(ctx context.Context, workflow *core.Workflow } // Set up database progress tracking - if runID != "" { + if runUUID != "" { + exec.SetDBRunUUID(runUUID) exec.SetDBRunID(runID) - exec.SetOnStepCompleted(func(stepCtx context.Context, dbRunID string) { - _ = database.IncrementRunCompletedSteps(stepCtx, dbRunID) + exec.SetOnStepCompleted(func(stepCtx context.Context, dbRunUUID string) { + _ = database.IncrementRunCompletedSteps(stepCtx, dbRunUUID) }) } @@ -807,21 +810,43 @@ func executeRunForTargetWithContext(ctx context.Context, workflow *core.Workflow // Handle target type mismatch error specially (print once, skip logging) if handleTargetTypeMismatchError(err) { // Update run status to failed in database - if runID != "" { - _ = database.UpdateRunStatus(ctx, runID, "failed", err.Error()) + // Use a fresh context for cleanup - the original ctx may be cancelled + if runUUID != "" { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = database.UpdateRunStatus(cleanupCtx, runUUID, "failed", err.Error()) + cleanupCancel() } return nil, err } - // Other errors: log normally - log.Error("Workflow execution failed", - zap.String("workflow", workflow.Name), - zap.String("target", target), - zap.Error(err), - ) - // Update run status to failed in database - if runID != "" { - _ = database.UpdateRunStatus(ctx, runID, "failed", err.Error()) + // Determine status based on error type + status := "failed" + if errors.Is(err, context.Canceled) { + status = "cancelled" + log.Warn("Workflow execution cancelled", + zap.String("workflow", workflow.Name), + zap.String("target", target), + ) + } else { + log.Error("Workflow execution failed", + zap.String("workflow", workflow.Name), + zap.String("target", target), + zap.Error(err), + ) + } + + // Update run status in database + // Use a fresh context for cleanup - the original ctx may be cancelled + if runUUID != "" { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second) + if updateErr := database.UpdateRunStatus(cleanupCtx, runUUID, status, err.Error()); updateErr != nil { + log.Error("Failed to update run status on cancellation", + zap.String("run_uuid", runUUID), + zap.String("status", status), + zap.Error(updateErr), + ) + } + cleanupCancel() } return nil, err } @@ -840,8 +865,8 @@ func executeRunForTargetWithContext(ctx context.Context, workflow *core.Workflow ) // Update run status to completed in database - if runID != "" { - _ = database.UpdateRunStatus(ctx, runID, "completed", "") + if runUUID != "" { + _ = database.UpdateRunStatus(ctx, runUUID, "completed", "") } // Print result summary for this target (skip if progress bar was used - it shows its own summary) @@ -852,25 +877,105 @@ func executeRunForTargetWithContext(ctx context.Context, workflow *core.Workflow return result, nil } +// calculateTotalSteps returns the appropriate step count based on workflow kind. +// For module workflows, it returns len(Steps). +// For flow workflows, it loads each module and sums their step counts. +func calculateTotalSteps(workflow *core.Workflow, loader *parser.Loader) int { + if workflow.Kind != core.KindFlow { + return len(workflow.Steps) + } + + // Flow workflow: sum steps from all modules + if loader == nil { + return len(workflow.Modules) + } + + log := logger.Get() + totalSteps := 0 + + for _, modRef := range workflow.Modules { + if modRef.Path == "" { + totalSteps++ + continue + } + + module, err := loader.LoadWorkflowByPath(modRef.Path) + if err != nil { + log.Warn("Failed to load module for step counting", + zap.String("module", modRef.Name), + zap.String("path", modRef.Path), + zap.Error(err), + ) + totalSteps++ + continue + } + + totalSteps += len(module.Steps) + } + + return totalSteps +} + +// computeWorkspace computes the workspace name from target and params +// This mirrors the executor's logic for computing TargetSpace +func computeWorkspace(target string, params map[string]string) string { + // If -S flag provided, use it directly + if spaceName := params["space_name"]; spaceName != "" { + return spaceName + } + + // Use heuristics to extract root domain/host (matches executor behavior) + heuristicsLevel := params["heuristics_check"] + if heuristicsLevel == "" { + heuristicsLevel = "basic" + } + if heuristicsLevel != "none" { + info, err := heuristics.Analyze(target, heuristicsLevel) + if err == nil && info != nil && info.RootDomain != "" { + return sanitizeTargetForWorkspace(info.RootDomain) + } + } + + // Otherwise, sanitize the target for filesystem safety + return sanitizeTargetForWorkspace(target) +} + +// sanitizeTargetForWorkspace creates a filesystem-safe workspace name from target +// This mirrors the executor's sanitizeTargetSpace function +func sanitizeTargetForWorkspace(target string) string { + sanitized := strings.Map(func(r rune) rune { + if strings.ContainsRune(`/\:*?"<>|`, r) { + return '_' + } + return r + }, target) + // Limit length to avoid filesystem issues + if len(sanitized) > 200 { + sanitized = sanitized[:200] + } + return sanitized +} + // createCLIRunRecord creates a run record in the database for CLI executions -func createCLIRunRecord(ctx context.Context, cfg *config.Config, workflow *core.Workflow, target string, params map[string]string) string { +// Returns the RunUUID (string) and Run.ID (int64) +func createCLIRunRecord(ctx context.Context, cfg *config.Config, workflow *core.Workflow, target string, params map[string]string, loader *parser.Loader) (string, int64) { log := logger.Get() // Connect to database _, err := database.Connect(cfg) if err != nil { log.Debug("Failed to connect to database for run record", zap.Error(err)) - return "" + return "", 0 } // Migrate database schema if needed if err := database.Migrate(ctx); err != nil { log.Debug("Failed to migrate database for run record", zap.Error(err)) - return "" + return "", 0 } now := time.Now() - runID := uuid.New().String() + runUUID := uuid.New().String() // Convert params to interface map paramsInterface := make(map[string]interface{}) @@ -878,9 +983,11 @@ func createCLIRunRecord(ctx context.Context, cfg *config.Config, workflow *core. paramsInterface[k] = v } + // Compute workspace from target and params + workspace := computeWorkspace(target, params) + run := &database.Run{ - ID: uuid.New().String(), - RunID: runID, + RunUUID: runUUID, WorkflowName: workflow.Name, WorkflowKind: string(workflow.Kind), Target: target, @@ -888,16 +995,17 @@ func createCLIRunRecord(ctx context.Context, cfg *config.Config, workflow *core. Status: "running", TriggerType: "cli", StartedAt: &now, - TotalSteps: len(workflow.Steps), + TotalSteps: calculateTotalSteps(workflow, loader), + Workspace: workspace, } if err := database.CreateRun(ctx, run); err != nil { log.Debug("Failed to create run record", zap.Error(err)) - return "" + return "", 0 } - log.Debug("Created run record", zap.String("run_id", runID)) - return runID + log.Debug("Created run record", zap.String("run_uuid", runUUID), zap.Int64("run_id", run.ID)) + return runUUID, run.ID } // collectTargets gathers targets from all input sources: flags, file, and stdin @@ -1178,7 +1286,7 @@ func printResultSummary(result *core.WorkflowResult) { // Log execution summary to state execution log file logger.Get().Info("Execution Summary", zap.String("workflow", result.WorkflowName), - zap.String("run_id", result.RunID), + zap.String("run_id", result.RunUUID), zap.String("target", result.Target), zap.String("status", string(result.Status)), zap.Duration("duration", result.EndTime.Sub(result.StartTime)), @@ -1200,7 +1308,7 @@ func printResultSummary(result *core.WorkflowResult) { printer.Section("Execution Summary") printer.KeyValue("Workflow", result.WorkflowName) - printer.KeyValue("Run ID", result.RunID) + printer.KeyValue("Run ID", result.RunUUID) printer.KeyValue("Target", result.Target) printer.KeyValue("Status", terminal.StatusBadge(string(result.Status))) printer.KeyValue("Duration", formatDuration(result.EndTime.Sub(result.StartTime))) @@ -1244,7 +1352,7 @@ func formatResultForCI(result *core.WorkflowResult) map[string]interface{} { return map[string]interface{}{ "workflow": result.WorkflowName, - "run_id": result.RunID, + "run_id": result.RunUUID, "target": result.Target, "status": string(result.Status), "duration": formatDuration(result.EndTime.Sub(result.StartTime)), diff --git a/pkg/cli/scan.go b/pkg/cli/scan.go index 8dab950..14bb361 100644 --- a/pkg/cli/scan.go +++ b/pkg/cli/scan.go @@ -46,7 +46,7 @@ func init() { scanCmd.Flags().BoolVarP(&distributedRun, "distributed-run", "D", false, "submit run to distributed worker queue (requires Redis)") scanCmd.Flags().StringVar(&redisURLRun, "redis-url", "", "Redis connection URL for distributed mode (overrides settings)") scanCmd.Flags().BoolVar(&repeatRun, "repeat", false, "repeat run after completion") - scanCmd.Flags().StringVar(&repeatWaitTime, "repeat-wait-time", "1h", "wait time between repeats (e.g., 30s, 20m, 10h, 1d)") + scanCmd.Flags().StringVar(&repeatWaitTime, "repeat-wait-time", "1m", "wait time between repeats (e.g., 30s, 20m, 10h, 1d)") scanCmd.Flags().StringVar(&runTimeout, "timeout", "", "run timeout (e.g., 2h, 3h, 1d)") scanCmd.Flags().BoolVar(&stdModule, "std-module", false, "read module YAML from stdin") scanCmd.Flags().BoolVar(&emptyTarget, "empty-target", false, "run without target (generates placeholder target)") diff --git a/pkg/cli/snapshot.go b/pkg/cli/snapshot.go index c3dc913..8628514 100644 --- a/pkg/cli/snapshot.go +++ b/pkg/cli/snapshot.go @@ -16,7 +16,6 @@ import ( var ( snapshotOutputPath string - snapshotForce bool snapshotSkipDB bool ) @@ -71,7 +70,7 @@ func init() { snapshotExportCmd.Flags().StringVarP(&snapshotOutputPath, "output", "o", "", "Custom output path for the snapshot") // Import flags - snapshotImportCmd.Flags().BoolVarP(&snapshotForce, "force", "f", false, "Overwrite existing workspace") + // Note: --force flag is now global (defined in root.go) snapshotImportCmd.Flags().BoolVar(&snapshotSkipDB, "skip-db", false, "Skip database import (files only)") // Add subcommands @@ -158,7 +157,7 @@ func runSnapshotImport(cmd *cobra.Command, args []string) error { var result *snapshot.ImportResult var err error - if snapshotForce { + if globalForce { result, err = snapshot.ForceImportWorkspace(source, cfg.WorkspacesPath, snapshotSkipDB, cfg) } else { result, err = snapshot.ImportWorkspace(source, cfg.WorkspacesPath, snapshotSkipDB, cfg) diff --git a/pkg/cli/update.go b/pkg/cli/update.go index 1a02c3f..a347a5c 100644 --- a/pkg/cli/update.go +++ b/pkg/cli/update.go @@ -17,7 +17,6 @@ import ( var ( updateCheck bool updateYes bool - updateForce bool updateVersion string ) @@ -32,7 +31,7 @@ var updateCmd = &cobra.Command{ func init() { updateCmd.Flags().BoolVar(&updateCheck, "check", false, "only check for updates without installing") updateCmd.Flags().BoolVarP(&updateYes, "yes", "y", false, "skip confirmation prompt") - updateCmd.Flags().BoolVar(&updateForce, "force", false, "force update even if current version is latest") + // Note: --force flag is now global (defined in root.go) updateCmd.Flags().StringVar(&updateVersion, "version", "", "update to a specific version (e.g., v5.1.0)") } @@ -84,7 +83,7 @@ func runUpdate(cmd *cobra.Command, args []string) error { return nil } - if !hasUpdate && !updateForce { + if !hasUpdate && !globalForce { printer.Success("You are running the latest version (%s)", currentVersion) return nil } @@ -128,9 +127,9 @@ func runUpdate(cmd *cobra.Command, args []string) error { var result *updater.UpdateResult if updateVersion != "" { - result, err = upd.UpdateToVersion(ctx, currentVersion, updateVersion, updateForce) + result, err = upd.UpdateToVersion(ctx, currentVersion, updateVersion, globalForce) } else { - result, err = upd.Update(ctx, currentVersion, updateForce) + result, err = upd.Update(ctx, currentVersion, globalForce) } if err != nil { diff --git a/pkg/cli/usage.go b/pkg/cli/usage.go index 75a7c21..e9f0c3a 100644 --- a/pkg/cli/usage.go +++ b/pkg/cli/usage.go @@ -947,6 +947,134 @@ func UsageFullExample() string { ` + docsFooter() } +// UsageClient returns the Long description for the client command +func UsageClient() string { + return terminal.BoldCyan("◆ Description") + ` + Interact with a remote osmedeus server via REST API. + +` + terminal.BoldCyan("▶ Environment Variables") + ` + ` + terminal.Yellow("OSM_REMOTE_URL") + ` Remote server URL (e.g., http://localhost:8002) + ` + terminal.Yellow("OSM_REMOTE_AUTH_KEY") + ` API authentication key for x-osm-api-key header + +` + terminal.BoldCyan("▶ Subcommands") + ` + • ` + terminal.Yellow("fetch") + ` - Fetch data from server (runs, assets, vulns, etc.) + • ` + terminal.Yellow("run") + ` - Create or cancel a run + • ` + terminal.Yellow("exec") + ` - Execute a function remotely + +` + terminal.BoldCyan("▷ Examples") + ` + ` + terminal.Green("# Configure via environment") + ` + export OSM_REMOTE_URL="http://localhost:8002" + export OSM_REMOTE_AUTH_KEY="your-api-key" + + ` + terminal.Green("# Fetch data from different tables") + ` + osmedeus client fetch ` + terminal.Yellow("--table") + ` assets + osmedeus client fetch ` + terminal.Yellow("-t") + ` runs + osmedeus client fetch ` + terminal.Yellow("-t") + ` vulnerabilities ` + terminal.Yellow("--severity") + ` critical + + ` + terminal.Green("# Create a run") + ` + osmedeus client run ` + terminal.Yellow("-f") + ` basic-recon ` + terminal.Yellow("-T") + ` example.com + + ` + terminal.Green("# Cancel a run") + ` + osmedeus client run ` + terminal.Yellow("--cancel") + ` abc123-run-uuid + + ` + terminal.Green("# Execute a function") + ` + osmedeus client exec 'log_info("Hello from remote")' + +` + docsFooter() +} + +// UsageClientFetch returns the Long description for the client fetch command +func UsageClientFetch() string { + return terminal.BoldCyan("◆ Description") + ` + Fetch data from the remote osmedeus server. + +` + terminal.BoldCyan("▶ Supported Tables") + ` + • ` + terminal.Yellow("runs") + ` - Workflow execution runs + • ` + terminal.Yellow("step_results") + ` - Step execution results + • ` + terminal.Yellow("artifacts") + ` - Output artifacts from runs + • ` + terminal.Yellow("assets") + ` - HTTP assets discovered during scans (default) + • ` + terminal.Yellow("event_logs") + ` - System event logs + • ` + terminal.Yellow("schedules") + ` - Scheduled workflow executions + • ` + terminal.Yellow("workspaces") + ` - Scan workspaces + • ` + terminal.Yellow("vulnerabilities") + ` - Discovered vulnerabilities + • ` + terminal.Yellow("asset_diffs") + ` - Asset diff snapshots + • ` + terminal.Yellow("vuln_diffs") + ` - Vulnerability diff snapshots + +` + terminal.BoldCyan("▷ Examples") + ` + ` + terminal.Green("# Fetch assets (default)") + ` + osmedeus client fetch + osmedeus client fetch ` + terminal.Yellow("-t") + ` assets ` + terminal.Yellow("-w") + ` example.com + + ` + terminal.Green("# Fetch runs") + ` + osmedeus client fetch ` + terminal.Yellow("--table") + ` runs + osmedeus client fetch ` + terminal.Yellow("-t") + ` runs ` + terminal.Yellow("--status") + ` running + + ` + terminal.Green("# Fetch vulnerabilities with severity filter") + ` + osmedeus client fetch ` + terminal.Yellow("-t") + ` vulnerabilities ` + terminal.Yellow("--severity") + ` critical + + ` + terminal.Green("# Fetch step results") + ` + osmedeus client fetch ` + terminal.Yellow("-t") + ` step_results + + ` + terminal.Green("# Pagination") + ` + osmedeus client fetch ` + terminal.Yellow("-t") + ` assets ` + terminal.Yellow("--limit") + ` 50 ` + terminal.Yellow("--offset") + ` 100 + + ` + terminal.Green("# JSON output") + ` + osmedeus client ` + terminal.Yellow("--json") + ` fetch ` + terminal.Yellow("-t") + ` runs + +` + docsFooter() +} + +// UsageClientRun returns the Long description for the client run command +func UsageClientRun() string { + return terminal.BoldCyan("◆ Description") + ` + Create or cancel a workflow run on the remote server. + +` + terminal.BoldCyan("▶ Create Mode Flags") + ` + ` + terminal.Yellow("-T, --target") + ` Target to run against (required) + One of: + ` + terminal.Yellow("-f, --flow") + ` Flow workflow name + ` + terminal.Yellow("-m, --module") + ` Module workflow name + +` + terminal.BoldCyan("▶ Cancel Mode") + ` + ` + terminal.Yellow("--cancel") + ` Run ID to cancel (switches to cancel mode) + +` + terminal.BoldCyan("▷ Examples") + ` + ` + terminal.Green("# Create a flow run") + ` + osmedeus client run ` + terminal.Yellow("-f") + ` basic-recon ` + terminal.Yellow("-T") + ` example.com + + ` + terminal.Green("# Create a module run") + ` + osmedeus client run ` + terminal.Yellow("-m") + ` subdomain ` + terminal.Yellow("-T") + ` example.com + + ` + terminal.Green("# Cancel a run by ID") + ` + osmedeus client run ` + terminal.Yellow("--cancel") + ` abc123-run-uuid + + ` + terminal.Green("# JSON output") + ` + osmedeus client ` + terminal.Yellow("--json") + ` run ` + terminal.Yellow("-f") + ` recon ` + terminal.Yellow("-T") + ` example.com + +` + docsFooter() +} + +// UsageClientExec returns the Long description for the client exec command +func UsageClientExec() string { + return terminal.BoldCyan("◆ Description") + ` + Execute a utility function on the remote server. + +` + terminal.BoldCyan("▷ Examples") + ` + ` + terminal.Green("# Execute a simple function") + ` + osmedeus client exec 'log_info("Hello from remote")' + + ` + terminal.Green("# With target variable") + ` + osmedeus client exec ` + terminal.Yellow("-t") + ` example.com 'fileExists("{{target}}/output.txt")' + + ` + terminal.Green("# Using --script flag") + ` + osmedeus client exec ` + terminal.Yellow("-s") + ` 'trim(" hello ")' + + ` + terminal.Green("# JSON output") + ` + osmedeus client ` + terminal.Yellow("--json") + ` exec 'trim(" test ")' + +` + docsFooter() +} + // docsFooter returns the documentation footer func docsFooter() string { return terminal.HiCyan("📖 Documentation: ") + terminal.HiWhite(core.DOCS) + "\n" diff --git a/pkg/cli/workflow.go b/pkg/cli/workflow.go index 6c08f8c..5c64b53 100644 --- a/pkg/cli/workflow.go +++ b/pkg/cli/workflow.go @@ -1101,7 +1101,7 @@ var workflowShowCmd = &cobra.Command{ // Metadata variables {terminal.Cyan("{{Version}}"), "Osmedeus version", terminal.Gray("")}, - {terminal.Cyan("{{TaskID}}"), "Unique task identifier (8 chars)", terminal.Yellow("")}, + {terminal.Cyan("{{RunUUID}}"), "Unique run identifier (UUID)", terminal.Yellow("")}, {terminal.Cyan("{{TaskDate}}"), "Task date (YYYY-MM-DD)", terminal.Yellow("")}, {terminal.Cyan("{{Today}}"), "Current date (YYYY-MM-DD)", terminal.Yellow("")}, {terminal.Cyan("{{TimeStamp}}"), "Unix timestamp", terminal.Yellow("")}, @@ -1113,7 +1113,7 @@ var workflowShowCmd = &cobra.Command{ // Compact: show variables in columns vars := []string{ "{{BaseFolder}}", "{{Binaries}}", "{{Data}}", "{{Workspaces}}", - "{{Target}}", "{{Output}}", "{{TaskID}}", "{{Today}}", + "{{Target}}", "{{Output}}", "{{RunUUID}}", "{{Today}}", "{{threads}}", "{{Version}}", "{{RandomString}}", } for i, v := range vars { diff --git a/pkg/server/handlers/asset_diff.go b/pkg/server/handlers/asset_diff.go index e8b4fdc..1db8b8d 100644 --- a/pkg/server/handlers/asset_diff.go +++ b/pkg/server/handlers/asset_diff.go @@ -141,6 +141,102 @@ func GetVulnerabilityDiff(cfg *config.Config) fiber.Handler { } } +// ListAssetDiffSnapshots handles listing stored asset diff snapshots +// @Summary List asset diff snapshots +// @Description Get a paginated list of stored asset diff snapshots +// @Tags Assets +// @Produce json +// @Param workspace query string false "Filter by workspace name" +// @Param offset query int false "Number of records to skip" default(0) +// @Param limit query int false "Maximum number of records to return" default(20) +// @Success 200 {object} map[string]interface{} "List of asset diff snapshots with pagination" +// @Failure 500 {object} map[string]interface{} "Failed to fetch asset diff snapshots" +// @Security BearerAuth +// @Router /osm/api/assets/diffs [get] +func ListAssetDiffSnapshots(cfg *config.Config) fiber.Handler { + return func(c *fiber.Ctx) error { + workspace := c.Query("workspace") + offset, _ := strconv.Atoi(c.Query("offset", "0")) + limit, _ := strconv.Atoi(c.Query("limit", "20")) + + if offset < 0 { + offset = 0 + } + if limit <= 0 { + limit = 20 + } + if limit > 10000 { + limit = 10000 + } + + ctx := context.Background() + result, err := database.ListAssetDiffSnapshots(ctx, workspace, offset, limit) + if err != nil { + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": true, + "message": err.Error(), + }) + } + + return c.JSON(fiber.Map{ + "data": result.Data, + "pagination": fiber.Map{ + "total": result.TotalCount, + "offset": result.Offset, + "limit": result.Limit, + }, + }) + } +} + +// ListVulnDiffSnapshots handles listing stored vulnerability diff snapshots +// @Summary List vulnerability diff snapshots +// @Description Get a paginated list of stored vulnerability diff snapshots +// @Tags Vulnerabilities +// @Produce json +// @Param workspace query string false "Filter by workspace name" +// @Param offset query int false "Number of records to skip" default(0) +// @Param limit query int false "Maximum number of records to return" default(20) +// @Success 200 {object} map[string]interface{} "List of vulnerability diff snapshots with pagination" +// @Failure 500 {object} map[string]interface{} "Failed to fetch vulnerability diff snapshots" +// @Security BearerAuth +// @Router /osm/api/vulnerabilities/diffs [get] +func ListVulnDiffSnapshots(cfg *config.Config) fiber.Handler { + return func(c *fiber.Ctx) error { + workspace := c.Query("workspace") + offset, _ := strconv.Atoi(c.Query("offset", "0")) + limit, _ := strconv.Atoi(c.Query("limit", "20")) + + if offset < 0 { + offset = 0 + } + if limit <= 0 { + limit = 20 + } + if limit > 10000 { + limit = 10000 + } + + ctx := context.Background() + result, err := database.ListVulnDiffSnapshots(ctx, workspace, offset, limit) + if err != nil { + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": true, + "message": err.Error(), + }) + } + + return c.JSON(fiber.Map{ + "data": result.Data, + "pagination": fiber.Map{ + "total": result.TotalCount, + "offset": result.Offset, + "limit": result.Limit, + }, + }) + } +} + // parseTime parses a time string in RFC3339 format or Unix timestamp func parseTime(s string) (time.Time, error) { // Try RFC3339 first diff --git a/pkg/server/handlers/eventlogs.go b/pkg/server/handlers/eventlogs.go index cda6555..e07b3bb 100644 --- a/pkg/server/handlers/eventlogs.go +++ b/pkg/server/handlers/eventlogs.go @@ -18,7 +18,7 @@ import ( // @Param name query string false "Filter by event name" // @Param source query string false "Filter by source (scheduler, api, webhook)" // @Param workspace query string false "Filter by workspace" -// @Param run_id query string false "Filter by run ID" +// @Param run_uuid query string false "Filter by run UUID" // @Param workflow_name query string false "Filter by workflow name" // @Param processed query string false "Filter by processed status (true/false)" // @Param offset query int false "Number of records to skip" default(0) @@ -50,7 +50,7 @@ func ListEventLogs(cfg *config.Config) fiber.Handler { Name: c.Query("name"), Source: c.Query("source"), Workspace: c.Query("workspace"), - RunID: c.Query("run_id"), + RunID: c.Query("run_uuid"), WorkflowName: c.Query("workflow_name"), Offset: offset, Limit: limit, diff --git a/pkg/server/handlers/handlers_test.go b/pkg/server/handlers/handlers_test.go index 863f8ef..1fc47a3 100644 --- a/pkg/server/handlers/handlers_test.go +++ b/pkg/server/handlers/handlers_test.go @@ -401,7 +401,7 @@ func TestListArtifactsVerifyExist(t *testing.T) { now := time.Now() art1 := &database.Artifact{ ID: "a1", - RunID: "r1", + RunID: 1, Workspace: "w1", Name: "file", ArtifactPath: filePath, @@ -413,7 +413,7 @@ func TestListArtifactsVerifyExist(t *testing.T) { } art2 := &database.Artifact{ ID: "a2", - RunID: "r1", + RunID: 1, Workspace: "w1", Name: "folder", ArtifactPath: folderPath, @@ -423,7 +423,7 @@ func TestListArtifactsVerifyExist(t *testing.T) { } art3 := &database.Artifact{ ID: "a3", - RunID: "r1", + RunID: 1, Workspace: "w1", Name: "missing", ArtifactPath: filepath.Join(tmpDir, "missing.txt"), diff --git a/pkg/server/handlers/jobs.go b/pkg/server/handlers/jobs.go index 369bbb6..ef6f128 100644 --- a/pkg/server/handlers/jobs.go +++ b/pkg/server/handlers/jobs.go @@ -8,12 +8,12 @@ import ( "github.com/j3ssie/osmedeus/v5/internal/database" ) -// JobStatus represents the aggregated status of a job +// JobStatus represents the aggregated status of a job (run group) type JobStatus struct { - JobID string `json:"job_id"` - Status string `json:"status"` // pending, running, completed, failed, partial - Runs []*database.Run `json:"runs"` - Progress JobProgress `json:"progress"` + RunGroupID string `json:"run_group_id"` + Status string `json:"status"` // pending, running, completed, failed, partial + Runs []*database.Run `json:"runs"` + Progress JobProgress `json:"progress"` } // JobProgress represents progress statistics for a job @@ -46,7 +46,7 @@ func GetJobStatus(cfg *config.Config) fiber.Handler { } ctx := context.Background() - runs, err := database.GetRunsByJobID(ctx, jobID) + runs, err := database.GetRunsByRunGroupID(ctx, jobID) if err != nil { return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ "error": true, @@ -80,10 +80,10 @@ func GetJobStatus(cfg *config.Config) fiber.Handler { status := aggregateStatus(progress) return c.JSON(fiber.Map{ - "job_id": jobID, - "status": status, - "runs": runs, - "progress": progress, + "run_group_id": jobID, + "status": status, + "runs": runs, + "progress": progress, }) } } diff --git a/pkg/server/handlers/runs.go b/pkg/server/handlers/runs.go index 76b4292..8e49769 100644 --- a/pkg/server/handlers/runs.go +++ b/pkg/server/handlers/runs.go @@ -27,6 +27,45 @@ func generateEmptyTarget() string { return fmt.Sprintf("empty-%s-%d", string(random), time.Now().Unix()) } +// calculateTotalSteps returns the appropriate step count based on workflow kind. +// For module workflows, it returns len(Steps). For flow workflows, it returns len(Modules). +func calculateTotalSteps(workflow *core.Workflow) int { + if workflow.Kind == core.KindFlow { + return len(workflow.Modules) + } + return len(workflow.Steps) +} + +// computeWorkspace computes the workspace name from target and params +// This mirrors the executor's logic for computing TargetSpace +func computeWorkspace(target string, params map[string]string) string { + // If space_name param provided (via -S flag or API), use it directly + if spaceName := params["space_name"]; spaceName != "" { + return spaceName + } + // Otherwise, sanitize the target for filesystem safety + return sanitizeTargetForWorkspace(target) +} + +// sanitizeTargetForWorkspace creates a filesystem-safe workspace name from target +// This mirrors the executor's sanitizeTargetSpace function +func sanitizeTargetForWorkspace(target string) string { + sanitized := make([]rune, 0, len(target)) + for _, r := range target { + if r == '/' || r == '\\' || r == ':' || r == '*' || r == '?' || r == '"' || r == '<' || r == '>' || r == '|' { + sanitized = append(sanitized, '_') + } else { + sanitized = append(sanitized, r) + } + } + result := string(sanitized) + // Limit length to avoid filesystem issues + if len(result) > 200 { + result = result[:200] + } + return result +} + // createRunRecord creates a database record for a run func createRunRecord(ctx context.Context, _ *config.Config, workflow *core.Workflow, target string, params map[string]string, triggerType, jobID string) (*database.Run, error) { now := time.Now() @@ -37,18 +76,21 @@ func createRunRecord(ctx context.Context, _ *config.Config, workflow *core.Workf paramsInterface[k] = v } + // Compute workspace from target and params + workspace := computeWorkspace(target, params) + run := &database.Run{ - ID: uuid.New().String(), - RunID: runID, + RunUUID: runID, WorkflowName: workflow.Name, WorkflowKind: string(workflow.Kind), Target: target, Params: paramsInterface, Status: "running", TriggerType: triggerType, - JobID: jobID, + RunGroupID: jobID, StartedAt: &now, - TotalSteps: len(workflow.Steps), + TotalSteps: calculateTotalSteps(workflow), + Workspace: workspace, } if err := database.CreateRun(ctx, run); err != nil { @@ -120,9 +162,11 @@ func executeRunsConcurrently( // Create run record in database run, err := createRunRecord(ctx, cfg, workflow, t, targetParams, "api", jobID) - var runID string + var runUUID string + var runID int64 if err == nil && run != nil { - runID = run.RunID + runUUID = run.RunUUID + runID = run.ID } // Execute workflow @@ -130,10 +174,11 @@ func executeRunsConcurrently( exec.SetServerMode(true) // Enable file logging for server mode // Set up database progress tracking - if runID != "" { + if runUUID != "" { + exec.SetDBRunUUID(runUUID) exec.SetDBRunID(runID) - exec.SetOnStepCompleted(func(stepCtx context.Context, dbRunID string) { - _ = database.IncrementRunCompletedSteps(stepCtx, dbRunID) + exec.SetOnStepCompleted(func(stepCtx context.Context, dbRunUUID string) { + _ = database.IncrementRunCompletedSteps(stepCtx, dbRunUUID) }) } @@ -145,11 +190,11 @@ func executeRunsConcurrently( } // Update run status in database - if runID != "" { + if runUUID != "" { if execErr != nil { - _ = database.UpdateRunStatus(ctx, runID, "failed", execErr.Error()) + _ = database.UpdateRunStatus(ctx, runUUID, "failed", execErr.Error()) } else { - _ = database.UpdateRunStatus(ctx, runID, "completed", "") + _ = database.UpdateRunStatus(ctx, runUUID, "completed", "") } } }(target) @@ -299,7 +344,7 @@ func CreateRun(cfg *config.Config) fiber.Handler { ctx := context.Background() run, _ := createRunRecord(ctx, cfgCopy, workflow, targets[0], params, "api", jobID) if run != nil { - runIDs = append(runIDs, run.RunID) + runIDs = append(runIDs, run.RunUUID) } exec := executor.NewExecutor() @@ -307,9 +352,10 @@ func CreateRun(cfg *config.Config) fiber.Handler { // Set up database progress tracking if run != nil { - exec.SetDBRunID(run.RunID) - exec.SetOnStepCompleted(func(stepCtx context.Context, dbRunID string) { - _ = database.IncrementRunCompletedSteps(stepCtx, dbRunID) + exec.SetDBRunUUID(run.RunUUID) + exec.SetDBRunID(run.ID) + exec.SetOnStepCompleted(func(stepCtx context.Context, dbRunUUID string) { + _ = database.IncrementRunCompletedSteps(stepCtx, dbRunUUID) }) } @@ -331,7 +377,7 @@ func CreateRun(cfg *config.Config) fiber.Handler { } }(func() string { if run != nil { - return run.RunID + return run.RunUUID } return "" }()) @@ -518,7 +564,7 @@ func CancelRun(cfg *config.Config) fiber.Handler { return c.JSON(fiber.Map{ "message": "Run cancelled successfully", "id": run.ID, - "run_id": run.RunID, + "run_id": run.RunUUID, }) } } diff --git a/pkg/server/handlers/steps.go b/pkg/server/handlers/steps.go new file mode 100644 index 0000000..d67671d --- /dev/null +++ b/pkg/server/handlers/steps.go @@ -0,0 +1,75 @@ +package handlers + +import ( + "context" + "strconv" + + "github.com/gofiber/fiber/v2" + "github.com/j3ssie/osmedeus/v5/internal/config" + "github.com/j3ssie/osmedeus/v5/internal/database" +) + +// ListStepResults handles listing all step results with pagination and filtering +// @Summary List step results +// @Description Get a paginated list of step results with optional filtering +// @Tags Steps +// @Produce json +// @Param workspace query string false "Filter by workspace name" +// @Param status query string false "Filter by status (pending, running, completed, failed)" +// @Param step_type query string false "Filter by step type (bash, function, etc.)" +// @Param run_id query int false "Filter by run ID" +// @Param offset query int false "Number of records to skip" default(0) +// @Param limit query int false "Maximum number of records to return" default(20) +// @Success 200 {object} map[string]interface{} "List of step results with pagination" +// @Failure 500 {object} map[string]interface{} "Failed to fetch step results" +// @Security BearerAuth +// @Router /osm/api/step-results [get] +func ListStepResults(cfg *config.Config) fiber.Handler { + return func(c *fiber.Ctx) error { + // Parse query parameters + workspace := c.Query("workspace") + status := c.Query("status") + stepType := c.Query("step_type") + runID, _ := strconv.ParseInt(c.Query("run_id", "0"), 10, 64) + offset, _ := strconv.Atoi(c.Query("offset", "0")) + limit, _ := strconv.Atoi(c.Query("limit", "20")) + + // Validate pagination + if offset < 0 { + offset = 0 + } + if limit <= 0 { + limit = 20 + } + if limit > 10000 { + limit = 10000 + } + + ctx := context.Background() + + // Get step results from database + result, err := database.ListStepResults(ctx, database.StepResultQuery{ + Workspace: workspace, + Status: status, + StepType: stepType, + RunID: runID, + Offset: offset, + Limit: limit, + }) + if err != nil { + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": true, + "message": err.Error(), + }) + } + + return c.JSON(fiber.Map{ + "data": result.Data, + "pagination": fiber.Map{ + "total": result.TotalCount, + "offset": result.Offset, + "limit": result.Limit, + }, + }) + } +} diff --git a/pkg/server/middleware/debug.go b/pkg/server/middleware/debug.go index f877295..eb492b6 100644 --- a/pkg/server/middleware/debug.go +++ b/pkg/server/middleware/debug.go @@ -97,6 +97,17 @@ func DebugErrorHandler(c *fiber.Ctx, err error) error { } } + // Custom message for 403 Forbidden + if code == fiber.StatusForbidden { + return c.Status(code).JSON(fiber.Map{ + "error": true, + "message": "Oh dear! It seems you've wandered off the path. If you'd like to see the UI page, please pop back root route at /", + "code": code, + "path": c.Path(), + "method": c.Method(), + }) + } + // Return detailed error response in debug mode return c.Status(code).JSON(fiber.Map{ "error": true, diff --git a/pkg/server/server.go b/pkg/server/server.go index 68c7b78..58caf23 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -284,13 +284,18 @@ func (s *Server) setupRoutes() { api.Get("/artifacts", handlers.ListArtifacts(s.config)) api.Get("/artifacts/:workspace_name", handlers.DownloadWorkspaceArtifact(s.config)) + // Step Results (global listing) + api.Get("/step-results", handlers.ListStepResults(s.config)) + // Assets api.Get("/assets", handlers.ListAssets(s.config)) api.Get("/assets/diff", handlers.GetAssetDiff(s.config)) + api.Get("/assets/diffs", handlers.ListAssetDiffSnapshots(s.config)) // Vulnerabilities api.Get("/vulnerabilities", handlers.ListVulnerabilities(s.config)) api.Get("/vulnerabilities/diff", handlers.GetVulnerabilityDiff(s.config)) + api.Get("/vulnerabilities/diffs", handlers.ListVulnDiffSnapshots(s.config)) api.Get("/vulnerabilities/summary", handlers.GetVulnerabilitySummary(s.config)) api.Get("/vulnerabilities/:id", handlers.GetVulnerability(s.config)) api.Post("/vulnerabilities", handlers.CreateVulnerability(s.config)) @@ -479,6 +484,14 @@ func errorHandler(c *fiber.Ctx, err error) error { code = e.Code } + // Custom message for 403 Forbidden + if code == fiber.StatusForbidden { + return c.Status(code).JSON(fiber.Map{ + "error": true, + "message": "Oh dear! It seems you've wandered off the path. If you'd like to see the UI page, please pop back root route at /", + }) + } + return c.Status(code).JSON(fiber.Map{ "error": true, "message": err.Error(), diff --git a/public/examples/osmedeus-base.example/markdown-report-templates/report-template.md b/public/examples/osmedeus-base.example/markdown-report-templates/report-template.md index 31f7206..a713c76 100644 --- a/public/examples/osmedeus-base.example/markdown-report-templates/report-template.md +++ b/public/examples/osmedeus-base.example/markdown-report-templates/report-template.md @@ -2,7 +2,7 @@ **Generated**: {{TaskDate}} **Target**: {{Target}} -**Task ID**: {{TaskID}} +**Run UUID**: {{RunUUID}} --- diff --git a/public/presets/sample-report-template.md b/public/presets/sample-report-template.md index 70ca3f1..cd3b0c6 100644 --- a/public/presets/sample-report-template.md +++ b/public/presets/sample-report-template.md @@ -2,7 +2,7 @@ **Generated**: {{TaskDate}} **Target**: {{Target}} -**Task ID**: {{TaskID}} +**Run UUID**: {{RunUUID}} --- diff --git a/test/testdata/workflows/extends/chain-b.yaml b/test/testdata/workflows/extends/chain-b.yaml index 530ac88..e68e632 100644 --- a/test/testdata/workflows/extends/chain-b.yaml +++ b/test/testdata/workflows/extends/chain-b.yaml @@ -5,8 +5,7 @@ extends: chain-a override: params: - param-a: - default: "overridden-by-b" + param-a: "overridden-by-b" # shorthand syntax steps: mode: append steps: diff --git a/test/testdata/workflows/extends/chain-c.yaml b/test/testdata/workflows/extends/chain-c.yaml index efa060b..bcb73e0 100644 --- a/test/testdata/workflows/extends/chain-c.yaml +++ b/test/testdata/workflows/extends/chain-c.yaml @@ -5,8 +5,7 @@ extends: chain-b override: params: - param-a: - default: "final-from-c" + param-a: "final-from-c" steps: mode: append steps: diff --git a/test/testdata/workflows/extends/child-override-steps.yaml b/test/testdata/workflows/extends/child-override-steps.yaml index cdf7af3..c57c0b3 100644 --- a/test/testdata/workflows/extends/child-override-steps.yaml +++ b/test/testdata/workflows/extends/child-override-steps.yaml @@ -5,8 +5,7 @@ extends: base-module override: params: - threads: - default: "20" + threads: "20" steps: mode: append steps: diff --git a/test/testdata/workflows/extends/child-simple.yaml b/test/testdata/workflows/extends/child-simple.yaml index 9eaa455..168ca9d 100644 --- a/test/testdata/workflows/extends/child-simple.yaml +++ b/test/testdata/workflows/extends/child-simple.yaml @@ -5,7 +5,5 @@ extends: base-module override: params: - threads: - default: "5" - timeout: - default: "1800" + threads: "5" + timeout: "1800" diff --git a/test/testdata/workflows/sample-report-workflow.yaml b/test/testdata/workflows/sample-report-workflow.yaml index 0202a28..a5db998 100644 --- a/test/testdata/workflows/sample-report-workflow.yaml +++ b/test/testdata/workflows/sample-report-workflow.yaml @@ -23,7 +23,7 @@ params: # {{Workspace}} - Current workspace name (usually the target) # {{Target}} - Target domain # {{Output}} - Output directory path -# {{TaskID}} - Current task/scan ID +# {{RunUUID}} - Current run UUID # {{TaskDate}} - Current date # {{Data}} - External data directory # {{Binaries}} - External binaries directory @@ -45,7 +45,7 @@ steps: **Workspace**: {{Workspace}} **Target**: {{Target}} **Generated**: {{TaskDate}} - **Task ID**: {{TaskID}} + **Run UUID**: {{RunUUID}} ---