mirror of
https://github.com/j3ssie/osmedeus.git
synced 2026-09-07 02:07:47 +02:00
Complete rewrite and re-architecture Osmedeus Engine in v5
This commit is contained in:
j3ssie
@@ -0,0 +1,5 @@
|
||||
# Exclude frontend files from GitHub language statistics
|
||||
public/ui/** linguist-vendored
|
||||
public/ui/**/*.js linguist-vendored
|
||||
public/ui/**/*.css linguist-vendored
|
||||
public/ui/**/*.html linguist-vendored
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
# If you prefer a global ignore file, keep this minimal.
|
||||
|
||||
# Binaries
|
||||
bin/
|
||||
dist/**
|
||||
dist/config.yaml
|
||||
osmedeus
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Go build artifacts
|
||||
*.test
|
||||
*.out
|
||||
*.cover
|
||||
coverage.out
|
||||
coverage.html
|
||||
|
||||
# OS / editor
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# cgo / compiler outputs
|
||||
*.o
|
||||
*.a
|
||||
*.obj
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# SQLite databases (test artifacts)
|
||||
*.sqlite
|
||||
*.sqlite-shm
|
||||
*.sqlite-wal
|
||||
|
||||
# Test workspace artifacts (state exports from executor tests)
|
||||
internal/executor/*/run-*.json
|
||||
internal/executor/*/run-*.yaml
|
||||
test/integration/*-test/
|
||||
!test/integration/*_test.go
|
||||
|
||||
OPTIMIZE.md
|
||||
OPTIMIZE-*.md
|
||||
PLANNING.md
|
||||
PLANNING-*.md
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
|
||||
version: 2
|
||||
|
||||
project_name: osmedeus
|
||||
|
||||
before:
|
||||
hooks:
|
||||
- go mod tidy
|
||||
|
||||
builds:
|
||||
- id: osmedeus
|
||||
main: ./cmd/osmedeus
|
||||
binary: osmedeus
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
goos:
|
||||
- linux
|
||||
- darwin
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
ldflags:
|
||||
- -s -w
|
||||
- -X main.BuildTime={{.Date}}
|
||||
- -X main.CommitHash={{.ShortCommit}}
|
||||
|
||||
archives:
|
||||
- id: default
|
||||
formats:
|
||||
- tar.gz
|
||||
name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
|
||||
files:
|
||||
- LICENSE*
|
||||
- README*
|
||||
- CHANGELOG*
|
||||
|
||||
checksum:
|
||||
name_template: "checksums.txt"
|
||||
algorithm: sha256
|
||||
|
||||
snapshot:
|
||||
version_template: "{{ .Tag }}-snapshot"
|
||||
|
||||
changelog:
|
||||
sort: asc
|
||||
filters:
|
||||
exclude:
|
||||
- "^docs:"
|
||||
- "^test:"
|
||||
- "^chore:"
|
||||
- Merge pull request
|
||||
- Merge branch
|
||||
|
||||
release:
|
||||
github:
|
||||
owner: osmedeus
|
||||
name: osmedeus
|
||||
draft: false
|
||||
prerelease: auto
|
||||
name_template: "{{.ProjectName}} {{.Tag}}"
|
||||
@@ -0,0 +1,182 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Build and Test Commands
|
||||
|
||||
```bash
|
||||
# Build
|
||||
make build # Build to bin/osmedeus
|
||||
make build-all # Cross-platform builds (linux, darwin, windows)
|
||||
|
||||
# Test
|
||||
make test-unit # Fast unit tests (no external dependencies)
|
||||
make test-integration # Integration tests (requires Docker)
|
||||
make test-e2e # E2E CLI tests (requires binary build)
|
||||
make test-e2e-ssh # SSH E2E tests (module & step level SSH runner)
|
||||
make test-e2e-api # API E2E tests (all endpoints with Redis + seeded DB)
|
||||
make test-distributed # Distributed run e2e tests (requires Docker for Redis)
|
||||
make test-docker # Docker runner tests
|
||||
make test-ssh # SSH runner unit tests (starts test SSH container)
|
||||
go test -v ./internal/functions/... # Run tests for specific package
|
||||
go test -v -run TestName ./... # Run single test by name
|
||||
|
||||
# Development
|
||||
make fmt # Format code
|
||||
make lint # Run golangci-lint
|
||||
make tidy # go mod tidy
|
||||
make run # Build and run
|
||||
|
||||
# Installation
|
||||
make install # Install to $GOBIN (or $GOPATH/bin)
|
||||
make swagger # Generate Swagger documentation
|
||||
|
||||
# Docker Toolbox
|
||||
make docker-toolbox # Build toolbox image (all tools pre-installed)
|
||||
make docker-toolbox-run # Start toolbox container
|
||||
make docker-toolbox-shell # Enter toolbox container shell
|
||||
|
||||
# UI
|
||||
make update-ui # Update embedded UI from dashboard build
|
||||
```
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
Osmedeus is a workflow engine for security automation. It executes YAML-defined workflows with support for multiple execution environments.
|
||||
|
||||
### Layered Architecture
|
||||
|
||||
```
|
||||
CLI/API (pkg/cli, pkg/server)
|
||||
↓
|
||||
Executor (internal/executor) - coordinates workflow execution
|
||||
↓
|
||||
StepDispatcher - routes to: BashExecutor, FunctionExecutor, ForeachExecutor, ParallelExecutor, RemoteBashExecutor, HTTPExecutor, LLMExecutor
|
||||
↓
|
||||
Runner (internal/runner) - executes commands via: HostRunner, DockerRunner, SSHRunner
|
||||
```
|
||||
|
||||
### Core Packages
|
||||
|
||||
| Package | Purpose |
|
||||
|---------|---------|
|
||||
| `internal/core` | Type definitions: Workflow, Step, Trigger, RunnerConfig, ExecutionContext |
|
||||
| `internal/parser` | YAML parsing, validation, and caching (Loader) |
|
||||
| `internal/executor` | Workflow execution engine with step dispatching |
|
||||
| `internal/runner` | Execution environments implementing Runner interface |
|
||||
| `internal/template` | `{{Variable}}` interpolation engine |
|
||||
| `internal/functions` | Utility functions via Otto JavaScript VM |
|
||||
| `internal/scheduler` | Cron, event, and file-watch triggers (fsnotify-based) |
|
||||
| `internal/database` | SQLite/PostgreSQL via Bun ORM |
|
||||
| `pkg/cli` | Cobra CLI commands |
|
||||
| `pkg/server` | Fiber REST API |
|
||||
| `internal/snapshot` | Workspace export/import as compressed ZIP archives |
|
||||
| `internal/installer` | Binary installation (direct-fetch and Nix modes) |
|
||||
| `internal/state` | Run state export for debugging and sharing |
|
||||
| `internal/updater` | Self-update functionality via GitHub releases |
|
||||
|
||||
### Key Types
|
||||
|
||||
```go
|
||||
WorkflowKind: "module" | "flow" // module = single unit, flow = orchestrates modules
|
||||
StepType: "bash" | "function" | "parallel-steps" | "foreach" | "remote-bash" | "http" | "llm"
|
||||
RunnerType: "host" | "docker" | "ssh"
|
||||
TriggerType: "cron" | "event" | "watch" | "manual"
|
||||
```
|
||||
|
||||
### Decision Routing
|
||||
|
||||
Steps support conditional branching via `decision` field with switch/case syntax:
|
||||
```yaml
|
||||
decision:
|
||||
switch: "{{variable}}"
|
||||
cases:
|
||||
"value1": { goto: step-a }
|
||||
"value2": { goto: step-b }
|
||||
default: { goto: fallback }
|
||||
```
|
||||
Use `goto: _end` to terminate workflow.
|
||||
|
||||
### Workflow Execution Flow
|
||||
|
||||
1. CLI parses args ▷ loads config from `~/osmedeus-base/osm-settings.yaml`
|
||||
2. Parser loads YAML workflow, validates, caches in Loader
|
||||
3. Executor initializes context with built-in variables (`{{Target}}`, `{{Output}}`, etc.)
|
||||
4. StepDispatcher routes each step to appropriate executor
|
||||
5. Runner executes commands, captures output
|
||||
6. Exports propagate to subsequent steps
|
||||
|
||||
### Template System
|
||||
|
||||
- `{{Variable}}` - standard template variables (Target, Output, threads, etc.)
|
||||
- `[[variable]]` - foreach loop variables (to avoid conflicts)
|
||||
- Functions evaluated via Otto JS runtime: `fileExists()`, `fileLength()`, `trim()`, etc.
|
||||
|
||||
## CLI Commands
|
||||
|
||||
```bash
|
||||
osmedeus run -f <flow> -t <target> # Run flow workflow
|
||||
osmedeus run -m <module> -t <target> # Run module workflow
|
||||
osmedeus run -m <m1> -m <m2> -t <target> # Run multiple modules in sequence
|
||||
osmedeus run -m <module> -t <target> --timeout 2h # With timeout
|
||||
osmedeus run -m <module> -t <target> --repeat # Repeat continuously
|
||||
osmedeus run -m <module> -T targets.txt -c 5 # Concurrent target scanning
|
||||
osmedeus run -m <module> -t <target> -P params.yaml # With params file
|
||||
osmedeus workflow list # List available workflows
|
||||
osmedeus workflow show <name> # Show workflow details
|
||||
osmedeus workflow validate <name> # Validate workflow YAML
|
||||
osmedeus func list # List utility functions
|
||||
osmedeus func e 'log_info("{{target}}")' # Evaluate function
|
||||
osmedeus --usage-example # Show all usage examples
|
||||
osmedeus server # Start REST API (see docs/api/ for endpoints)
|
||||
osmedeus server --master # Start as distributed master
|
||||
osmedeus worker join # Join as distributed worker
|
||||
osmedeus install binary --name <name> # Install specific binary
|
||||
osmedeus install binary --all # Install all binaries
|
||||
osmedeus install binary --name <name> --check # Check if binary is installed
|
||||
osmedeus install binary --all --check # Check all binaries status
|
||||
osmedeus install binary --nix-build-install # Install binaries via Nix
|
||||
osmedeus install binary --nix-installation # Install Nix package manager
|
||||
osmedeus install binary --list-registry-nix-build # List Nix binaries
|
||||
osmedeus install binary --list-registry-direct-fetch # List direct-fetch binaries
|
||||
osmedeus install env # Add binaries to PATH (auto-detects shell)
|
||||
osmedeus install env --all # Add to all shell configs
|
||||
osmedeus update # Self-update to latest version
|
||||
osmedeus update --check # Check for updates without installing
|
||||
osmedeus snapshot export <workspace> # Export workspace as ZIP
|
||||
osmedeus snapshot import <source> # Import from file or URL
|
||||
osmedeus snapshot list # List available snapshots
|
||||
osmedeus run -m <module> -t <target> -G # Run with progress bar (shorthand)
|
||||
```
|
||||
|
||||
## API Documentation
|
||||
|
||||
REST API documentation with curl examples is in `docs/api/`. Key endpoint categories:
|
||||
- **Runs**: Create, list, cancel, get steps/artifacts
|
||||
- **Workflows**: List, get details, refresh index
|
||||
- **Schedules**: Full CRUD + enable/disable/trigger
|
||||
- **Assets/Workspaces**: Query discovered data
|
||||
- **Event Logs**: Query execution events
|
||||
- **Functions**: Execute utility functions via API
|
||||
- **Snapshots**: Export/import workspace archives
|
||||
- **LLM**: OpenAI-compatible chat completions and embeddings
|
||||
- **Install**: Binary registry and installation management
|
||||
|
||||
## Adding New Features
|
||||
|
||||
**New Step Type**: Add constant in `core/types.go`, create executor implementing `StepExecutor` interface in `internal/executor/`, register in `PluginRegistry` via `dispatcher.go`
|
||||
|
||||
**New Runner**: Implement Runner interface in `internal/runner/`, add type constant, register in runner factory
|
||||
|
||||
**New CLI Command**: Create in `pkg/cli/`, add to `rootCmd` in `init()`
|
||||
|
||||
**New API Endpoint**: Add handler in `pkg/server/handlers/`, register route in `server.go`, document in `docs/api/`
|
||||
|
||||
**New Utility Function**: Add Go implementation in `internal/functions/`, register in `otto_runtime.go`
|
||||
|
||||
## Architecture Notes
|
||||
|
||||
- **Executor**: Fresh instances created per target/request - no global singleton
|
||||
- **Step Dispatcher**: Uses plugin registry pattern for extensible step type handling
|
||||
- **Scheduler**: File watching uses fsnotify for instant inotify-based notifications
|
||||
- **Decision Routing**: Uses switch/case syntax for conditional workflow branching
|
||||
+1199
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,22 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2020 j3ssie
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
.PHONY: build run test test-unit test-integration test-workflow-integration test-e2e test-e2e-verbose test-e2e-ssh test-e2e-api test-e2e-nix test-e2e-install test-docker test-ssh test-distributed test-all test-summary test-ci clean install-gotestsum lint fmt db-seed db-clean db-migrate run-server-debug swagger update-ui snapshot-release github-release docker-toolbox docker-toolbox-run docker-toolbox-shell
|
||||
|
||||
# Go parameters
|
||||
GOCMD=go
|
||||
GOBUILD=$(GOCMD) build
|
||||
GOTEST=$(GOCMD) test
|
||||
GOGET=$(GOCMD) get
|
||||
GOFMT=$(GOCMD) fmt
|
||||
GOMOD=$(GOCMD) mod
|
||||
BINARY_NAME=osmedeus
|
||||
BINARY_DIR=build/bin
|
||||
|
||||
# Console output prefix (cyan color)
|
||||
PREFIX=\033[36m[*]\033[0m
|
||||
|
||||
# Gotestsum configuration - check GOPATH/bin first, then use go test fallback
|
||||
GOPATH_BIN=$(shell go env GOPATH)/bin
|
||||
GOTESTSUM_PATH=$(shell command -v gotestsum 2>/dev/null || echo $(GOPATH_BIN)/gotestsum)
|
||||
GOTESTSUM_EXISTS=$(shell test -x $(GOTESTSUM_PATH) && echo yes || echo no)
|
||||
|
||||
# GOBIN for install target (falls back to GOPATH/bin if GOBIN is not set)
|
||||
GOBIN_PATH=$(shell go env GOBIN)
|
||||
ifeq ($(GOBIN_PATH),)
|
||||
GOBIN_PATH=$(GOPATH_BIN)
|
||||
endif
|
||||
|
||||
ifeq ($(GOTESTSUM_EXISTS),yes)
|
||||
TESTCMD=@$(GOTESTSUM_PATH)
|
||||
TESTFLAGS=--format testdox --format-hide-empty-pkg --hide-summary=skipped,output --
|
||||
else
|
||||
TESTCMD=$(GOTEST)
|
||||
TESTFLAGS=-v
|
||||
endif
|
||||
|
||||
# Build flags
|
||||
VERSION=$(shell cat internal/core/constants.go | grep 'VERSION =' | cut -d '"' -f 2)
|
||||
AUTHOR=$(shell cat internal/core/constants.go | grep 'AUTHOR =' | cut -d '"' -f 2)
|
||||
BUILD_TIME=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
COMMIT_HASH=$(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
LDFLAGS=-ldflags "-X main.BuildTime=$(BUILD_TIME) -X main.CommitHash=$(COMMIT_HASH)"
|
||||
|
||||
# Default target
|
||||
all: build
|
||||
|
||||
# Build the application and install to GOBIN
|
||||
build:
|
||||
@echo "$(PREFIX) Building $(BINARY_NAME)..."
|
||||
@mkdir -p $(BINARY_DIR)
|
||||
$(GOBUILD) $(LDFLAGS) -o $(BINARY_DIR)/$(BINARY_NAME) ./cmd/osmedeus
|
||||
@echo "$(PREFIX) Installing $(BINARY_NAME) to $(GOBIN_PATH)..."
|
||||
@cp $(BINARY_DIR)/$(BINARY_NAME) $(GOBIN_PATH)/
|
||||
|
||||
# Build for multiple platforms
|
||||
build-all: build-linux build-darwin build-windows
|
||||
|
||||
build-linux:
|
||||
@echo "$(PREFIX) Building for Linux..."
|
||||
GOOS=linux GOARCH=amd64 $(GOBUILD) $(LDFLAGS) -o $(BINARY_DIR)/$(BINARY_NAME)-linux-amd64 ./cmd/osmedeus
|
||||
|
||||
build-darwin:
|
||||
@echo "$(PREFIX) Building for macOS..."
|
||||
GOOS=darwin GOARCH=amd64 $(GOBUILD) $(LDFLAGS) -o $(BINARY_DIR)/$(BINARY_NAME)-darwin-amd64 ./cmd/osmedeus
|
||||
GOOS=darwin GOARCH=arm64 $(GOBUILD) $(LDFLAGS) -o $(BINARY_DIR)/$(BINARY_NAME)-darwin-arm64 ./cmd/osmedeus
|
||||
|
||||
build-windows:
|
||||
@echo "$(PREFIX) Building for Windows..."
|
||||
GOOS=windows GOARCH=amd64 $(GOBUILD) $(LDFLAGS) -o $(BINARY_DIR)/$(BINARY_NAME)-windows-amd64.exe ./cmd/osmedeus
|
||||
|
||||
# Run the application
|
||||
run:
|
||||
$(GOBUILD) $(LDFLAGS) -o $(BINARY_DIR)/$(BINARY_NAME) ./cmd/osmedeus
|
||||
./$(BINARY_DIR)/$(BINARY_NAME)
|
||||
|
||||
# Run with specific command
|
||||
run-server: build
|
||||
@echo "$(PREFIX) Starting server..."
|
||||
./$(BINARY_DIR)/$(BINARY_NAME) serve
|
||||
|
||||
# Run server in debug mode without authentication
|
||||
run-server-debug: build
|
||||
@echo "$(PREFIX) Starting debug server (no auth)..."
|
||||
./$(BINARY_DIR)/$(BINARY_NAME) serve -A --debug
|
||||
|
||||
# Install gotestsum (idempotent - silent if already installed)
|
||||
install-gotestsum:
|
||||
@if [ ! -x "$(GOPATH_BIN)/gotestsum" ]; then \
|
||||
echo "Installing gotestsum..."; \
|
||||
go install gotest.tools/gotestsum@latest; \
|
||||
fi
|
||||
|
||||
# Run tests (install gotestsum first)
|
||||
test: install-gotestsum
|
||||
$(TESTCMD) $(TESTFLAGS) -race ./...
|
||||
|
||||
# Run tests with coverage
|
||||
test-coverage: install-gotestsum
|
||||
$(TESTCMD) $(TESTFLAGS) -race -coverprofile=coverage.out ./...
|
||||
$(GOCMD) tool cover -html=coverage.out -o coverage.html
|
||||
|
||||
# Unit tests (fast, no external dependencies)
|
||||
test-unit: install-gotestsum
|
||||
$(TESTCMD) $(TESTFLAGS) -short ./...
|
||||
|
||||
# Integration tests (requires Docker for some tests)
|
||||
test-integration: install-gotestsum
|
||||
$(TESTCMD) $(TESTFLAGS) -run Integration ./...
|
||||
|
||||
# Workflow integration tests (test/integration/)
|
||||
test-workflow-integration: install-gotestsum
|
||||
$(TESTCMD) $(TESTFLAGS) ./test/integration/...
|
||||
|
||||
# E2E CLI tests (requires binary to be built first)
|
||||
test-e2e: build install-gotestsum
|
||||
$(TESTCMD) $(TESTFLAGS) ./test/e2e/...
|
||||
|
||||
# E2E CLI tests with verbose output (for debugging)
|
||||
test-e2e-verbose: build install-gotestsum
|
||||
@$(GOPATH_BIN)/gotestsum --format standard-verbose -- -v ./test/e2e/...
|
||||
|
||||
# Docker runner tests
|
||||
test-docker: install-gotestsum
|
||||
docker-compose -f docker-compose.test.yaml up -d
|
||||
$(TESTCMD) $(TESTFLAGS) -run Docker ./internal/runner/...
|
||||
docker-compose -f docker-compose.test.yaml down
|
||||
|
||||
# SSH runner tests (using linuxserver/openssh-server)
|
||||
test-ssh: install-gotestsum
|
||||
docker-compose -f build/docker/docker-compose.test.yaml up -d ssh-server
|
||||
sleep 5
|
||||
$(TESTCMD) $(TESTFLAGS) -run SSH ./internal/runner/...
|
||||
docker-compose -f build/docker/docker-compose.test.yaml down
|
||||
|
||||
# SSH E2E tests (full workflow tests with SSH runner)
|
||||
test-e2e-ssh: build install-gotestsum
|
||||
@echo "$(PREFIX) Starting SSH server for E2E tests..."
|
||||
docker-compose -f build/docker/docker-compose.test.yaml up -d ssh-server
|
||||
@echo "$(PREFIX) Waiting for SSH server to be ready..."
|
||||
@sleep 5
|
||||
@echo "$(PREFIX) Running SSH E2E tests..."
|
||||
$(TESTCMD) $(TESTFLAGS) -run SSH ./test/e2e/...
|
||||
@echo "$(PREFIX) Cleaning up..."
|
||||
docker-compose -f build/docker/docker-compose.test.yaml down -v
|
||||
|
||||
# Distributed scan e2e tests (requires Docker for Redis)
|
||||
test-distributed: build install-gotestsum
|
||||
@echo "$(PREFIX) Starting Redis for distributed tests..."
|
||||
docker-compose -f build/docker/docker-compose.distributed-test.yaml up -d
|
||||
@echo "$(PREFIX) Waiting for Redis to be ready..."
|
||||
@sleep 3
|
||||
@echo "$(PREFIX) Running distributed tests..."
|
||||
$(TESTCMD) $(TESTFLAGS) -run Distributed ./test/e2e/...
|
||||
@echo "$(PREFIX) Cleaning up..."
|
||||
docker-compose -f build/docker/docker-compose.distributed-test.yaml down -v
|
||||
|
||||
# API E2E tests (requires Docker for Redis, builds binary first)
|
||||
test-e2e-api: build install-gotestsum
|
||||
@echo "$(PREFIX) Starting Redis for API tests..."
|
||||
docker-compose -f build/docker/docker-compose.distributed-test.yaml up -d
|
||||
@echo "$(PREFIX) Waiting for Redis to be ready..."
|
||||
@sleep 3
|
||||
@echo "$(PREFIX) Running API E2E tests..."
|
||||
$(TESTCMD) $(TESTFLAGS) -run API ./test/e2e/...
|
||||
@echo "$(PREFIX) Cleaning up..."
|
||||
docker-compose -f build/docker/docker-compose.distributed-test.yaml down -v
|
||||
|
||||
# Nix E2E tests (requires Docker for Nix container)
|
||||
test-e2e-nix: build install-gotestsum
|
||||
@echo "$(PREFIX) Building Nix test container..."
|
||||
docker-compose -f build/docker/docker-compose.nix-test.yaml build
|
||||
@echo "$(PREFIX) Starting Nix test container..."
|
||||
docker-compose -f build/docker/docker-compose.nix-test.yaml up -d
|
||||
@echo "$(PREFIX) Waiting for Nix container to be ready..."
|
||||
@sleep 3
|
||||
@echo "$(PREFIX) Running Nix E2E tests..."
|
||||
$(TESTCMD) $(TESTFLAGS) -run TestNix ./test/e2e/...
|
||||
@echo "$(PREFIX) Cleaning up..."
|
||||
docker-compose -f build/docker/docker-compose.nix-test.yaml down -v
|
||||
|
||||
# Install E2E tests (workflow and base installation from zip/URL/git)
|
||||
test-e2e-install: build install-gotestsum
|
||||
@echo "$(PREFIX) Running install E2E tests..."
|
||||
$(TESTCMD) $(TESTFLAGS) -run TestInstall ./test/e2e/...
|
||||
|
||||
# All tests
|
||||
test-all: test-unit test-integration
|
||||
|
||||
# Quick test summary (pass/fail only)
|
||||
test-summary: install-gotestsum
|
||||
@$(GOPATH_BIN)/gotestsum --format dots-v2 -- -v ./...
|
||||
|
||||
# Test with JUnit XML output (for CI)
|
||||
test-ci: install-gotestsum
|
||||
@$(GOPATH_BIN)/gotestsum --junitfile test-results.xml --format testdox --format-hide-empty-pkg --hide-summary=skipped,output -- -v -race ./...
|
||||
|
||||
# Clean build artifacts
|
||||
clean:
|
||||
@echo "$(PREFIX) Cleaning..."
|
||||
rm -rf $(BINARY_DIR)
|
||||
rm -f coverage.out coverage.html test-results.xml
|
||||
|
||||
# Format code
|
||||
fmt:
|
||||
$(GOFMT) ./...
|
||||
|
||||
# Lint code
|
||||
lint:
|
||||
golangci-lint run
|
||||
|
||||
# Tidy dependencies
|
||||
tidy:
|
||||
$(GOMOD) tidy
|
||||
|
||||
# Download dependencies
|
||||
deps:
|
||||
$(GOMOD) download
|
||||
|
||||
# Update dependencies
|
||||
update-deps:
|
||||
$(GOGET) -u ./...
|
||||
$(GOMOD) tidy
|
||||
|
||||
# Generate code (if needed)
|
||||
generate:
|
||||
$(GOCMD) generate ./...
|
||||
|
||||
# Generate swagger documentation
|
||||
swagger:
|
||||
@echo "$(PREFIX) Generating swagger documentation..."
|
||||
swag init -g pkg/server/server.go -o docs/api-swagger/ --packageName apiswagger
|
||||
|
||||
# Update embedded UI from dashboard build
|
||||
update-ui:
|
||||
@echo "$(PREFIX) Updating embedded UI..."
|
||||
rm -rf public/ui/*
|
||||
cp -R ../osmedeus-dashboard/build/* public/ui/
|
||||
@echo "$(PREFIX) UI updated successfully!"
|
||||
|
||||
# Development setup
|
||||
dev-setup: install-gotestsum
|
||||
@echo "$(PREFIX) Setting up development environment..."
|
||||
$(GOMOD) download
|
||||
@echo "$(PREFIX) Done!"
|
||||
|
||||
# Docker build
|
||||
docker-build:
|
||||
docker build -t osmedeus:$(VERSION) .
|
||||
|
||||
# Docker run
|
||||
docker-run:
|
||||
docker run -p 8002:8002 osmedeus:$(VERSION)
|
||||
|
||||
# Docker toolbox build (with all tools pre-installed)
|
||||
docker-toolbox:
|
||||
@echo "$(PREFIX) Building osmedeus-toolbox Docker image..."
|
||||
docker-compose -f build/docker/docker-compose.toolbox.yaml build \
|
||||
--build-arg BUILD_TIME=$(BUILD_TIME) \
|
||||
--build-arg COMMIT_HASH=$(COMMIT_HASH)
|
||||
@echo "$(PREFIX) osmedeus-toolbox image built successfully!"
|
||||
@echo "$(PREFIX) Run with: docker-compose -f build/docker/docker-compose.toolbox.yaml up -d"
|
||||
|
||||
# Docker toolbox run
|
||||
docker-toolbox-run:
|
||||
@echo "$(PREFIX) Starting osmedeus-toolbox container..."
|
||||
docker-compose -f build/docker/docker-compose.toolbox.yaml up -d
|
||||
@echo "$(PREFIX) Container started! Enter with: docker exec -it osmedeus-toolbox bash"
|
||||
|
||||
# Docker toolbox shell (interactive)
|
||||
docker-toolbox-shell:
|
||||
docker exec -it osmedeus-toolbox bash
|
||||
|
||||
# Release commands (GoReleaser)
|
||||
snapshot-release:
|
||||
@echo "$(PREFIX) Building $(BINARY_NAME)..."
|
||||
@mkdir -p $(BINARY_DIR)
|
||||
$(GOBUILD) $(LDFLAGS) -o $(BINARY_DIR)/$(BINARY_NAME) ./cmd/osmedeus
|
||||
@echo "$(PREFIX) Installing $(BINARY_NAME) to $(GOBIN_PATH)..."
|
||||
@cp $(BINARY_DIR)/$(BINARY_NAME) $(GOBIN_PATH)/
|
||||
@echo "$(PREFIX) Update registry-metadata-direct-fetch.json..."
|
||||
cp ../osmedeus-registry/registry-metadata-direct-fetch.json public/presets/registry-metadata-direct-fetch.json
|
||||
@echo "$(PREFIX) Building snapshot release..."
|
||||
export GORELEASER_CURRENT_TAG="$(VERSION)" && goreleaser release --snapshot --clean
|
||||
@echo "$(PREFIX) Install script copied to dist/install.sh"
|
||||
cp ../osmedeus-registry/install.sh dist/install.sh
|
||||
@echo "$(PREFIX) Prepare registry-metadata-direct-fetch.json"
|
||||
cp ../osmedeus-registry/registry-metadata-direct-fetch.json dist/registry-metadata-direct-fetch.json
|
||||
|
||||
|
||||
github-release:
|
||||
@echo "$(PREFIX) Building and publishing GitHub release..."
|
||||
export GORELEASER_CURRENT_TAG="$(VERSION)" && goreleaser release --clean
|
||||
|
||||
# Database commands
|
||||
db-seed: build
|
||||
@echo "$(PREFIX) Seeding database..."
|
||||
./$(BINARY_DIR)/$(BINARY_NAME) db seed
|
||||
|
||||
db-clean: build
|
||||
@echo "$(PREFIX) Cleaning database..."
|
||||
./$(BINARY_DIR)/$(BINARY_NAME) db clean --force
|
||||
|
||||
db-migrate: build
|
||||
@echo "$(PREFIX) Running database migrations..."
|
||||
./$(BINARY_DIR)/$(BINARY_NAME) db migrate
|
||||
|
||||
# Help
|
||||
help:
|
||||
@echo ""
|
||||
@echo "\033[32m Osmedeus $(VERSION) - A Modern Orchestration Engine for Security\033[0m"
|
||||
@echo "\033[36m Crafted with \033[31m<3\033[35m by $(AUTHOR) \033[0m"
|
||||
@echo "\033[34m ──────────────────────────────────────────────────\033[0m"
|
||||
@echo ""
|
||||
@echo "\033[33m BUILD\033[0m"
|
||||
@echo " make build Build and install binary to \$$GOBIN (or \$$GOPATH/bin)"
|
||||
@echo " make build-all Build for all platforms"
|
||||
@echo " make clean Clean build artifacts"
|
||||
@echo ""
|
||||
@echo "\033[33m RUN\033[0m"
|
||||
@echo " make run Build and run the application"
|
||||
@echo " make run-server Build and start the server"
|
||||
@echo " make run-server-debug Build and start server in debug mode (no auth)"
|
||||
@echo ""
|
||||
@echo "\033[33m TEST\033[0m"
|
||||
@echo " make test Run all tests"
|
||||
@echo " make test-unit Run unit tests (fast)"
|
||||
@echo " make test-integration Run integration tests"
|
||||
@echo " make test-e2e Run E2E CLI tests"
|
||||
@echo " make test-e2e-verbose Run E2E tests with verbose output"
|
||||
@echo " make test-coverage Run tests with coverage report"
|
||||
@echo " make test-summary Quick pass/fail summary"
|
||||
@echo " make test-ci Run tests with JUnit XML output"
|
||||
@echo ""
|
||||
@echo "\033[33m DEVELOPMENT\033[0m"
|
||||
@echo " make dev-setup Set up development environment"
|
||||
@echo " make fmt Format code"
|
||||
@echo " make lint Run linter"
|
||||
@echo " make tidy Tidy go.mod dependencies"
|
||||
@echo " make deps Download dependencies"
|
||||
@echo " make swagger Generate swagger documentation"
|
||||
@echo " make update-ui Update embedded UI from dashboard build"
|
||||
@echo ""
|
||||
@echo "\033[33m DOCKER\033[0m"
|
||||
@echo " make docker-build Build Docker image"
|
||||
@echo " make docker-run Run Docker container"
|
||||
@echo " make docker-toolbox Build toolbox image (all tools pre-installed)"
|
||||
@echo " make docker-toolbox-run Start toolbox container"
|
||||
@echo " make docker-toolbox-shell Enter toolbox container shell"
|
||||
@echo " make test-docker Run Docker runner tests"
|
||||
@echo " make test-ssh Run SSH runner unit tests"
|
||||
@echo " make test-e2e-ssh Run SSH E2E tests (full workflows)"
|
||||
@echo " make test-e2e-api Run API E2E tests (all endpoints)"
|
||||
@echo " make test-e2e-nix Run Nix mode E2E tests (requires Docker)"
|
||||
@echo " make test-e2e-install Run install E2E tests (workflow/base from zip/URL/git)"
|
||||
@echo " make test-distributed Run distributed scan e2e tests"
|
||||
@echo ""
|
||||
@echo "\033[33m RELEASE\033[0m"
|
||||
@echo " make snapshot-release Build local snapshot release (no publish)"
|
||||
@echo " make github-release Build and publish GitHub release"
|
||||
@echo ""
|
||||
@echo "\033[33m DATABASE\033[0m"
|
||||
@echo " make db-seed Seed database with sample data"
|
||||
@echo " make db-clean Clean all data from database"
|
||||
@echo " make db-migrate Run database migrations"
|
||||
@echo ""
|
||||
@@ -0,0 +1,199 @@
|
||||
# Osmedeus
|
||||
|
||||
<p align="center">
|
||||
<a href="https://www.osmedeus.org"><img alt="Osmedeus" src="https://raw.githubusercontent.com/osmedeus/assets/main/osm-logo-with-white-border.png" height="140" /></a>
|
||||
<br />
|
||||
<strong>Osmedeus - A Modern Orchestration Engine for Security</strong>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://docs.osmedeus.org/"><img src="https://img.shields.io/badge/Documentation-0078D4?style=for-the-badge&logo=GitBook&logoColor=39ff14&labelColor=black&color=black"></a>
|
||||
<a href="https://docs.osmedeus.org/donation/"><img src="https://img.shields.io/badge/Sponsors-0078D4?style=for-the-badge&logo=GitHub-Sponsors&logoColor=39ff14&labelColor=black&color=black"></a>
|
||||
<a href="https://twitter.com/OsmedeusEngine"><img src="https://img.shields.io/badge/%40OsmedeusEngine-0078D4?style=for-the-badge&logo=Twitter&logoColor=39ff14&labelColor=black&color=black"></a>
|
||||
<a href="https://discord.gg/gy4SWhpaPU"><img src="https://img.shields.io/badge/Discord%20Server-0078D4?style=for-the-badge&logo=Discord&logoColor=39ff14&labelColor=black&color=black"></a>
|
||||
<a href="https://github.com/j3ssie/osmedeus/releases"><img src="https://img.shields.io/github/release/j3ssie/osmedeus?style=for-the-badge&labelColor=black&color=2fc414&logo=Github"></a>
|
||||
</p>
|
||||
</p>
|
||||
|
||||
## What is Osmedeus?
|
||||
|
||||
[Osmedeus](https://www.osmedeus.org) is a security focused declarative orchestration engine that simplifies complex workflow automation into auditable YAML definitions, complete with encrypted data handling, secure credential management, and sandboxed execution.
|
||||
|
||||
Built for both beginners and experts, it delivers powerful, composable automation without sacrificing the integrity and safety of your infrastructure.
|
||||
|
||||
## Features
|
||||
|
||||
- **Declarative YAML Workflows** - Define reconnaissance pipelines using simple, readable YAML syntax
|
||||
- **Two Workflow Types** - Modules for single execution units, Flows for multi-module orchestration
|
||||
- **Multiple Runners** - Execute on local host, Docker containers, or remote machines via SSH
|
||||
- **Distributed Execution** - Scale with Redis-based master-worker pattern for parallel scanning
|
||||
- **Event-Driven Triggers** - Cron scheduling, file watching, and event-based workflow triggers
|
||||
- **Decision Routing** - Conditional workflow branching with switch/case syntax
|
||||
- **Template Engine** - Powerful variable interpolation with built-in and custom variables
|
||||
- **Utility Functions** - Rich function library for file operations, string manipulation, and JSON processing
|
||||
- **REST API Server** - Manage and trigger workflows programmatically
|
||||
- **Database Support** - SQLite (default) and PostgreSQL for asset tracking
|
||||
- **Notifications** - Telegram bot and webhook integrations
|
||||
- **Cloud Storage** - S3-compatible storage for artifact management
|
||||
- **LLM Integration** - AI-powered workflow steps with chat completions and embeddings
|
||||
|
||||
See [Documentation Page](https://docs.osmedeus.org/) for more details.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
curl -sSL http://www.osmedeus.org/install.sh | bash
|
||||
```
|
||||
|
||||
See [Quickstart](https://docs.osmedeus.org/quickstart/) for quick setup and [Installation](https://docs.osmedeus.org/installation/) for advanced configurations.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Run a module workflow
|
||||
osmedeus run -m recon -t example.com
|
||||
|
||||
# Run a flow workflow
|
||||
osmedeus run -f general -t example.com
|
||||
|
||||
# Multiple targets with concurrency
|
||||
osmedeus run -m recon -T targets.txt -c 5
|
||||
|
||||
# Dry-run mode (preview)
|
||||
osmedeus run -f general -t example.com --dry-run
|
||||
|
||||
# Start API server
|
||||
osmedeus serve
|
||||
|
||||
# List available workflows
|
||||
osmedeus workflow list
|
||||
|
||||
# Show all usage examples
|
||||
osmedeus --usage-example
|
||||
```
|
||||
|
||||
## Docker
|
||||
|
||||
```bash
|
||||
# Show help
|
||||
docker run --rm osmedeus:latest --help
|
||||
|
||||
# Run a scan
|
||||
docker run --rm -v $(pwd)/output:/root/workspaces-osmedeus \
|
||||
osmedeus:latest run -f general -t example.com
|
||||
```
|
||||
|
||||
For more CLI usage and example commands, refer to the [CLI Reference](https://docs.osmedeus.org/getting-started/cli).
|
||||
|
||||
|
||||
| CLI Usage | Web UI Assets | Web UI Workflow |
|
||||
|-----------|--------------|-----------------|
|
||||
|  |  |  |
|
||||
|
||||
## Core Components
|
||||
|
||||
### Trigger
|
||||
|
||||
| Type | Description | Use Case |
|
||||
|------|-------------|----------|
|
||||
| **Cron** | Schedule workflows at specific times | Regular scans |
|
||||
| **File Watch** | Trigger workflows when files change | Continuous monitoring |
|
||||
| **Event** | Trigger workflows based on external events | Integration with other tools |
|
||||
| **Webhook** | Trigger workflows based on HTTP requests | External system integration |
|
||||
| **Manual** | Trigger workflows manually via CLI or API | One-time tasks |
|
||||
|
||||
### Workflows
|
||||
|
||||
| Type | Description | Use Case |
|
||||
|------|-------------|----------|
|
||||
| **Module** | Single execution unit with sequential/parallel steps | Individual scanning tasks |
|
||||
| **Flow** | Orchestrates multiple modules with dependencies | Complete reconnaissance pipelines |
|
||||
|
||||
### Runners
|
||||
|
||||
| Runner | Description |
|
||||
|--------|-------------|
|
||||
| **Host** | Local machine execution (default) |
|
||||
| **Docker** | Container-based execution |
|
||||
| **SSH** | Remote machine execution |
|
||||
|
||||
### Step Types
|
||||
|
||||
| Type | Description |
|
||||
|------|-------------|
|
||||
| `bash` | Execute shell commands |
|
||||
| `function` | Call utility functions |
|
||||
| `foreach` | Iterate over file contents |
|
||||
| `parallel-steps` | Run multiple steps concurrently |
|
||||
| `remote-bash` | Per-step Docker/SSH execution |
|
||||
| `http` | Make HTTP requests |
|
||||
| `llm` | AI-powered processing |
|
||||
|
||||
### Workflow Example
|
||||
|
||||
```yaml
|
||||
kind: module
|
||||
name: demo-bash
|
||||
description: Demo bash steps with functions and exports
|
||||
|
||||
params:
|
||||
- name: target
|
||||
required: true
|
||||
|
||||
steps:
|
||||
- name: setup
|
||||
type: bash
|
||||
command: mkdir -p {{Output}}/demo && echo "{{Target}}" > {{Output}}/demo/target.txt
|
||||
exports:
|
||||
target_file: "{{Output}}/demo/target.txt"
|
||||
|
||||
- name: run-parallel
|
||||
type: bash
|
||||
parallel_commands:
|
||||
- 'echo "Thread 1: {{Target}}" >> {{Output}}/demo/results.txt'
|
||||
- 'echo "Thread 2: {{Target}}" >> {{Output}}/demo/results.txt'
|
||||
|
||||
- name: check-result
|
||||
type: function
|
||||
function: 'fileLength("{{Output}}/demo/results.txt")'
|
||||
exports:
|
||||
line_count: "output"
|
||||
|
||||
- name: summary
|
||||
type: bash
|
||||
command: 'echo "Processed {{Target}} with {{line_count}} lines"'
|
||||
|
||||
```
|
||||
|
||||
For writing your first workflow, refer to the [Workflow Overview](https://docs.osmedeus.org/workflows/overview).
|
||||
|
||||
## Roadmap and Status
|
||||
|
||||
The high-level ambitious plan for the project, in order:
|
||||
|
||||
| # | Step | Status |
|
||||
| :-: | ----------------------------------------------------------------------------- | :----: |
|
||||
| 1 | Osmedeus Engine reforged with a next-generation architecture | ✅ |
|
||||
| 2 | Flexible workflows and step types | ✅ |
|
||||
| 3 | Beautiful UI for visualize results and workflow diagram | ✅ |
|
||||
| 4 | Rewriting the workflow to adapt to new architecture and syntax | ⚠️ |
|
||||
| 5 | Testing more utility functions like notifications | ⚠️ |
|
||||
| 6 | Generate diff reports showing new/removed/unchanged assets between runs. | ❌ |
|
||||
| 7 | Adding step type from cloud provider that can be run via serverless | ❌ |
|
||||
| N | Fancy features (to be expanded upon later) | ❌ |
|
||||
|
||||
## Documentation
|
||||
|
||||
| Topic | Link |
|
||||
|----------------------|----------------------------------------------------------------------------------------------------------|
|
||||
| Getting Started | [docs.osmedeus.org/getting-started](https://docs.osmedeus.org/getting-started) |
|
||||
| CLI Usage & Examples | [docs.osmedeus.org/getting-started/cli](https://docs.osmedeus.org/getting-started/cli) |
|
||||
| Writing Workflows | [docs.osmedeus.org/workflows/overview](https://docs.osmedeus.org/workflows/overview) |
|
||||
| Deployment | [docs.osmedeus.org/deployment](https://docs.osmedeus.org/deployment) |
|
||||
| Architecture | [docs.osmedeus.org/concepts/architecture](https://docs.osmedeus.org/concepts/architecture) |
|
||||
| Development | [docs.osmedeus.org/development](https://docs.osmedeus.org/development) and [HACKING.md](HACKING.md) |
|
||||
| Extending Osmedeus | [docs.osmedeus.org/development/extending-osmedeus](https://docs.osmedeus.org/development/extending-osmedeus) |
|
||||
| Full Documentation | [docs.osmedeus.org](https://docs.osmedeus.org) |
|
||||
|
||||
## License
|
||||
|
||||
Osmedeus is made with ♥ by [@j3ssie](https://twitter.com/j3ssie) and it is released under the MIT license.
|
||||
@@ -0,0 +1,390 @@
|
||||
# Deployment Guide
|
||||
|
||||
This guide covers building, deploying, and running Osmedeus in various environments.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Go 1.21+ (for local builds)
|
||||
- Docker 20.10+ (for containerized deployment)
|
||||
- Docker Compose 2.0+ (for distributed mode)
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Local build and run
|
||||
make build
|
||||
./build/bin/osmedeus serve
|
||||
|
||||
# Docker single container
|
||||
docker build -t osmedeus:latest -f build/docker/Dockerfile .
|
||||
docker run -p 8001:8001 osmedeus:latest
|
||||
|
||||
# Distributed mode with Docker Compose
|
||||
docker-compose -f build/docker/docker-compose.yml up -d
|
||||
```
|
||||
|
||||
## Building
|
||||
|
||||
### Local Build
|
||||
|
||||
```bash
|
||||
# Build for current platform
|
||||
make build
|
||||
|
||||
# Cross-platform builds
|
||||
make build-all # All platforms
|
||||
make build-linux # Linux amd64
|
||||
make build-darwin # macOS amd64 + arm64
|
||||
make build-windows # Windows amd64
|
||||
|
||||
# Output location
|
||||
./build/bin/osmedeus
|
||||
```
|
||||
|
||||
### Docker Build
|
||||
|
||||
```bash
|
||||
# Production image (minimal, ~50MB)
|
||||
docker build -t osmedeus:5.0.0 -f build/docker/Dockerfile .
|
||||
|
||||
# Development image (with hot-reload)
|
||||
docker build -t osmedeus:dev -f build/docker/Dockerfile.dev .
|
||||
|
||||
# With custom version
|
||||
docker build --build-arg VERSION=5.1.0 -t osmedeus:5.1.0 -f build/docker/Dockerfile .
|
||||
```
|
||||
|
||||
## Deployment Modes
|
||||
|
||||
### Single Host
|
||||
|
||||
#### Direct Binary
|
||||
|
||||
```bash
|
||||
# Run server
|
||||
./build/bin/osmedeus serve --port 8001
|
||||
|
||||
# Run with authentication disabled (development only)
|
||||
./build/bin/osmedeus serve -A
|
||||
|
||||
# Run a scan
|
||||
./build/bin/osmedeus scan -f general -t example.com
|
||||
```
|
||||
|
||||
#### Docker Container
|
||||
|
||||
```bash
|
||||
# Basic server
|
||||
docker run -d \
|
||||
--name osmedeus \
|
||||
-p 8001:8001 \
|
||||
-v osmedeus-data:/root/osmedeus-base \
|
||||
-v workspaces:/root/workspaces-osmedeus \
|
||||
osmedeus:latest
|
||||
|
||||
# With custom workflows
|
||||
docker run -d \
|
||||
--name osmedeus \
|
||||
-p 8001:8001 \
|
||||
-v /path/to/workflows:/root/osmedeus-base/workflows \
|
||||
-v /path/to/workspaces:/root/workspaces-osmedeus \
|
||||
osmedeus:latest
|
||||
```
|
||||
|
||||
### Distributed Mode (Master/Worker)
|
||||
|
||||
Distributed mode allows scaling scan workloads across multiple worker nodes using Redis as a message queue.
|
||||
|
||||
#### Architecture
|
||||
|
||||
```
|
||||
┌─────────────┐
|
||||
│ Client │
|
||||
└──────┬──────┘
|
||||
│ REST API
|
||||
┌──────▼──────┐
|
||||
│ Master │
|
||||
│ (Server) │
|
||||
└──────┬──────┘
|
||||
│
|
||||
┌──────▼──────┐
|
||||
│ Redis │
|
||||
│ (Queue) │
|
||||
└──────┬──────┘
|
||||
┌────────────┼────────────┐
|
||||
│ │ │
|
||||
┌─────▼────┐ ┌─────▼────┐ ┌─────▼────┐
|
||||
│ Worker 1 │ │ Worker 2 │ │ Worker N │
|
||||
└──────────┘ └──────────┘ └──────────┘
|
||||
```
|
||||
|
||||
#### Docker Compose Setup
|
||||
|
||||
```bash
|
||||
# Start with 2 workers (default)
|
||||
docker-compose -f build/docker/docker-compose.yml up -d
|
||||
|
||||
# Scale to 5 workers
|
||||
docker-compose -f build/docker/docker-compose.yml up -d --scale worker=5
|
||||
|
||||
# View logs
|
||||
docker-compose -f build/docker/docker-compose.yml logs -f
|
||||
|
||||
# Stop all services
|
||||
docker-compose -f build/docker/docker-compose.yml down
|
||||
|
||||
# Stop and remove volumes
|
||||
docker-compose -f build/docker/docker-compose.yml down -v
|
||||
```
|
||||
|
||||
#### Manual Distributed Setup
|
||||
|
||||
If not using Docker Compose:
|
||||
|
||||
```bash
|
||||
# 1. Start Redis
|
||||
docker run -d --name redis -p 6379:6379 redis:7-alpine
|
||||
|
||||
# 2. Start Master
|
||||
./build/bin/osmedeus serve --master --port 8001
|
||||
|
||||
# 3. Start Workers (on same or different machines)
|
||||
./build/bin/osmedeus worker join --redis-url redis://localhost:6379
|
||||
```
|
||||
|
||||
#### Submitting Distributed Scans
|
||||
|
||||
```bash
|
||||
# Submit scan to distributed queue
|
||||
./build/bin/osmedeus scan -f general -t example.com -D
|
||||
|
||||
# With custom Redis URL
|
||||
./build/bin/osmedeus scan -f general -t example.com -D --redis-url redis://redis-host:6379
|
||||
|
||||
# Check worker status
|
||||
./build/bin/osmedeus worker status
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Configuration File
|
||||
|
||||
Default location: `~/osmedeus-base/osm-settings.yaml`
|
||||
|
||||
```yaml
|
||||
base_folder: ~/osmedeus-base
|
||||
|
||||
environments:
|
||||
binaries_path: "{{base_folder}}/binaries"
|
||||
data: "{{base_folder}}/data"
|
||||
workspaces: ~/workspaces-osmedeus
|
||||
workflows: "{{base_folder}}/workflows"
|
||||
|
||||
server:
|
||||
host: 0.0.0.0
|
||||
port: 8001
|
||||
|
||||
# Required for distributed mode
|
||||
redis:
|
||||
host: localhost
|
||||
port: 6379
|
||||
password: "" # Optional
|
||||
db: 0
|
||||
|
||||
database:
|
||||
db_engine: sqlite # or postgresql
|
||||
db_path: "{{base_folder}}/osm-data.db"
|
||||
|
||||
client:
|
||||
username: admin
|
||||
password: admin
|
||||
jwt:
|
||||
secret: "change-this-in-production"
|
||||
expiration_minutes: 60
|
||||
|
||||
scan_tactic:
|
||||
aggressive: 40
|
||||
default: 10
|
||||
gently: 5
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `REDIS_HOST` | Redis hostname | localhost |
|
||||
| `REDIS_PORT` | Redis port | 6379 |
|
||||
| `OSM_BASE_FOLDER` | Base folder path | ~/osmedeus-base |
|
||||
|
||||
### Command Line Overrides
|
||||
|
||||
```bash
|
||||
# Override base folder
|
||||
osmedeus -b /custom/path scan -f general -t example.com
|
||||
|
||||
# Override workflow folder
|
||||
osmedeus -F /custom/workflows workflow list
|
||||
|
||||
# Override Redis URL (distributed mode)
|
||||
osmedeus scan -f general -t example.com -D --redis-url redis://user:pass@host:6379/0
|
||||
```
|
||||
|
||||
## Docker Compose Reference
|
||||
|
||||
The included `build/docker/docker-compose.yml` provides a complete distributed setup:
|
||||
|
||||
### Services
|
||||
|
||||
| Service | Purpose | Ports |
|
||||
|---------|---------|-------|
|
||||
| `redis` | Task queue and coordination | 6379 |
|
||||
| `master` | API server and task distributor | 8001 |
|
||||
| `worker` | Task executor (scalable) | - |
|
||||
|
||||
### Volumes
|
||||
|
||||
| Volume | Purpose |
|
||||
|--------|---------|
|
||||
| `redis-data` | Redis persistence |
|
||||
| `osmedeus-data` | Workflows and configuration |
|
||||
| `workspaces` | Scan output data |
|
||||
|
||||
### Scaling
|
||||
|
||||
```bash
|
||||
# Scale workers dynamically
|
||||
docker-compose -f build/docker/docker-compose.yml up -d --scale worker=10
|
||||
|
||||
# View running containers
|
||||
docker-compose -f build/docker/docker-compose.yml ps
|
||||
```
|
||||
|
||||
## Production Considerations
|
||||
|
||||
### Security
|
||||
|
||||
1. **Authentication**: Never use `-A` (no-auth) in production
|
||||
2. **JWT Secret**: Change the default JWT secret in config
|
||||
3. **TLS**: Use a reverse proxy (nginx, traefik) for HTTPS
|
||||
4. **Network**: Restrict Redis access to internal network only
|
||||
|
||||
```yaml
|
||||
# Example: Secure JWT configuration
|
||||
client:
|
||||
jwt:
|
||||
secret: "your-256-bit-secret-key-here"
|
||||
expiration_minutes: 30
|
||||
```
|
||||
|
||||
### Resource Limits
|
||||
|
||||
Worker resource limits in docker-compose.yml:
|
||||
|
||||
```yaml
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 1G
|
||||
reservations:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
```
|
||||
|
||||
Adjust based on workflow requirements.
|
||||
|
||||
### Health Checks
|
||||
|
||||
The Docker image includes built-in health checks:
|
||||
|
||||
```bash
|
||||
# Check master health
|
||||
curl http://localhost:8001/health
|
||||
|
||||
# Check readiness
|
||||
curl http://localhost:8001/health/ready
|
||||
```
|
||||
|
||||
### Logging
|
||||
|
||||
```bash
|
||||
# View master logs
|
||||
docker logs osmedeus-master -f
|
||||
|
||||
# View all worker logs
|
||||
docker-compose -f build/docker/docker-compose.yml logs -f worker
|
||||
|
||||
# Log levels are controlled by --verbose/-v flag
|
||||
./build/bin/osmedeus -v serve
|
||||
```
|
||||
|
||||
### Database Options
|
||||
|
||||
For production, consider PostgreSQL instead of SQLite:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
db_engine: postgresql
|
||||
db_host: postgres-host
|
||||
db_port: 5432
|
||||
db_name: osmedeus
|
||||
db_user: osmedeus
|
||||
db_password: secure-password
|
||||
```
|
||||
|
||||
### Backup
|
||||
|
||||
```bash
|
||||
# Backup volumes
|
||||
docker run --rm \
|
||||
-v osmedeus-data:/data \
|
||||
-v $(pwd):/backup \
|
||||
alpine tar czf /backup/osmedeus-backup.tar.gz /data
|
||||
|
||||
# Backup workspaces
|
||||
docker run --rm \
|
||||
-v workspaces:/data \
|
||||
-v $(pwd):/backup \
|
||||
alpine tar czf /backup/workspaces-backup.tar.gz /data
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**Workers not connecting:**
|
||||
```bash
|
||||
# Check Redis connectivity
|
||||
docker exec osmedeus-redis redis-cli ping
|
||||
|
||||
# Check worker logs
|
||||
docker-compose logs worker
|
||||
```
|
||||
|
||||
**Scans not executing:**
|
||||
```bash
|
||||
# Verify workflow exists
|
||||
./build/bin/osmedeus workflow list
|
||||
|
||||
# Check master logs
|
||||
docker logs osmedeus-master
|
||||
```
|
||||
|
||||
**Port conflicts:**
|
||||
```bash
|
||||
# Use different ports
|
||||
docker run -p 8080:8001 osmedeus:latest
|
||||
```
|
||||
|
||||
### Useful Commands
|
||||
|
||||
```bash
|
||||
# Environment health check
|
||||
./build/bin/osmedeus health
|
||||
|
||||
# Validate workflows
|
||||
./build/bin/osmedeus workflow validate <workflow-name>
|
||||
|
||||
# Test workflow (dry-run)
|
||||
./build/bin/osmedeus scan -f general -t example.com --dry-run
|
||||
```
|
||||
@@ -0,0 +1,50 @@
|
||||
# Osmedeus Production Environment Variables
|
||||
# ==========================================
|
||||
# Copy this file to .env and update with your secure values:
|
||||
# cp .env.example .env
|
||||
#
|
||||
# IMPORTANT: Never commit .env to version control!
|
||||
|
||||
# =============================================================================
|
||||
# PostgreSQL Configuration
|
||||
# =============================================================================
|
||||
# Database credentials - MUST be changed for production
|
||||
POSTGRES_USER=osmedeus
|
||||
POSTGRES_PASSWORD=your_secure_postgres_password_here
|
||||
POSTGRES_DB=osmedeus
|
||||
|
||||
# =============================================================================
|
||||
# Redis Configuration (Optional)
|
||||
# =============================================================================
|
||||
# Uncomment and set if you want Redis authentication
|
||||
# REDIS_PASSWORD=your_secure_redis_password_here
|
||||
|
||||
# =============================================================================
|
||||
# Server Configuration
|
||||
# =============================================================================
|
||||
# External port for the API server
|
||||
OSM_SERVER_PORT=8002
|
||||
|
||||
# Timezone for logs and timestamps
|
||||
TZ=UTC
|
||||
|
||||
# =============================================================================
|
||||
# Worker Configuration
|
||||
# =============================================================================
|
||||
# Number of worker replicas for distributed scanning
|
||||
WORKER_REPLICAS=2
|
||||
|
||||
# =============================================================================
|
||||
# IMPORTANT NOTES
|
||||
# =============================================================================
|
||||
# 1. After setting POSTGRES_PASSWORD here, update the same value in:
|
||||
# osm-settings.production.yaml -> database.password
|
||||
#
|
||||
# 2. Generate secure passwords with:
|
||||
# openssl rand -base64 24
|
||||
#
|
||||
# 3. Generate JWT secret (for osm-settings.production.yaml) with:
|
||||
# openssl rand -base64 32
|
||||
#
|
||||
# 4. If using REDIS_PASSWORD, update osm-settings.production.yaml:
|
||||
# redis.password: "your_redis_password"
|
||||
@@ -0,0 +1,61 @@
|
||||
# Osmedeus Production Dockerfile
|
||||
# Ubuntu/Debian-based image with essential tools for security scanning
|
||||
|
||||
# Stage 1: Build osmedeus binary
|
||||
FROM golang:1.22-bookworm AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy go mod files first for better layer caching
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the binary
|
||||
ARG VERSION=5.0.0
|
||||
ARG BUILD_TIME
|
||||
ARG COMMIT_HASH
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build \
|
||||
-ldflags "-s -w -X main.BuildTime=${BUILD_TIME} -X main.CommitHash=${COMMIT_HASH}" \
|
||||
-o /app/bin/osmedeus ./cmd/osmedeus
|
||||
|
||||
# Stage 2: Runtime with essential tools
|
||||
FROM golang:1.22-bookworm
|
||||
|
||||
# Install essential tools
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
git \
|
||||
curl \
|
||||
wget \
|
||||
ca-certificates \
|
||||
python3 \
|
||||
python3-pip \
|
||||
chromium \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ln -sf /usr/bin/python3 /usr/bin/python
|
||||
|
||||
# Copy osmedeus binary from builder
|
||||
COPY --from=builder /app/bin/osmedeus /usr/local/bin/osmedeus
|
||||
|
||||
# Create base directories
|
||||
RUN mkdir -p /root/osmedeus-base /root/workspaces-osmedeus
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
# Initialize osmedeus base folder with preset workflows
|
||||
RUN osmedeus install base --preset
|
||||
|
||||
# Set up PATH for external binaries
|
||||
ENV PATH="/root/osmedeus-base/external-binaries:${PATH}"
|
||||
|
||||
# Expose default server port
|
||||
EXPOSE 8002
|
||||
|
||||
# Default entrypoint - exposes osmedeus CLI only
|
||||
ENTRYPOINT ["osmedeus"]
|
||||
|
||||
# Default command shows help (user can override with run/server/etc.)
|
||||
CMD ["--help"]
|
||||
@@ -0,0 +1,34 @@
|
||||
# Dockerfile for Nix e2e tests
|
||||
# Uses the official nixos/nix image with experimental features enabled
|
||||
# Builds the osmedeus binary inside the container for architecture compatibility
|
||||
|
||||
FROM ubuntu:22.04
|
||||
|
||||
# Avoid interactive prompts
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Install dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl \
|
||||
xz-utils \
|
||||
ca-certificates \
|
||||
sudo \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create nix build group and users (required even for --no-daemon)
|
||||
RUN groupadd -r nixbld \
|
||||
&& for i in $(seq 1 10); do \
|
||||
useradd -r -g nixbld -G nixbld \
|
||||
-d /var/empty -s /usr/sbin/nologin nixbld$i; \
|
||||
done
|
||||
|
||||
# Install Nix (single-user, container-compatible)
|
||||
RUN sh <(curl -L https://nixos.org/nix/install) --no-daemon
|
||||
|
||||
# Ensure Nix is available in all shells
|
||||
ENV PATH="/root/.nix-profile/bin:/root/.nix-profile/sbin:$PATH"
|
||||
|
||||
# Optional but recommended
|
||||
RUN nix-channel --update && nix profile add nixpkgs#nixFlakes
|
||||
|
||||
CMD [ "bash" ]
|
||||
@@ -0,0 +1,34 @@
|
||||
# Osmedeus Development Dockerfile
|
||||
# Full Go toolchain for development and debugging
|
||||
|
||||
FROM golang:1.21-alpine
|
||||
|
||||
# Install development dependencies
|
||||
RUN apk add --no-cache \
|
||||
git \
|
||||
make \
|
||||
bash \
|
||||
curl \
|
||||
vim \
|
||||
&& go install github.com/cosmtrek/air@latest
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy go mod files first for better layer caching
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the binary
|
||||
RUN go build -o build/bin/osmedeus ./cmd/osmedeus
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p /root/osmedeus-base /root/workspaces-osmedeus
|
||||
|
||||
# Expose default server port
|
||||
EXPOSE 8002
|
||||
|
||||
# Default command: run server without auth (dev mode)
|
||||
CMD ["./build/bin/osmedeus", "serve", "-A"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Dockerfile for Nix e2e tests
|
||||
# Uses the official nixos/nix image with experimental features enabled
|
||||
# Builds the osmedeus binary inside the container for architecture compatibility
|
||||
|
||||
FROM nixos/nix:latest
|
||||
|
||||
# Enable flakes and nix-command experimental features
|
||||
RUN mkdir -p /root/.config/nix && \
|
||||
echo "experimental-features = nix-command flakes" > /root/.config/nix/nix.conf
|
||||
|
||||
# Install Go using Nix (for building the binary)
|
||||
RUN nix profile add nixpkgs#go
|
||||
|
||||
# Create app directory
|
||||
WORKDIR /app
|
||||
|
||||
# Copy source code for building
|
||||
COPY go.mod go.sum ./
|
||||
COPY cmd/ cmd/
|
||||
COPY internal/ internal/
|
||||
COPY pkg/ pkg/
|
||||
COPY public/ public/
|
||||
COPY docs/ docs/
|
||||
|
||||
# Build the binary inside the container (disable CGO for static build)
|
||||
RUN CGO_ENABLED=0 go build -o /app/bin/osmedeus ./cmd/osmedeus && \
|
||||
chmod +x /app/bin/osmedeus
|
||||
|
||||
# Command to keep container running for test execution
|
||||
CMD ["sleep", "infinity"]
|
||||
@@ -0,0 +1,99 @@
|
||||
# Osmedeus Toolbox Dockerfile
|
||||
# Full-featured image with all security tools pre-installed via Nix and direct-fetch
|
||||
# Based on latest Golang with Python3 and Nix package manager
|
||||
|
||||
# Stage 1: Build osmedeus binary
|
||||
FROM golang:latest AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy go mod files first for better layer caching
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the binary
|
||||
ARG VERSION=5.0.0
|
||||
ARG BUILD_TIME
|
||||
ARG COMMIT_HASH
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build \
|
||||
-ldflags "-s -w -X main.BuildTime=${BUILD_TIME} -X main.CommitHash=${COMMIT_HASH}" \
|
||||
-o /app/bin/osmedeus ./cmd/osmedeus
|
||||
|
||||
# Stage 2: Toolbox runtime with all tools
|
||||
FROM golang:latest
|
||||
|
||||
# Install system dependencies and Python3
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
wget \
|
||||
git \
|
||||
unzip \
|
||||
jq \
|
||||
bash \
|
||||
xz-utils \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-venv \
|
||||
chromium \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create symlink for python
|
||||
RUN ln -sf /usr/bin/python3 /usr/bin/python
|
||||
|
||||
# Install Nix package manager (single-user mode for Docker)
|
||||
RUN mkdir -m 0755 /nix && \
|
||||
curl -L https://nixos.org/nix/install | sh -s -- --no-daemon
|
||||
|
||||
# Enable Nix experimental features for flakes
|
||||
RUN mkdir -p /root/.config/nix && \
|
||||
echo "experimental-features = nix-command flakes" > /root/.config/nix/nix.conf
|
||||
|
||||
# Set up Nix environment
|
||||
ENV PATH="/root/.nix-profile/bin:/nix/var/nix/profiles/default/bin:${PATH}"
|
||||
ENV NIX_PATH="/root/.nix-defexpr/channels"
|
||||
|
||||
# Source Nix profile in bashrc
|
||||
RUN echo '. /root/.nix-profile/etc/profile.d/nix.sh' >> /root/.bashrc
|
||||
|
||||
# Copy osmedeus binary from builder
|
||||
COPY --from=builder /app/bin/osmedeus /usr/local/bin/osmedeus
|
||||
|
||||
# Create base directories
|
||||
RUN mkdir -p /root/osmedeus-base /root/workspaces-osmedeus
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
# Initialize osmedeus base folder with sample workflows
|
||||
RUN osmedeus install base --sample
|
||||
|
||||
# Use bash shell for subsequent commands to source nix profile
|
||||
SHELL ["/bin/bash", "-lc"]
|
||||
|
||||
# Install binaries via Nix (nix-build-install)
|
||||
RUN . /root/.nix-profile/etc/profile.d/nix.sh && \
|
||||
osmedeus install binary --all --nix-build-install || true
|
||||
|
||||
# Install remaining binaries via direct-fetch (including optional ones)
|
||||
RUN osmedeus install binary --all --install-optional || true
|
||||
|
||||
# Set up PATH for external binaries
|
||||
ENV PATH="/root/osmedeus-base/external-binaries:/root/.nix-profile/bin:/nix/var/nix/profiles/default/bin:${PATH}"
|
||||
|
||||
# Run osmedeus health check to verify installation
|
||||
RUN osmedeus health || true
|
||||
|
||||
# Expose default server port
|
||||
EXPOSE 8002
|
||||
|
||||
# Use bash as default shell
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
# Default entrypoint
|
||||
ENTRYPOINT ["osmedeus"]
|
||||
|
||||
# Default command shows help
|
||||
CMD ["--help"]
|
||||
@@ -0,0 +1,12 @@
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: osm-test-redis
|
||||
ports:
|
||||
- "6399:6379" # Use non-standard port to avoid conflicts with local Redis
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 1s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,11 @@
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
nix-test:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: build/docker/Dockerfile.nix-test
|
||||
container_name: osm-test-nix
|
||||
# Binary is built inside the container, no volume mount needed
|
||||
command: ["sleep", "infinity"]
|
||||
restart: "no"
|
||||
@@ -0,0 +1,29 @@
|
||||
version: '3.8'
|
||||
|
||||
# Simple PostgreSQL setup for testing database schema
|
||||
# Usage:
|
||||
# docker-compose -f docker-compose.postgres-test.yaml up -d
|
||||
# osmedeus db seed --config build/docker/osm-settings.postgres-test.yaml
|
||||
# docker-compose -f docker-compose.postgres-test.yaml down -v
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: osmedeus-postgres-test
|
||||
environment:
|
||||
POSTGRES_USER: osmedeus
|
||||
POSTGRES_PASSWORD: test_password_123
|
||||
POSTGRES_DB: osmedeus
|
||||
ports:
|
||||
- "5432:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U osmedeus"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
volumes:
|
||||
- postgres-test-data:/var/lib/postgresql/data
|
||||
|
||||
volumes:
|
||||
postgres-test-data:
|
||||
driver: local
|
||||
@@ -0,0 +1,155 @@
|
||||
version: '3.8'
|
||||
|
||||
# Osmedeus Production Stack with PostgreSQL
|
||||
# ==========================================
|
||||
# Usage:
|
||||
# 1. Copy .env.example to .env and configure secrets
|
||||
# 2. Copy osm-settings.production.yaml to osm-settings.yaml and adjust if needed
|
||||
# 3. Start: docker-compose -f build/docker/docker-compose.production.yaml up -d
|
||||
# 4. Scale workers: docker-compose -f build/docker/docker-compose.production.yaml up -d --scale worker=5
|
||||
#
|
||||
# First-time setup:
|
||||
# docker-compose -f build/docker/docker-compose.production.yaml up -d postgres redis
|
||||
# # Wait for healthy status, then start the app:
|
||||
# docker-compose -f build/docker/docker-compose.production.yaml up -d
|
||||
|
||||
services:
|
||||
# PostgreSQL - Primary database for persistent storage
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: osmedeus-postgres
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-osmedeus}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-osmedeus}
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-osmedeus} -d ${POSTGRES_DB:-osmedeus}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
networks:
|
||||
- osmedeus-network
|
||||
# Uncomment to expose PostgreSQL externally (not recommended for production)
|
||||
# ports:
|
||||
# - "5432:5432"
|
||||
|
||||
# Redis - Message queue for distributed task processing
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: osmedeus-redis
|
||||
restart: unless-stopped
|
||||
command: >
|
||||
redis-server
|
||||
--appendonly yes
|
||||
--maxmemory 512mb
|
||||
--maxmemory-policy allkeys-lru
|
||||
${REDIS_PASSWORD:+--requirepass ${REDIS_PASSWORD}}
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- osmedeus-network
|
||||
# Uncomment to expose Redis externally (not recommended for production)
|
||||
# ports:
|
||||
# - "6379:6379"
|
||||
|
||||
# Master Node - API server and task coordinator
|
||||
server:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: build/docker/Dockerfile
|
||||
image: osmedeus:latest
|
||||
container_name: osmedeus-server
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${OSM_SERVER_PORT:-8002}:8002"
|
||||
environment:
|
||||
# These are passed to the container but config is read from mounted file
|
||||
- TZ=${TZ:-UTC}
|
||||
volumes:
|
||||
- osmedeus-data:/root/osmedeus-base
|
||||
- workspaces:/root/workspaces-osmedeus
|
||||
- ./osm-settings.production.yaml:/root/osmedeus-base/osm-settings.yaml:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
command: ["serve", "--master"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8002/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
start_period: 30s
|
||||
retries: 3
|
||||
networks:
|
||||
- osmedeus-network
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
|
||||
# Worker Nodes - Execute distributed scan tasks
|
||||
worker:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: build/docker/Dockerfile
|
||||
image: osmedeus:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- TZ=${TZ:-UTC}
|
||||
volumes:
|
||||
- osmedeus-data:/root/osmedeus-base
|
||||
- workspaces:/root/workspaces-osmedeus
|
||||
- ./osm-settings.production.yaml:/root/osmedeus-base/osm-settings.yaml:ro
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
server:
|
||||
condition: service_healthy
|
||||
command: ["worker", "join", "--redis-url", "redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}redis:6379"]
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2'
|
||||
memory: 2G
|
||||
reservations:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
networks:
|
||||
- osmedeus-network
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "3"
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
driver: local
|
||||
name: osmedeus-postgres-data
|
||||
redis-data:
|
||||
driver: local
|
||||
name: osmedeus-redis-data
|
||||
osmedeus-data:
|
||||
driver: local
|
||||
name: osmedeus-app-data
|
||||
workspaces:
|
||||
driver: local
|
||||
name: osmedeus-workspaces
|
||||
|
||||
networks:
|
||||
osmedeus-network:
|
||||
driver: bridge
|
||||
name: osmedeus-network
|
||||
@@ -0,0 +1,16 @@
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
ssh-server:
|
||||
image: linuxserver/openssh-server:latest
|
||||
container_name: osm-test-ssh
|
||||
environment:
|
||||
- PUID=1000
|
||||
- PGID=1000
|
||||
- TZ=UTC
|
||||
- PASSWORD_ACCESS=true
|
||||
- USER_NAME=testuser
|
||||
- USER_PASSWORD=testpass
|
||||
ports:
|
||||
- "2222:2222"
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,31 @@
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
osmedeus-toolbox:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: build/docker/Dockerfile.toolbox
|
||||
args:
|
||||
VERSION: "5.0.0"
|
||||
BUILD_TIME: "${BUILD_TIME:-unknown}"
|
||||
COMMIT_HASH: "${COMMIT_HASH:-unknown}"
|
||||
image: osmedeus-toolbox:latest
|
||||
container_name: osmedeus-toolbox
|
||||
hostname: osmedeus-toolbox
|
||||
volumes:
|
||||
# Persist workspaces and scan results
|
||||
- osmedeus-workspaces:/root/workspaces-osmedeus
|
||||
# Persist database
|
||||
- osmedeus-data:/root/osmedeus-base
|
||||
ports:
|
||||
- "8002:8002"
|
||||
environment:
|
||||
- OSMEDEUS_BASE=/root/osmedeus-base
|
||||
# Keep container running for interactive use
|
||||
stdin_open: true
|
||||
tty: true
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
osmedeus-workspaces:
|
||||
osmedeus-data:
|
||||
@@ -0,0 +1,97 @@
|
||||
version: '3.8'
|
||||
|
||||
# Osmedeus Master-Worker Architecture with Redis
|
||||
# Usage:
|
||||
# docker-compose -f build/docker/docker-compose.yml up -d
|
||||
# docker-compose -f build/docker/docker-compose.yml up -d --scale worker=5
|
||||
|
||||
services:
|
||||
# Redis - Message queue for distributed task processing
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: osmedeus-redis
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- osmedeus-network
|
||||
|
||||
# Master Node - Coordinates tasks and exposes API
|
||||
master:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: build/docker/Dockerfile
|
||||
image: osmedeus:latest
|
||||
container_name: osmedeus-master
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8002:8002"
|
||||
environment:
|
||||
- REDIS_HOST=redis
|
||||
- REDIS_PORT=6379
|
||||
volumes:
|
||||
- osmedeus-data:/root/osmedeus-base
|
||||
- workspaces:/root/workspaces-osmedeus
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
command: ["serve", "--master", "-A"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8002/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
start_period: 10s
|
||||
retries: 3
|
||||
networks:
|
||||
- osmedeus-network
|
||||
|
||||
# Worker Nodes - Execute distributed tasks
|
||||
worker:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: build/docker/Dockerfile
|
||||
image: osmedeus:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- REDIS_HOST=redis
|
||||
- REDIS_PORT=6379
|
||||
volumes:
|
||||
- osmedeus-data:/root/osmedeus-base
|
||||
- workspaces:/root/workspaces-osmedeus
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
master:
|
||||
condition: service_healthy
|
||||
command: ["worker", "join", "--redis-url", "redis://redis:6379"]
|
||||
deploy:
|
||||
replicas: 2
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1'
|
||||
memory: 1G
|
||||
reservations:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
networks:
|
||||
- osmedeus-network
|
||||
|
||||
volumes:
|
||||
redis-data:
|
||||
driver: local
|
||||
osmedeus-data:
|
||||
driver: local
|
||||
workspaces:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
osmedeus-network:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,12 @@
|
||||
# Osmedeus PostgreSQL Test Configuration
|
||||
# Used for testing database schema with Docker PostgreSQL
|
||||
|
||||
database:
|
||||
db_engine: postgresql
|
||||
host: localhost
|
||||
port: 5432
|
||||
username: osmedeus
|
||||
password: test_password_123
|
||||
db_name: osmedeus
|
||||
ssl_mode: disable
|
||||
connection_timeout: 30
|
||||
@@ -0,0 +1,162 @@
|
||||
# Osmedeus Production Configuration
|
||||
# ==================================
|
||||
# This configuration is optimized for standalone Docker deployment with SQLite.
|
||||
# Copy this file and adjust values for your environment.
|
||||
#
|
||||
# Usage:
|
||||
# 1. Mount to /root/osmedeus-base/osm-settings.yaml
|
||||
# 2. Update passwords and secrets with secure values
|
||||
# 3. Adjust resource limits based on your infrastructure
|
||||
|
||||
# =============================================================================
|
||||
# Environment Paths
|
||||
# =============================================================================
|
||||
environment:
|
||||
# Binary tools directory
|
||||
binaries: "{{base_folder}}/external-binaries"
|
||||
|
||||
# Data directory for wordlists, templates, etc.
|
||||
external_data: "{{base_folder}}/external-data"
|
||||
|
||||
# External configuration files
|
||||
external_configs: "{{base_folder}}/external-configs"
|
||||
|
||||
# Output directory for scan workspaces
|
||||
workspaces: /root/workspaces-osmedeus
|
||||
|
||||
# Workflow YAML files directory
|
||||
workflows: "{{base_folder}}/workflows"
|
||||
|
||||
# Workspace snapshots directory
|
||||
snapshot: "{{base_folder}}/snapshot"
|
||||
|
||||
# =============================================================================
|
||||
# Database Configuration - SQLite (Standalone)
|
||||
# =============================================================================
|
||||
database:
|
||||
# Use SQLite for standalone Docker deployment
|
||||
db_engine: sqlite
|
||||
|
||||
# SQLite database file path
|
||||
db_path: "{{base_folder}}/database-osm.sqlite"
|
||||
|
||||
# Connection timeout in seconds
|
||||
connection_timeout: 60
|
||||
|
||||
# PostgreSQL settings (uncomment for distributed deployment)
|
||||
# db_engine: postgresql
|
||||
# host: postgres
|
||||
# port: 5432
|
||||
# username: osmedeus
|
||||
# password: "CHANGE_ME_POSTGRES_PASSWORD"
|
||||
# db_name: osmedeus
|
||||
# ssl_mode: disable
|
||||
|
||||
# =============================================================================
|
||||
# Server Configuration
|
||||
# =============================================================================
|
||||
server:
|
||||
# Bind to all interfaces (required for Docker)
|
||||
host: "0.0.0.0"
|
||||
|
||||
# API server port
|
||||
port: 8002
|
||||
|
||||
# UI static files path
|
||||
ui_path: "{{base_folder}}/ui/"
|
||||
|
||||
# Workspace static files URL prefix (auto-generated if empty)
|
||||
workspace_prefix_key: ""
|
||||
|
||||
# Authentication credentials
|
||||
# IMPORTANT: Change these for production!
|
||||
simple_user_map_key:
|
||||
admin: "CHANGE_ME_ADMIN_PASSWORD"
|
||||
|
||||
# JWT settings
|
||||
jwt:
|
||||
# IMPORTANT: Use a strong, unique secret (min 32 characters recommended)
|
||||
# Generate with: openssl rand -base64 32
|
||||
secret_signing_key: "CHANGE_ME_JWT_SECRET_MIN_32_CHARS"
|
||||
|
||||
# Token expiration in minutes (1440 = 24 hours)
|
||||
expiration_minutes: 1440
|
||||
|
||||
# =============================================================================
|
||||
# Scan Tactic Configuration
|
||||
# =============================================================================
|
||||
scan_tactic:
|
||||
# Production-optimized thread counts
|
||||
aggressive: 50
|
||||
default: 20
|
||||
gently: 5
|
||||
|
||||
# =============================================================================
|
||||
# Redis Configuration (Optional)
|
||||
# =============================================================================
|
||||
# Required for distributed mode with workers
|
||||
redis:
|
||||
# Redis hostname (matches docker-compose service name)
|
||||
host: redis
|
||||
|
||||
# Redis port
|
||||
port: 6379
|
||||
|
||||
# Redis authentication (if REDIS_PASSWORD is set in .env)
|
||||
username: ""
|
||||
password: ""
|
||||
|
||||
# Redis database number
|
||||
db: 0
|
||||
|
||||
# Connection timeout
|
||||
connection_timeout: 60
|
||||
|
||||
# =============================================================================
|
||||
# Global Variables
|
||||
# =============================================================================
|
||||
# API keys and secrets for external services
|
||||
# Add your API keys here for use in workflows
|
||||
global_variables:
|
||||
# GitHub API key for authenticated requests
|
||||
# - name: GITHUB_API_KEY
|
||||
# value: "ghp_xxxxxxxxxxxx"
|
||||
# as_env: true
|
||||
|
||||
# Shodan API key
|
||||
# - name: SHODAN_API_KEY
|
||||
# value: "xxxxxxxxxxxx"
|
||||
# as_env: true
|
||||
|
||||
# =============================================================================
|
||||
# Notification Configuration (Optional)
|
||||
# =============================================================================
|
||||
notification:
|
||||
# Telegram notifications
|
||||
telegram:
|
||||
enabled: false
|
||||
bot_token: ""
|
||||
chat_id: ""
|
||||
|
||||
# =============================================================================
|
||||
# Cloud Storage Configuration (Optional)
|
||||
# =============================================================================
|
||||
# S3-compatible storage for workspace backups
|
||||
cdn_storage:
|
||||
enabled: false
|
||||
access_key_id: ""
|
||||
secret_access_key: ""
|
||||
bucket: ""
|
||||
region: ""
|
||||
endpoint: ""
|
||||
|
||||
# =============================================================================
|
||||
# LLM Configuration (Optional)
|
||||
# =============================================================================
|
||||
# AI/LLM provider settings for intelligent analysis
|
||||
llm:
|
||||
enabled: false
|
||||
provider: openai
|
||||
api_key: ""
|
||||
model: "gpt-4"
|
||||
base_url: ""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,739 @@
|
||||
{
|
||||
"schemes": [
|
||||
"http",
|
||||
"https"
|
||||
],
|
||||
"swagger": "2.0",
|
||||
"info": {
|
||||
"description": "Modern Orchestration Engine for Security - REST API for managing security automation workflows, scans, and distributed task execution.",
|
||||
"title": "Osmedeus API",
|
||||
"termsOfService": "https://docs.osmedeus.org/terms/",
|
||||
"contact": {
|
||||
"name": "Osmedeus Support",
|
||||
"url": "https://github.com/j3ssie/osmedeus",
|
||||
"email": "support@osmedeus.org"
|
||||
},
|
||||
"license": {
|
||||
"name": "MIT",
|
||||
"url": "https://opensource.org/licenses/MIT"
|
||||
},
|
||||
"version": "5.0.0"
|
||||
},
|
||||
"host": "localhost:8002",
|
||||
"basePath": "/",
|
||||
"paths": {
|
||||
"/": {
|
||||
"get": {
|
||||
"description": "Get server version and info",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Info"
|
||||
],
|
||||
"summary": "Server info",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Server information",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/login": {
|
||||
"post": {
|
||||
"description": "Authenticate user and get JWT token",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Auth"
|
||||
],
|
||||
"summary": "User login",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "Login credentials",
|
||||
"name": "credentials",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/pkg_server_handlers.LoginRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "JWT token",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Invalid credentials",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/new-scan": {
|
||||
"post": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Execute a workflow against a target",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Scans"
|
||||
],
|
||||
"summary": "Create a new scan",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "Scan configuration",
|
||||
"name": "scan",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/pkg_server_handlers.CreateScanRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"202": {
|
||||
"description": "Scan started",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Workflow not found",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/tasks": {
|
||||
"get": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Get a list of all running and completed tasks",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Distributed"
|
||||
],
|
||||
"summary": "List all tasks",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "List of running and completed tasks",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failed to list tasks",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Submit a new task to the distributed worker queue",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Distributed"
|
||||
],
|
||||
"summary": "Submit a new task",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "Task configuration",
|
||||
"name": "task",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/pkg_server_handlers.SubmitTaskRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"202": {
|
||||
"description": "Task submitted",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failed to submit task",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/tasks/{id}": {
|
||||
"get": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Get details for a specific task by ID",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Distributed"
|
||||
],
|
||||
"summary": "Get task details",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Task ID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Task details",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Task not found",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/workers": {
|
||||
"get": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Get a list of all registered workers in the distributed pool",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Distributed"
|
||||
],
|
||||
"summary": "List all workers",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "List of workers",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failed to list workers",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/workers/{id}": {
|
||||
"get": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Get details for a specific worker by ID",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Distributed"
|
||||
],
|
||||
"summary": "Get worker details",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Worker ID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Worker details",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Worker not found",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failed to get worker",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/workflows": {
|
||||
"get": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Get a list of all available workflows with details",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Workflows"
|
||||
],
|
||||
"summary": "List all workflows",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "List of workflows",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failed to load workflows",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/workflows/{name}": {
|
||||
"get": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Get detailed information about a specific workflow. Use show_yaml=true to get raw YAML content.",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Workflows"
|
||||
],
|
||||
"summary": "Get workflow details",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Workflow name",
|
||||
"name": "name",
|
||||
"in": "path",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "boolean",
|
||||
"description": "Return raw YAML content instead of JSON",
|
||||
"name": "show_yaml",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Workflow details",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Workflow not found",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/workspaces": {
|
||||
"get": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Get a list of all scan workspaces",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Workspaces"
|
||||
],
|
||||
"summary": "List all workspaces",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "List of workspaces",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failed to read workspaces",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/upload-file": {
|
||||
"post": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Upload a file containing a list of inputs (targets, URLs, etc.) for later use in scans",
|
||||
"consumes": [
|
||||
"multipart/form-data"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Files"
|
||||
],
|
||||
"summary": "Upload input file",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "file",
|
||||
"description": "Input file to upload",
|
||||
"name": "file",
|
||||
"in": "formData",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "File uploaded with path",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/workflow-upload": {
|
||||
"post": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Upload a raw YAML workflow file and save it to the workflows directory",
|
||||
"consumes": [
|
||||
"multipart/form-data"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Workflows"
|
||||
],
|
||||
"summary": "Upload workflow file",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "file",
|
||||
"description": "Workflow YAML file",
|
||||
"name": "file",
|
||||
"in": "formData",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Workflow uploaded",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request or YAML",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/osm/api/snapshot-download/{workspace_name}": {
|
||||
"get": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Compress a workspace folder into a zip file and download it",
|
||||
"produces": [
|
||||
"application/zip"
|
||||
],
|
||||
"tags": [
|
||||
"Snapshots"
|
||||
],
|
||||
"summary": "Download workspace snapshot",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Workspace name",
|
||||
"name": "workspace_name",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Zip file download",
|
||||
"schema": {
|
||||
"type": "file"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Workspace not found",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Failed to create snapshot",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/health": {
|
||||
"get": {
|
||||
"description": "Check if the server is running",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Health"
|
||||
],
|
||||
"summary": "Health check",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "status: ok",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/health/ready": {
|
||||
"get": {
|
||||
"description": "Check if the server is ready to accept requests",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"Health"
|
||||
],
|
||||
"summary": "Readiness check",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "status: ready",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"pkg_server_handlers.CreateScanRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"concurrency": {
|
||||
"description": "Number of concurrent scans (default: 1)",
|
||||
"type": "integer",
|
||||
"default": 1
|
||||
},
|
||||
"flow": {
|
||||
"description": "Flow workflow name",
|
||||
"type": "string"
|
||||
},
|
||||
"module": {
|
||||
"description": "Module workflow name",
|
||||
"type": "string"
|
||||
},
|
||||
"params": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"description": "Single target to scan",
|
||||
"type": "string"
|
||||
},
|
||||
"targets": {
|
||||
"description": "Array of targets to scan",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"target_file": {
|
||||
"description": "Path to file containing targets (one per line)",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"pkg_server_handlers.LoginRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"password": {
|
||||
"type": "string"
|
||||
},
|
||||
"username": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"pkg_server_handlers.SubmitTaskRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"params": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
},
|
||||
"target": {
|
||||
"type": "string"
|
||||
},
|
||||
"workflow_kind": {
|
||||
"type": "string"
|
||||
},
|
||||
"workflow_name": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"securityDefinitions": {
|
||||
"BearerAuth": {
|
||||
"description": "JWT Bearer token authentication. Format: \"Bearer {token}\"",
|
||||
"type": "apiKey",
|
||||
"name": "Authorization",
|
||||
"in": "header"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,487 @@
|
||||
basePath: /
|
||||
definitions:
|
||||
pkg_server_handlers.CreateScanRequest:
|
||||
properties:
|
||||
concurrency:
|
||||
default: 1
|
||||
description: Number of concurrent scans (default: 1)
|
||||
type: integer
|
||||
flow:
|
||||
description: Flow workflow name
|
||||
type: string
|
||||
module:
|
||||
description: Module workflow name
|
||||
type: string
|
||||
params:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
target:
|
||||
description: Single target to scan
|
||||
type: string
|
||||
targets:
|
||||
description: Array of targets to scan
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
target_file:
|
||||
description: Path to file containing targets (one per line)
|
||||
type: string
|
||||
type: object
|
||||
pkg_server_handlers.LoginRequest:
|
||||
properties:
|
||||
password:
|
||||
type: string
|
||||
username:
|
||||
type: string
|
||||
type: object
|
||||
pkg_server_handlers.SubmitTaskRequest:
|
||||
properties:
|
||||
params:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
target:
|
||||
type: string
|
||||
workflow_kind:
|
||||
type: string
|
||||
workflow_name:
|
||||
type: string
|
||||
type: object
|
||||
host: localhost:8002
|
||||
info:
|
||||
contact:
|
||||
email: support@osmedeus.org
|
||||
name: Osmedeus Support
|
||||
url: https://github.com/j3ssie/osmedeus
|
||||
description: Modern Orchestration Engine for Security - REST API for managing security
|
||||
automation workflows, scans, and distributed task execution.
|
||||
license:
|
||||
name: MIT
|
||||
url: https://opensource.org/licenses/MIT
|
||||
termsOfService: https://docs.osmedeus.org/terms/
|
||||
title: Osmedeus API
|
||||
version: 5.0.0
|
||||
paths:
|
||||
/:
|
||||
get:
|
||||
description: Get server version and info
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: Server information
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
summary: Server info
|
||||
tags:
|
||||
- Info
|
||||
/osm/api/login:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: Authenticate user and get JWT token
|
||||
parameters:
|
||||
- description: Login credentials
|
||||
in: body
|
||||
name: credentials
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/pkg_server_handlers.LoginRequest'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: JWT token
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
"400":
|
||||
description: Invalid request
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"401":
|
||||
description: Invalid credentials
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
summary: User login
|
||||
tags:
|
||||
- Auth
|
||||
/osm/api/new-scan:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: Execute a workflow against a target
|
||||
parameters:
|
||||
- description: Scan configuration
|
||||
in: body
|
||||
name: scan
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/pkg_server_handlers.CreateScanRequest'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"202":
|
||||
description: Scan started
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"400":
|
||||
description: Invalid request
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"404":
|
||||
description: Workflow not found
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Create a new scan
|
||||
tags:
|
||||
- Scans
|
||||
/osm/api/upload-file:
|
||||
post:
|
||||
consumes:
|
||||
- multipart/form-data
|
||||
description: Upload a file containing a list of inputs (targets, URLs, etc.)
|
||||
for later use in scans
|
||||
parameters:
|
||||
- description: Input file to upload
|
||||
in: formData
|
||||
name: file
|
||||
required: true
|
||||
type: file
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: File uploaded with path
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"400":
|
||||
description: Invalid request
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Upload input file
|
||||
tags:
|
||||
- Files
|
||||
/osm/api/workflow-upload:
|
||||
post:
|
||||
consumes:
|
||||
- multipart/form-data
|
||||
description: Upload a raw YAML workflow file and save it to the workflows directory
|
||||
parameters:
|
||||
- description: Workflow YAML file
|
||||
in: formData
|
||||
name: file
|
||||
required: true
|
||||
type: file
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"201":
|
||||
description: Workflow uploaded
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"400":
|
||||
description: Invalid request or YAML
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Upload workflow file
|
||||
tags:
|
||||
- Workflows
|
||||
/osm/api/tasks:
|
||||
get:
|
||||
description: Get a list of all running and completed tasks
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: List of running and completed tasks
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"500":
|
||||
description: Failed to list tasks
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: List all tasks
|
||||
tags:
|
||||
- Distributed
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: Submit a new task to the distributed worker queue
|
||||
parameters:
|
||||
- description: Task configuration
|
||||
in: body
|
||||
name: task
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/pkg_server_handlers.SubmitTaskRequest'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"202":
|
||||
description: Task submitted
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"400":
|
||||
description: Invalid request
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"500":
|
||||
description: Failed to submit task
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Submit a new task
|
||||
tags:
|
||||
- Distributed
|
||||
/osm/api/tasks/{id}:
|
||||
get:
|
||||
description: Get details for a specific task by ID
|
||||
parameters:
|
||||
- description: Task ID
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
type: string
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: Task details
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"404":
|
||||
description: Task not found
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Get task details
|
||||
tags:
|
||||
- Distributed
|
||||
/osm/api/workers:
|
||||
get:
|
||||
description: Get a list of all registered workers in the distributed pool
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: List of workers
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"500":
|
||||
description: Failed to list workers
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: List all workers
|
||||
tags:
|
||||
- Distributed
|
||||
/osm/api/workers/{id}:
|
||||
get:
|
||||
description: Get details for a specific worker by ID
|
||||
parameters:
|
||||
- description: Worker ID
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
type: string
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: Worker details
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"404":
|
||||
description: Worker not found
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"500":
|
||||
description: Failed to get worker
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Get worker details
|
||||
tags:
|
||||
- Distributed
|
||||
/osm/api/workflows:
|
||||
get:
|
||||
description: Get a list of all available workflows with details
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: List of workflows
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"500":
|
||||
description: Failed to load workflows
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: List all workflows
|
||||
tags:
|
||||
- Workflows
|
||||
/osm/api/workflows/{name}:
|
||||
get:
|
||||
description: Get detailed information about a specific workflow. Use show_yaml=true
|
||||
to get raw YAML content.
|
||||
parameters:
|
||||
- description: Workflow name
|
||||
in: path
|
||||
name: name
|
||||
required: true
|
||||
type: string
|
||||
- description: Return raw YAML content instead of JSON
|
||||
in: query
|
||||
name: show_yaml
|
||||
type: boolean
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: Workflow details
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"404":
|
||||
description: Workflow not found
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Get workflow details
|
||||
tags:
|
||||
- Workflows
|
||||
/osm/api/workspaces:
|
||||
get:
|
||||
description: Get a list of all scan workspaces
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: List of workspaces
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"500":
|
||||
description: Failed to read workspaces
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: List all workspaces
|
||||
tags:
|
||||
- Workspaces
|
||||
/osm/api/snapshot-download/{workspace_name}:
|
||||
get:
|
||||
description: Compress a workspace folder into a zip file and download it
|
||||
parameters:
|
||||
- description: Workspace name
|
||||
in: path
|
||||
name: workspace_name
|
||||
required: true
|
||||
type: string
|
||||
produces:
|
||||
- application/zip
|
||||
responses:
|
||||
"200":
|
||||
description: Zip file download
|
||||
schema:
|
||||
type: file
|
||||
"404":
|
||||
description: Workspace not found
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"500":
|
||||
description: Failed to create snapshot
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Download workspace snapshot
|
||||
tags:
|
||||
- Snapshots
|
||||
/health:
|
||||
get:
|
||||
description: Check if the server is running
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: 'status: ok'
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
summary: Health check
|
||||
tags:
|
||||
- Health
|
||||
/health/ready:
|
||||
get:
|
||||
description: Check if the server is ready to accept requests
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: 'status: ready'
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
summary: Readiness check
|
||||
tags:
|
||||
- Health
|
||||
schemes:
|
||||
- http
|
||||
- https
|
||||
securityDefinitions:
|
||||
BearerAuth:
|
||||
description: 'JWT Bearer token authentication. Format: "Bearer {token}"'
|
||||
in: header
|
||||
name: Authorization
|
||||
type: apiKey
|
||||
swagger: "2.0"
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,60 @@
|
||||
# Osmedeus API Documentation
|
||||
|
||||
## Overview
|
||||
|
||||
The Osmedeus API provides a RESTful interface for managing security automation workflows, runs, and distributed task execution.
|
||||
|
||||
**Base URL:** `http://localhost:8002`
|
||||
|
||||
**Default Port:** `8002`
|
||||
|
||||
## Authentication
|
||||
|
||||
Most API endpoints require JWT authentication. First, obtain a token via the login endpoint, then include it in subsequent requests using the `Authorization: Bearer <token>` header.
|
||||
|
||||
See [Authentication](authentication.md) for details.
|
||||
|
||||
## API Reference
|
||||
|
||||
| Category | Description |
|
||||
|----------|-------------|
|
||||
| [Public Endpoints](public.md) | Server info, health checks, Swagger docs |
|
||||
| [Authentication](authentication.md) | Login and JWT token management |
|
||||
| [Workflows](workflows.md) | List, view, and refresh workflows |
|
||||
| [Runs](runs.md) | Create and manage workflow executions |
|
||||
| [File Uploads](uploads.md) | Upload target files and workflows |
|
||||
| [Snapshots](snapshots.md) | Download workspace snapshots |
|
||||
| [Workspaces](workspaces.md) | List and manage workspaces |
|
||||
| [Assets](assets.md) | View discovered assets |
|
||||
| [Vulnerabilities](vulnerabilities.md) | View and manage vulnerabilities |
|
||||
| [Event Logs](event-logs.md) | View execution event logs |
|
||||
| [Functions](functions.md) | Execute and list utility functions |
|
||||
| [System Statistics](system.md) | Get aggregated system stats |
|
||||
| [Settings](settings.md) | Manage server configuration |
|
||||
| [Installation](install.md) | Install binaries and workflows |
|
||||
| [Schedules](schedules.md) | Manage scheduled workflows |
|
||||
| [Distributed Mode](distributed.md) | Worker and task management |
|
||||
| [LLM API](llm.md) | Large Language Model API |
|
||||
| [Reference](reference.md) | Error codes, pagination, cron expressions, step types |
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Get server info (no auth required)
|
||||
curl http://localhost:8002/server-info
|
||||
|
||||
# Login and get token
|
||||
export TOKEN=$(curl -s -X POST http://localhost:8002/osm/api/login \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"username": "osmedeus", "password": "admin"}' | jq -r '.token')
|
||||
|
||||
# List workflows
|
||||
curl http://localhost:8002/osm/api/workflows \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
|
||||
# Start a scan
|
||||
curl -X POST http://localhost:8002/osm/api/runs \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"flow": "subdomain-enum", "target": "example.com"}'
|
||||
```
|
||||
@@ -0,0 +1,123 @@
|
||||
# Assets
|
||||
|
||||
## List Assets
|
||||
|
||||
Get a paginated list of assets with optional workspace filtering.
|
||||
|
||||
**List all assets:**
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/assets \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**List assets with pagination:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/assets?offset=0&limit=100" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Filter by workspace:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/assets?workspace=example.com" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Combine workspace filter with pagination:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/assets?workspace=example.com&offset=50&limit=25" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": 1,
|
||||
"workspace": "example.com",
|
||||
"asset_value": "api.example.com",
|
||||
"url": "https://api.example.com",
|
||||
"input": "api.example.com",
|
||||
"scheme": "https",
|
||||
"method": "GET",
|
||||
"path": "/",
|
||||
"status_code": 200,
|
||||
"content_type": "application/json",
|
||||
"content_length": 4523,
|
||||
"title": "API Documentation",
|
||||
"words": 523,
|
||||
"lines": 89,
|
||||
"host_ip": "93.184.216.34",
|
||||
"a": ["93.184.216.34", "93.184.216.35"],
|
||||
"tls": "TLS 1.3",
|
||||
"asset_type": "web",
|
||||
"tech": ["nginx/1.21.0", "nodejs", "express"],
|
||||
"time": "245ms",
|
||||
"remarks": "production",
|
||||
"source": "httpx",
|
||||
"created_at": "2025-01-15T10:30:00Z",
|
||||
"updated_at": "2025-01-15T10:30:00Z"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"workspace": "example.com",
|
||||
"asset_value": "admin.example.com",
|
||||
"url": "https://admin.example.com",
|
||||
"input": "admin.example.com",
|
||||
"scheme": "https",
|
||||
"method": "GET",
|
||||
"path": "/login",
|
||||
"status_code": 401,
|
||||
"content_type": "text/html",
|
||||
"content_length": 2156,
|
||||
"title": "Admin Login - Example Corp",
|
||||
"words": 156,
|
||||
"lines": 45,
|
||||
"host_ip": "93.184.216.36",
|
||||
"a": ["93.184.216.36"],
|
||||
"tls": "TLS 1.2",
|
||||
"asset_type": "web",
|
||||
"tech": ["nginx/1.20.0", "php/8.1", "wordpress"],
|
||||
"time": "312ms",
|
||||
"remarks": "admin-panel",
|
||||
"source": "httpx",
|
||||
"created_at": "2025-01-15T10:31:00Z",
|
||||
"updated_at": "2025-01-15T10:31:00Z"
|
||||
}
|
||||
],
|
||||
"pagination": {
|
||||
"total": 500,
|
||||
"offset": 0,
|
||||
"limit": 20
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Asset Fields Reference:**
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `id` | int | Unique asset identifier |
|
||||
| `workspace` | string | Workspace/scan target name |
|
||||
| `asset_value` | string | Primary asset identifier (hostname/subdomain) |
|
||||
| `url` | string | Full URL of the asset |
|
||||
| `input` | string | Original input value |
|
||||
| `scheme` | string | Protocol scheme (http, https) |
|
||||
| `method` | string | HTTP method used |
|
||||
| `path` | string | URL path |
|
||||
| `status_code` | int | HTTP response status code |
|
||||
| `content_type` | string | Response content type |
|
||||
| `content_length` | int | Response body size in bytes |
|
||||
| `title` | string | HTML page title |
|
||||
| `words` | int | Word count in response |
|
||||
| `lines` | int | Line count in response |
|
||||
| `host_ip` | string | Resolved IP address |
|
||||
| `a` | array | DNS A records |
|
||||
| `tls` | string | TLS version information |
|
||||
| `asset_type` | string | Asset type classification |
|
||||
| `tech` | array | Detected technologies |
|
||||
| `time` | string | Response time |
|
||||
| `remarks` | string | Custom labels/remarks |
|
||||
| `source` | string | Discovery source (httpx, nuclei, etc.) |
|
||||
| `created_at` | timestamp | Creation timestamp |
|
||||
| `updated_at` | timestamp | Last update timestamp |
|
||||
@@ -0,0 +1,108 @@
|
||||
# Authentication
|
||||
|
||||
Most API endpoints require JWT authentication. First, obtain a token via the login endpoint, then include it in subsequent requests.
|
||||
|
||||
## Login
|
||||
|
||||
**POST** `/osm/api/login`
|
||||
|
||||
Authenticate and obtain a JWT token.
|
||||
|
||||
### Request
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/login \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"username": "osmedeus",
|
||||
"password": "your-password"
|
||||
}'
|
||||
```
|
||||
|
||||
### Request Body
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `username` | string | Yes | Username configured in server settings |
|
||||
| `password` | string | Yes | Password for the user |
|
||||
|
||||
### Response (200 OK)
|
||||
|
||||
```json
|
||||
{
|
||||
"token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6Im9zbWVkZXVzIiwiZXhwIjoxNzA0MDY3MjAwLCJpYXQiOjE3MDQwNjM2MDB9.abc123..."
|
||||
}
|
||||
```
|
||||
|
||||
### Error Responses
|
||||
|
||||
**400 Bad Request** - Invalid request body:
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Invalid request body"
|
||||
}
|
||||
```
|
||||
|
||||
**401 Unauthorized** - Invalid credentials:
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Invalid credentials"
|
||||
}
|
||||
```
|
||||
|
||||
## Token Details
|
||||
|
||||
- **Algorithm**: HS256 (HMAC-SHA256)
|
||||
- **Expiration**: Configurable via `server.jwt.expiration_minutes` in settings (default: 60 minutes)
|
||||
- **Claims**: Contains `username`, `exp` (expiration), and `iat` (issued at)
|
||||
|
||||
## Using the Token
|
||||
|
||||
Include the token in subsequent requests using the `Authorization: Bearer <token>` header:
|
||||
|
||||
```bash
|
||||
# Store token in environment variable
|
||||
export TOKEN="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9..."
|
||||
|
||||
# Use in API requests
|
||||
curl http://localhost:8002/osm/api/workflows \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
### Authentication Errors
|
||||
|
||||
**401 Unauthorized** - Missing header:
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Missing authorization header"
|
||||
}
|
||||
```
|
||||
|
||||
**401 Unauthorized** - Invalid format:
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Invalid authorization header format"
|
||||
}
|
||||
```
|
||||
|
||||
**401 Unauthorized** - Expired or invalid token:
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Invalid or expired token"
|
||||
}
|
||||
```
|
||||
|
||||
## Disabling Authentication
|
||||
|
||||
Authentication can be disabled by starting the server with the `--no-auth` flag:
|
||||
|
||||
```bash
|
||||
osmedeus server --no-auth
|
||||
```
|
||||
|
||||
When disabled, all API endpoints are accessible without a token.
|
||||
@@ -0,0 +1,241 @@
|
||||
# Distributed Mode
|
||||
|
||||
These endpoints are only available when running the server in master mode.
|
||||
|
||||
## List Workers
|
||||
|
||||
Get a list of all registered workers in the distributed pool.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/workers \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "worker-001",
|
||||
"hostname": "worker1.example.com",
|
||||
"ip_address": "192.168.1.10",
|
||||
"status": "idle",
|
||||
"current_task": null,
|
||||
"joined_at": "2025-01-15T08:00:00Z",
|
||||
"last_heartbeat": "2025-01-15T10:30:00Z",
|
||||
"tasks_complete": 150,
|
||||
"tasks_failed": 2,
|
||||
"capabilities": ["docker", "nmap", "nuclei"],
|
||||
"cpu_cores": 8,
|
||||
"memory_gb": 16,
|
||||
"version": "1.0.0"
|
||||
},
|
||||
{
|
||||
"id": "worker-002",
|
||||
"hostname": "worker2.example.com",
|
||||
"ip_address": "192.168.1.11",
|
||||
"status": "busy",
|
||||
"current_task": "task-12345",
|
||||
"joined_at": "2025-01-15T08:05:00Z",
|
||||
"last_heartbeat": "2025-01-15T10:30:05Z",
|
||||
"tasks_complete": 120,
|
||||
"tasks_failed": 1,
|
||||
"capabilities": ["docker", "nmap", "nuclei", "masscan"],
|
||||
"cpu_cores": 16,
|
||||
"memory_gb": 32,
|
||||
"version": "1.0.0"
|
||||
}
|
||||
],
|
||||
"count": 2
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Get Worker
|
||||
|
||||
Get details of a specific worker.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/workers/worker-001 \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"id": "worker-001",
|
||||
"hostname": "worker1.example.com",
|
||||
"ip_address": "192.168.1.10",
|
||||
"status": "busy",
|
||||
"current_task": "task-12345",
|
||||
"joined_at": "2025-01-15T08:00:00Z",
|
||||
"last_heartbeat": "2025-01-15T10:30:00Z",
|
||||
"tasks_complete": 150,
|
||||
"tasks_failed": 2,
|
||||
"capabilities": ["docker", "nmap", "nuclei"],
|
||||
"cpu_cores": 8,
|
||||
"memory_gb": 16,
|
||||
"version": "1.0.0"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## List Tasks
|
||||
|
||||
Get a list of all running and completed tasks.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/tasks \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"running": [
|
||||
{
|
||||
"id": "task-12345",
|
||||
"scan_id": "scan-abc123",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"workflow_kind": "flow",
|
||||
"target": "example.com",
|
||||
"params": {"threads": "50", "timeout": "60"},
|
||||
"status": "running",
|
||||
"worker_id": "worker-001",
|
||||
"progress": 45,
|
||||
"current_step": "run-httpx",
|
||||
"created_at": "2025-01-15T10:00:00Z",
|
||||
"started_at": "2025-01-15T10:01:00Z"
|
||||
},
|
||||
{
|
||||
"id": "task-12346",
|
||||
"scan_id": "scan-def456",
|
||||
"workflow_name": "port-scan",
|
||||
"workflow_kind": "module",
|
||||
"target": "test.com",
|
||||
"params": {"ports": "top-1000"},
|
||||
"status": "running",
|
||||
"worker_id": "worker-002",
|
||||
"progress": 80,
|
||||
"current_step": "nmap-scan",
|
||||
"created_at": "2025-01-15T10:05:00Z",
|
||||
"started_at": "2025-01-15T10:06:00Z"
|
||||
}
|
||||
],
|
||||
"completed": [
|
||||
{
|
||||
"task_id": "task-12340",
|
||||
"scan_id": "scan-xyz789",
|
||||
"status": "completed",
|
||||
"output": "Scan completed: 150 subdomains found, 89 alive hosts",
|
||||
"error": "",
|
||||
"exports": {
|
||||
"subdomains": "/workspaces/example.com/subdomains.txt",
|
||||
"alive_hosts": "/workspaces/example.com/alive.txt"
|
||||
},
|
||||
"completed_at": "2025-01-15T09:30:00Z",
|
||||
"duration_seconds": 1800
|
||||
},
|
||||
{
|
||||
"task_id": "task-12339",
|
||||
"scan_id": "scan-uvw456",
|
||||
"status": "failed",
|
||||
"output": "",
|
||||
"error": "Connection timeout to target",
|
||||
"exports": {},
|
||||
"completed_at": "2025-01-15T09:15:00Z",
|
||||
"duration_seconds": 300
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Get Task
|
||||
|
||||
Get details of a specific task.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/tasks/task-12345 \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response (running task):**
|
||||
```json
|
||||
{
|
||||
"id": "task-12345",
|
||||
"scan_id": "scan-abc123",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"workflow_kind": "flow",
|
||||
"target": "example.com",
|
||||
"params": {"threads": "50", "timeout": "60"},
|
||||
"status": "running",
|
||||
"worker_id": "worker-001",
|
||||
"progress": 45,
|
||||
"current_step": "run-httpx",
|
||||
"created_at": "2025-01-15T10:00:00Z",
|
||||
"started_at": "2025-01-15T10:01:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
**Response (completed task):**
|
||||
```json
|
||||
{
|
||||
"task_id": "task-12345",
|
||||
"scan_id": "scan-abc123",
|
||||
"status": "completed",
|
||||
"output": "Scan completed: 150 subdomains found, 89 alive hosts",
|
||||
"error": "",
|
||||
"exports": {
|
||||
"subdomains": "/workspaces/example.com/subdomains.txt",
|
||||
"alive_hosts": "/workspaces/example.com/alive.txt",
|
||||
"httpx_json": "/workspaces/example.com/httpx.json"
|
||||
},
|
||||
"completed_at": "2025-01-15T10:30:00Z",
|
||||
"duration_seconds": 1740
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Submit Task
|
||||
|
||||
Submit a new task to the distributed worker queue.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/tasks \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workflow_name": "subdomain-enum",
|
||||
"workflow_kind": "flow",
|
||||
"target": "example.com"
|
||||
}'
|
||||
```
|
||||
|
||||
**With parameters:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/tasks \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workflow_name": "subdomain-enum",
|
||||
"workflow_kind": "flow",
|
||||
"target": "example.com",
|
||||
"params": {
|
||||
"threads": 50,
|
||||
"timeout": 30
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Task submitted",
|
||||
"task_id": "task-12346"
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,178 @@
|
||||
# Event Logs
|
||||
|
||||
## List Event Logs
|
||||
|
||||
Get a paginated list of event logs with optional filtering.
|
||||
|
||||
**List all event logs:**
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/event-logs \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**With pagination:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/event-logs?offset=0&limit=50" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Filter by workspace:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/event-logs?workspace=example.com" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Filter by topic:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/event-logs?topic=run.completed" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Filter by run ID:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/event-logs?run_id=abc12345" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Multiple filters:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/event-logs?workspace=example.com&processed=false&limit=100" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `topic` | string | Filter by event topic (e.g., "run.started", "run.completed") |
|
||||
| `name` | string | Filter by event name |
|
||||
| `source` | string | Filter by event source (e.g., "executor", "scheduler", "api") |
|
||||
| `workspace` | string | Filter by workspace name |
|
||||
| `run_id` | string | Filter by run ID |
|
||||
| `workflow_name` | string | Filter by workflow name |
|
||||
| `processed` | bool | Filter by processed status ("true" or "false") |
|
||||
| `offset` | int | Pagination offset (default: 0) |
|
||||
| `limit` | int | Maximum records to return (default: 20, max: 10000) |
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": 1,
|
||||
"topic": "run.completed",
|
||||
"event_id": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"name": "subdomain-enum-completed",
|
||||
"source": "executor",
|
||||
"data_type": "scan",
|
||||
"data": "{\"scan_id\":\"abc12345\",\"target\":\"example.com\",\"duration_ms\":3600000,\"assets_found\":150,\"steps_completed\":10}",
|
||||
"workspace": "example.com",
|
||||
"run_id": "abc12345",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"processed": true,
|
||||
"processed_at": "2025-01-15T10:30:00Z",
|
||||
"error": "",
|
||||
"created_at": "2025-01-15T09:30:00Z"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"topic": "run.started",
|
||||
"event_id": "660e8400-e29b-41d4-a716-446655440001",
|
||||
"name": "port-scan-started",
|
||||
"source": "api",
|
||||
"data_type": "scan",
|
||||
"data": "{\"scan_id\":\"def67890\",\"target\":\"test.com\",\"params\":{\"ports\":\"top-1000\"}}",
|
||||
"workspace": "test.com",
|
||||
"run_id": "def67890",
|
||||
"workflow_name": "port-scan",
|
||||
"processed": true,
|
||||
"processed_at": "2025-01-15T11:00:00Z",
|
||||
"error": "",
|
||||
"created_at": "2025-01-15T11:00:00Z"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"topic": "asset.discovered",
|
||||
"event_id": "770e8400-e29b-41d4-a716-446655440002",
|
||||
"name": "httpx-asset-found",
|
||||
"source": "executor",
|
||||
"data_type": "asset",
|
||||
"data": "{\"url\":\"https://api.example.com\",\"status_code\":200,\"title\":\"API Documentation\",\"tech\":[\"nginx\",\"nodejs\"]}",
|
||||
"workspace": "example.com",
|
||||
"run_id": "abc12345",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"processed": true,
|
||||
"processed_at": "2025-01-15T10:15:00Z",
|
||||
"error": "",
|
||||
"created_at": "2025-01-15T10:15:00Z"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"topic": "schedule.triggered",
|
||||
"event_id": "880e8400-e29b-41d4-a716-446655440003",
|
||||
"name": "daily-scan-triggered",
|
||||
"source": "scheduler",
|
||||
"data_type": "schedule",
|
||||
"data": "{\"schedule_id\":\"sch_1234567890\",\"trigger_type\":\"cron\",\"schedule\":\"0 2 * * *\"}",
|
||||
"workspace": "example.com",
|
||||
"run_id": "ghi11111",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"processed": true,
|
||||
"processed_at": "2025-01-16T02:00:00Z",
|
||||
"error": "",
|
||||
"created_at": "2025-01-16T02:00:00Z"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"topic": "run.failed",
|
||||
"event_id": "990e8400-e29b-41d4-a716-446655440004",
|
||||
"name": "nuclei-scan-failed",
|
||||
"source": "executor",
|
||||
"data_type": "scan",
|
||||
"data": "{\"scan_id\":\"jkl22222\",\"target\":\"unreachable.com\",\"error\":\"connection timeout\"}",
|
||||
"workspace": "unreachable.com",
|
||||
"run_id": "jkl22222",
|
||||
"workflow_name": "nuclei-scan",
|
||||
"processed": false,
|
||||
"processed_at": null,
|
||||
"error": "connection timeout after 5 retries",
|
||||
"created_at": "2025-01-15T14:00:00Z"
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"topic": "step.completed",
|
||||
"event_id": "aae8400-e29b-41d4-a716-446655440005",
|
||||
"name": "run-subfinder-completed",
|
||||
"source": "executor",
|
||||
"data_type": "step",
|
||||
"data": "{\"step_name\":\"run-subfinder\",\"duration_ms\":45000,\"output_lines\":150}",
|
||||
"workspace": "example.com",
|
||||
"run_id": "abc12345",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"processed": true,
|
||||
"processed_at": "2025-01-15T10:01:45Z",
|
||||
"error": "",
|
||||
"created_at": "2025-01-15T10:01:45Z"
|
||||
}
|
||||
],
|
||||
"pagination": {
|
||||
"total": 150,
|
||||
"offset": 0,
|
||||
"limit": 20
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Available Event Topics:**
|
||||
|
||||
| Topic | Description |
|
||||
|-------|-------------|
|
||||
| `run.started` | Workflow execution started |
|
||||
| `run.completed` | Workflow execution completed successfully |
|
||||
| `run.failed` | Workflow execution failed |
|
||||
| `asset.discovered` | New asset discovered during scan |
|
||||
| `asset.updated` | Existing asset information updated |
|
||||
| `webhook.received` | External webhook received |
|
||||
| `schedule.triggered` | Scheduled workflow triggered |
|
||||
| `step.completed` | Individual step completed |
|
||||
| `step.failed` | Individual step failed |
|
||||
@@ -0,0 +1,149 @@
|
||||
# Functions
|
||||
|
||||
## Execute Utility Function
|
||||
|
||||
Execute a utility function script with template rendering and JavaScript execution.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/functions/eval \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"script": "trim(\" hello \")"
|
||||
}'
|
||||
```
|
||||
|
||||
**With target variable:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/functions/eval \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"script": "fileExists(\"{{target}}\")",
|
||||
"target": "/tmp/test.txt"
|
||||
}'
|
||||
```
|
||||
|
||||
**With custom parameters:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/functions/eval \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"script": "log_info(\"{{host}}:{{port}}\")",
|
||||
"params": {
|
||||
"host": "localhost",
|
||||
"port": "8080"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
**Request Body:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `script` | string | Yes | The JavaScript script to execute |
|
||||
| `target` | string | No | Target value for `{{target}}` variable |
|
||||
| `params` | object | No | Additional parameters for template rendering |
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"result": "hello",
|
||||
"rendered_script": "trim(\" hello \")"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## List Utility Functions
|
||||
|
||||
Get a categorized list of all available utility functions.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/functions/list \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"functions": {
|
||||
"file": [
|
||||
{"name": "fileExists(path)", "description": "Check if file exists", "return_type": "bool"},
|
||||
{"name": "fileLength(path)", "description": "Count non-empty lines in file", "return_type": "int"},
|
||||
{"name": "readFile(path)", "description": "Read entire file contents", "return_type": "string"},
|
||||
{"name": "writeFile(path, content)", "description": "Write content to file", "return_type": "bool"},
|
||||
{"name": "appendFile(path, content)", "description": "Append content to file", "return_type": "bool"},
|
||||
{"name": "removeFile(path)", "description": "Delete a file", "return_type": "bool"},
|
||||
{"name": "copyFile(src, dst)", "description": "Copy file to destination", "return_type": "bool"},
|
||||
{"name": "mergeFiles(pattern, output)", "description": "Merge multiple files matching pattern", "return_type": "bool"}
|
||||
],
|
||||
"string": [
|
||||
{"name": "trim(str)", "description": "Trim whitespace", "return_type": "string"},
|
||||
{"name": "split(str, delim)", "description": "Split string by delimiter", "return_type": "[]string"},
|
||||
{"name": "replace(str, old, new)", "description": "Replace all occurrences", "return_type": "string"},
|
||||
{"name": "contains(str, substr)", "description": "Check if string contains substring", "return_type": "bool"},
|
||||
{"name": "toLowerCase(str)", "description": "Convert string to lowercase", "return_type": "string"},
|
||||
{"name": "toUpperCase(str)", "description": "Convert string to uppercase", "return_type": "string"},
|
||||
{"name": "join(array, delim)", "description": "Join array elements with delimiter", "return_type": "string"}
|
||||
],
|
||||
"utility": [
|
||||
{"name": "len(val)", "description": "Get length of string or array", "return_type": "int"},
|
||||
{"name": "exec_cmd(command)", "description": "Execute bash command and return output", "return_type": "string"},
|
||||
{"name": "isEmpty(val)", "description": "Check if value is empty/nil", "return_type": "bool"},
|
||||
{"name": "commandExists(name)", "description": "Check if command is installed", "return_type": "bool"},
|
||||
{"name": "sleep(seconds)", "description": "Sleep for specified seconds", "return_type": "void"}
|
||||
],
|
||||
"http": [
|
||||
{"name": "http_get(url)", "description": "Make HTTP GET request", "return_type": "object"},
|
||||
{"name": "http_post(url, body)", "description": "Make HTTP POST request with JSON body", "return_type": "object"},
|
||||
{"name": "httpRequest(url, method, headers, body)", "description": "Make HTTP request with full control", "return_type": "object"}
|
||||
],
|
||||
"logging": [
|
||||
{"name": "log_info(message)", "description": "Log info message with [INFO] prefix", "return_type": "void"},
|
||||
{"name": "log_debug(message)", "description": "Log debug message with [DEBUG] prefix", "return_type": "void"},
|
||||
{"name": "log_warn(message)", "description": "Log warning message with [WARN] prefix", "return_type": "void"},
|
||||
{"name": "log_error(message)", "description": "Log error message with [ERROR] prefix", "return_type": "void"}
|
||||
],
|
||||
"generation": [
|
||||
{"name": "randomString(length)", "description": "Generate random alphanumeric string", "return_type": "string"},
|
||||
{"name": "uuid()", "description": "Generate UUID v4", "return_type": "string"},
|
||||
{"name": "timestamp()", "description": "Get current Unix timestamp", "return_type": "int"}
|
||||
],
|
||||
"encoding": [
|
||||
{"name": "base64Encode(str)", "description": "Encode string to base64", "return_type": "string"},
|
||||
{"name": "base64Decode(str)", "description": "Decode base64 string", "return_type": "string"},
|
||||
{"name": "urlEncode(str)", "description": "URL encode string", "return_type": "string"},
|
||||
{"name": "urlDecode(str)", "description": "URL decode string", "return_type": "string"}
|
||||
],
|
||||
"unix_commands": [
|
||||
{"name": "sortUnix(inputFile, outputFile)", "description": "Sort file and remove duplicates", "return_type": "bool"},
|
||||
{"name": "diff_unix(file1, file2, outputFile)", "description": "Get difference between two files", "return_type": "bool"},
|
||||
{"name": "gitClone(url, destPath)", "description": "Clone git repository", "return_type": "bool"},
|
||||
{"name": "gitPull(repoPath)", "description": "Pull latest changes from git remote", "return_type": "bool"}
|
||||
],
|
||||
"database": [
|
||||
{"name": "db_insert_asset(workspace, data)", "description": "Insert asset into database", "return_type": "bool"},
|
||||
{"name": "db_query_assets(workspace, filter)", "description": "Query assets from database", "return_type": "[]object"},
|
||||
{"name": "db_update_workspace_stats(workspace)", "description": "Update workspace statistics", "return_type": "bool"}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Available Function Categories:**
|
||||
- `file` - File operations (fileExists, readFile, removeFile, etc.)
|
||||
- `string` - String manipulation (trim, split, replace, etc.)
|
||||
- `type_conversion` - Type conversions (parseInt, toString, etc.)
|
||||
- `utility` - General utilities (len, isEmpty, exec_cmd)
|
||||
- `logging` - Logging functions (log_info, log_debug)
|
||||
- `http` - HTTP requests
|
||||
- `generation` - Random values (randomString, uuid)
|
||||
- `encoding` - Base64 encode/decode
|
||||
- `notification` - Telegram notifications
|
||||
- `cdn_storage` - Cloud storage operations
|
||||
- `unix_commands` - Unix command wrappers (sortUnix, gitClone, etc.)
|
||||
- `archive` - Archive operations (zip_dir, unzip_dir)
|
||||
- `markdown` - Markdown rendering functions
|
||||
- `database` - Database operations
|
||||
@@ -0,0 +1,300 @@
|
||||
# Installation
|
||||
|
||||
## Get Registry Info
|
||||
|
||||
Fetch binary registry metadata with installation status. Supports two modes:
|
||||
- `direct-fetch` (default): Binary download URLs from registry JSON
|
||||
- `nix-build`: Nix flake binaries grouped by category
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `registry_mode` | string | `direct-fetch` | Registry mode: `direct-fetch` or `nix-build` |
|
||||
|
||||
---
|
||||
|
||||
### Direct-Fetch Mode (Default)
|
||||
|
||||
Returns binary metadata with download URLs for each platform/architecture.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/registry-info \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"registry_mode": "direct-fetch",
|
||||
"registry_url": "https://raw.githubusercontent.com/osmedeus/osmedeus-base/main/registry-metadata.json",
|
||||
"binaries": {
|
||||
"nuclei": {
|
||||
"desc": "Vulnerability scanner",
|
||||
"tags": ["vuln", "scanner"],
|
||||
"version": "3.0.0",
|
||||
"linux": {
|
||||
"amd64": "https://github.com/projectdiscovery/nuclei/releases/download/v3.0.0/nuclei_3.0.0_linux_amd64.zip",
|
||||
"arm64": "https://github.com/projectdiscovery/nuclei/releases/download/v3.0.0/nuclei_3.0.0_linux_arm64.zip"
|
||||
},
|
||||
"darwin": {
|
||||
"amd64": "https://github.com/projectdiscovery/nuclei/releases/download/v3.0.0/nuclei_3.0.0_darwin_amd64.zip",
|
||||
"arm64": "https://github.com/projectdiscovery/nuclei/releases/download/v3.0.0/nuclei_3.0.0_darwin_arm64.zip"
|
||||
},
|
||||
"installed": true,
|
||||
"path": "/usr/local/bin/nuclei"
|
||||
},
|
||||
"amass": {
|
||||
"desc": "In-depth attack surface mapping",
|
||||
"tags": ["recon", "subdomain"],
|
||||
"version": "4.0.0",
|
||||
"linux": {
|
||||
"amd64": "https://github.com/owasp-amass/amass/releases/download/v4.0.0/amass_linux_amd64.zip"
|
||||
},
|
||||
"darwin": {
|
||||
"amd64": "https://github.com/owasp-amass/amass/releases/download/v4.0.0/amass_darwin_amd64.zip"
|
||||
},
|
||||
"installed": false,
|
||||
"path": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Response Fields (direct-fetch):**
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `registry_mode` | string | Always `"direct-fetch"` |
|
||||
| `registry_url` | string | URL of the binary registry source |
|
||||
| `binaries` | object | Map of binary names to their metadata |
|
||||
| `binaries[name].desc` | string | Description of the binary tool |
|
||||
| `binaries[name].tags` | []string | Tags/categories for the binary |
|
||||
| `binaries[name].version` | string | Version of the binary |
|
||||
| `binaries[name].linux` | object | Linux download URLs by architecture (amd64, arm64) |
|
||||
| `binaries[name].darwin` | object | macOS download URLs by architecture |
|
||||
| `binaries[name].windows` | object | Windows download URLs by architecture |
|
||||
| `binaries[name].command-linux` | object | Linux install commands by architecture |
|
||||
| `binaries[name].command-darwin` | object | macOS install commands by architecture |
|
||||
| `binaries[name].installed` | boolean | Whether the binary is currently installed |
|
||||
| `binaries[name].path` | string | Full path to the installed binary |
|
||||
|
||||
---
|
||||
|
||||
### Nix-Build Mode
|
||||
|
||||
Returns Nix flake binaries grouped by category with registry metadata.
|
||||
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/registry-info?registry_mode=nix-build" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"registry_mode": "nix-build",
|
||||
"nix_installed": true,
|
||||
"categories": [
|
||||
{
|
||||
"name": "Subdomain",
|
||||
"tools": [
|
||||
{
|
||||
"name": "amass",
|
||||
"desc": "In-depth attack surface mapping and asset discovery",
|
||||
"tags": ["recon", "subdomain"],
|
||||
"version": "4.2.0",
|
||||
"repo_link": "https://github.com/owasp-amass/amass",
|
||||
"installed": true,
|
||||
"path": "/home/user/.nix-profile/bin/amass"
|
||||
},
|
||||
{
|
||||
"name": "subfinder",
|
||||
"desc": "Fast passive subdomain enumeration tool",
|
||||
"tags": ["recon", "subdomain"],
|
||||
"version": "2.6.0",
|
||||
"installed": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Vuln",
|
||||
"tools": [
|
||||
{
|
||||
"name": "nuclei",
|
||||
"desc": "Fast, customizable vulnerability scanner",
|
||||
"tags": ["vuln", "scanner"],
|
||||
"version": "3.0.0",
|
||||
"installed": true,
|
||||
"path": "/home/user/.nix-profile/bin/nuclei"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Response Fields (nix-build):**
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `registry_mode` | string | Always `"nix-build"` |
|
||||
| `nix_installed` | boolean | Whether Nix package manager is installed |
|
||||
| `categories` | array | List of tool categories from flake.nix |
|
||||
| `categories[].name` | string | Category name (e.g., "Subdomain", "Vuln") |
|
||||
| `categories[].tools` | array | List of tools in this category |
|
||||
| `categories[].tools[].name` | string | Binary name |
|
||||
| `categories[].tools[].desc` | string | Description from registry |
|
||||
| `categories[].tools[].tags` | []string | Tags from registry |
|
||||
| `categories[].tools[].version` | string | Version from registry |
|
||||
| `categories[].tools[].repo_link` | string | Repository URL |
|
||||
| `categories[].tools[].installed` | boolean | Whether the binary is installed |
|
||||
| `categories[].tools[].path` | string | Full path to installed binary |
|
||||
|
||||
---
|
||||
|
||||
## Install Binaries or Workflows
|
||||
|
||||
Install binaries from registry or workflows from git/zip URL. Supports two installation modes for binaries.
|
||||
|
||||
**Endpoint:** `POST /osm/api/registry-install`
|
||||
|
||||
**Request Body:**
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `type` | string | **Required.** Either `"binary"` or `"workflow"` |
|
||||
| `names` | []string | Binary names to install (for `type=binary`) |
|
||||
| `install_all` | bool | Install all binaries from registry (for `type=binary`) |
|
||||
| `source` | string | Git URL, zip URL, or file path (for `type=workflow`) |
|
||||
| `registry_url` | string | Custom registry URL (optional, for `type=binary`) |
|
||||
| `registry_mode` | string | `"direct-fetch"` (default) or `"nix-build"` |
|
||||
|
||||
---
|
||||
|
||||
### Install Binaries (Direct-Fetch Mode)
|
||||
|
||||
Downloads binaries directly from GitHub releases or configured URLs.
|
||||
|
||||
**Install specific binaries:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/registry-install \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"type": "binary",
|
||||
"names": ["nuclei", "httpx", "ffuf"]
|
||||
}'
|
||||
```
|
||||
|
||||
**Install all binaries from registry:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/registry-install \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"type": "binary",
|
||||
"install_all": true
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Binary installation completed",
|
||||
"registry_mode": "direct-fetch",
|
||||
"installed": ["nuclei", "httpx"],
|
||||
"installed_count": 2,
|
||||
"binaries_folder": "/home/user/osmedeus-base/binaries",
|
||||
"failed": [
|
||||
{"name": "ffuf", "error": "download failed"}
|
||||
],
|
||||
"failed_count": 1
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Install Binaries (Nix-Build Mode)
|
||||
|
||||
Installs binaries via Nix package manager using `nix profile add`.
|
||||
|
||||
**Install specific binaries via Nix:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/registry-install \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"type": "binary",
|
||||
"names": ["amass", "subfinder", "nuclei"],
|
||||
"registry_mode": "nix-build"
|
||||
}'
|
||||
```
|
||||
|
||||
**Install all Nix binaries:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/registry-install \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"type": "binary",
|
||||
"install_all": true,
|
||||
"registry_mode": "nix-build"
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Nix binary installation completed",
|
||||
"registry_mode": "nix-build",
|
||||
"installed": ["amass", "subfinder", "nuclei"],
|
||||
"installed_count": 3,
|
||||
"binaries_folder": "/home/user/osmedeus-base/binaries"
|
||||
}
|
||||
```
|
||||
|
||||
**Error (Nix not installed):**
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Nix is not installed. Install Nix first or use registry_mode=direct-fetch"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Install Workflow
|
||||
|
||||
Install a workflow from a git repository or zip archive.
|
||||
|
||||
**Install workflow from git URL:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/registry-install \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"type": "workflow",
|
||||
"source": "https://github.com/osmedeus/osmedeus-workflow.git"
|
||||
}'
|
||||
```
|
||||
|
||||
**Install workflow from zip URL:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/registry-install \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"type": "workflow",
|
||||
"source": "https://example.com/custom-workflow.zip"
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Workflow installed successfully",
|
||||
"source": "https://github.com/osmedeus/osmedeus-workflow.git",
|
||||
"workflow_folder": "/home/user/osmedeus-base/workflow"
|
||||
}
|
||||
```
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
# LLM API
|
||||
|
||||
Direct API access to Large Language Model capabilities without requiring workflow execution.
|
||||
|
||||
## Chat Completion
|
||||
|
||||
Send a chat completion request to the configured LLM provider.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/llm/v1/chat/completions \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a security analyst."},
|
||||
{"role": "user", "content": "Analyze the security posture of example.com"}
|
||||
],
|
||||
"max_tokens": 1000,
|
||||
"temperature": 0.7
|
||||
}'
|
||||
```
|
||||
|
||||
**Request Body:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `messages` | array | Yes | Array of message objects with `role` and `content` |
|
||||
| `model` | string | No | Model to use (defaults to provider's default model) |
|
||||
| `max_tokens` | int | No | Maximum tokens in response |
|
||||
| `temperature` | float | No | Sampling temperature (0.0-2.0) |
|
||||
| `top_p` | float | No | Top-p sampling parameter |
|
||||
| `top_k` | int | No | Top-k sampling parameter |
|
||||
| `n` | int | No | Number of completions to generate |
|
||||
| `stream` | bool | No | Enable streaming (not yet supported) |
|
||||
| `tools` | array | No | Tool definitions for function calling |
|
||||
| `tool_choice` | string/object | No | Tool selection strategy |
|
||||
| `response_format` | object | No | Response format (`{"type": "json_object"}`) |
|
||||
|
||||
**Message Roles:**
|
||||
- `system` - System prompt to set assistant behavior
|
||||
- `user` - User message
|
||||
- `assistant` - Previous assistant response
|
||||
- `tool` - Tool call result
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"id": "chatcmpl-abc123",
|
||||
"model": "gpt-4",
|
||||
"content": "Based on my analysis of example.com...",
|
||||
"finish_reason": "stop",
|
||||
"usage": {
|
||||
"prompt_tokens": 50,
|
||||
"completion_tokens": 200,
|
||||
"total_tokens": 250
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## With Tools (Function Calling)
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/llm/v1/chat/completions \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"messages": [
|
||||
{"role": "user", "content": "What DNS records exist for example.com?"}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "dns_lookup",
|
||||
"description": "Look up DNS records for a domain",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"domain": {"type": "string", "description": "Domain to look up"},
|
||||
"record_type": {"type": "string", "enum": ["A", "AAAA", "MX", "TXT", "NS"]}
|
||||
},
|
||||
"required": ["domain"]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"tool_choice": "auto"
|
||||
}'
|
||||
```
|
||||
|
||||
**Response with Tool Calls:**
|
||||
```json
|
||||
{
|
||||
"id": "chatcmpl-xyz789",
|
||||
"model": "gpt-4",
|
||||
"content": null,
|
||||
"finish_reason": "tool_calls",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_abc123",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "dns_lookup",
|
||||
"arguments": "{\"domain\": \"example.com\", \"record_type\": \"A\"}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 25,
|
||||
"total_tokens": 125
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Generate Embeddings
|
||||
|
||||
Generate vector embeddings for input text.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/llm/v1/embeddings \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": ["security analysis", "vulnerability assessment"],
|
||||
"model": "text-embedding-3-small"
|
||||
}'
|
||||
```
|
||||
|
||||
**Request Body:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `input` | array | Yes | Array of strings to embed |
|
||||
| `model` | string | No | Embedding model (defaults to provider's model) |
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"model": "text-embedding-3-small",
|
||||
"embeddings": [
|
||||
[0.0023, -0.0045, 0.0178, ...],
|
||||
[0.0112, -0.0067, 0.0234, ...]
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"total_tokens": 10
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,95 @@
|
||||
# Public Endpoints
|
||||
|
||||
These endpoints do not require authentication.
|
||||
|
||||
## Server Info
|
||||
|
||||
Get server version and information.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/server-info
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Oh dear me, how delightful to notice you're taking a look at this! I'm ever so pleased to let you know that osmedeus is ticking along quite nicely, thank you.",
|
||||
"version": "v5.0.0",
|
||||
"repo": "https://github.com/j3ssie/osmedeus",
|
||||
"author": "j3ssie",
|
||||
"docs": "https://docs.osmedeus.org"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Health Check
|
||||
|
||||
Check if the server is running.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/health
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"status": "ok"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Readiness Check
|
||||
|
||||
Check if the server is ready to accept requests.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/health/ready
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"status": "ready"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Swagger Documentation
|
||||
|
||||
Access the interactive Swagger UI documentation.
|
||||
|
||||
```bash
|
||||
# Open in browser
|
||||
open http://localhost:8002/swagger/index.html
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Web UI
|
||||
|
||||
The web UI is served at the root path. It uses embedded UI files by default, with an option to serve from an external path.
|
||||
|
||||
```bash
|
||||
# Access the web UI in browser
|
||||
open http://localhost:8002/
|
||||
```
|
||||
|
||||
**UI Serving Priority:**
|
||||
1. If `ui_path` is configured and exists, serves from that directory
|
||||
2. Otherwise, serves embedded UI files from `public/ui/`
|
||||
|
||||
---
|
||||
|
||||
## Workspace Files
|
||||
|
||||
Scan output files can be accessed directly via the workspace path. This endpoint is only available when `workspace_prefix` is configured in server settings.
|
||||
|
||||
```bash
|
||||
# Access run outputs (no authentication required)
|
||||
curl http://localhost:8002/ws/{workspace_prefix}/example.com/subdomain/final.txt
|
||||
```
|
||||
|
||||
The workspace path serves files from the configured workspaces directory with directory listing enabled.
|
||||
@@ -0,0 +1,374 @@
|
||||
# API Reference
|
||||
|
||||
## Error Responses
|
||||
|
||||
All endpoints return errors in a consistent format:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Error description"
|
||||
}
|
||||
```
|
||||
|
||||
**Common HTTP Status Codes:**
|
||||
- `200` - Success
|
||||
- `201` - Created
|
||||
- `202` - Accepted (async operation started)
|
||||
- `400` - Bad Request (invalid input)
|
||||
- `401` - Unauthorized (missing or invalid token)
|
||||
- `404` - Not Found
|
||||
- `500` - Internal Server Error
|
||||
|
||||
---
|
||||
|
||||
## Pagination
|
||||
|
||||
Endpoints that return lists support pagination via query parameters:
|
||||
|
||||
| Parameter | Default | Max | Description |
|
||||
|-----------|---------|-----|-------------|
|
||||
| `offset` | 0 | - | Number of records to skip |
|
||||
| `limit` | 20 | 10000 | Maximum records to return |
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/assets?offset=100&limit=50" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Cron Expression Reference
|
||||
|
||||
Schedules use standard cron expressions:
|
||||
|
||||
```
|
||||
┌───────────── minute (0-59)
|
||||
│ ┌───────────── hour (0-23)
|
||||
│ │ ┌───────────── day of month (1-31)
|
||||
│ │ │ ┌───────────── month (1-12)
|
||||
│ │ │ │ ┌───────────── day of week (0-6, Sunday=0)
|
||||
│ │ │ │ │
|
||||
* * * * *
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
- `0 2 * * *` - Every day at 2:00 AM
|
||||
- `0 0 * * 0` - Every Sunday at midnight
|
||||
- `*/30 * * * *` - Every 30 minutes
|
||||
- `0 9-17 * * 1-5` - Every hour from 9 AM to 5 PM, Monday to Friday
|
||||
|
||||
---
|
||||
|
||||
## Workflow Step Types
|
||||
|
||||
Reference documentation for workflow step types used in YAML workflow definitions.
|
||||
|
||||
### bash
|
||||
|
||||
Execute shell commands on the local system or configured runner.
|
||||
|
||||
```yaml
|
||||
- name: run-nuclei
|
||||
type: bash
|
||||
log: "Running nuclei scan"
|
||||
command: nuclei -u {{Target}} -o {{Output}}/nuclei.txt
|
||||
timeout: 3600
|
||||
exports:
|
||||
nuclei_results: "{{Output}}/nuclei.txt"
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `name` | string | Yes | Unique step name |
|
||||
| `type` | string | Yes | Must be `bash` |
|
||||
| `command` | string | No* | Single command to execute |
|
||||
| `commands` | array | No* | Sequential commands |
|
||||
| `parallel_commands` | array | No* | Commands to run in parallel |
|
||||
| `timeout` | int | No | Timeout in seconds |
|
||||
| `log` | string | No | Log message displayed during execution |
|
||||
| `pre_condition` | string | No | Condition that must be true to run |
|
||||
| `exports` | map | No | Variables to export after execution |
|
||||
|
||||
*One of `command`, `commands`, or `parallel_commands` is required.
|
||||
|
||||
---
|
||||
|
||||
### function
|
||||
|
||||
Execute utility functions written in JavaScript via Otto VM.
|
||||
|
||||
```yaml
|
||||
- name: check-file
|
||||
type: function
|
||||
log: "Checking if results exist"
|
||||
function: fileExists("{{Output}}/results.txt")
|
||||
exports:
|
||||
has_results: "{{check_file_output}}"
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `name` | string | Yes | Unique step name |
|
||||
| `type` | string | Yes | Must be `function` |
|
||||
| `function` | string | No* | Single function to execute |
|
||||
| `functions` | array | No* | Sequential functions |
|
||||
| `parallel_functions` | array | No* | Functions to run in parallel |
|
||||
|
||||
*One of `function`, `functions`, or `parallel_functions` is required.
|
||||
|
||||
---
|
||||
|
||||
### parallel-steps
|
||||
|
||||
Run multiple steps concurrently.
|
||||
|
||||
```yaml
|
||||
- name: parallel-scans
|
||||
type: parallel-steps
|
||||
log: "Running scans in parallel"
|
||||
parallel_steps:
|
||||
- name: nuclei-scan
|
||||
type: bash
|
||||
command: nuclei -u {{Target}}
|
||||
- name: httpx-scan
|
||||
type: bash
|
||||
command: httpx -u {{Target}}
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `name` | string | Yes | Unique step name |
|
||||
| `type` | string | Yes | Must be `parallel-steps` |
|
||||
| `parallel_steps` | array | Yes | Array of steps to run concurrently |
|
||||
|
||||
---
|
||||
|
||||
### foreach
|
||||
|
||||
Iterate over items from a file or array.
|
||||
|
||||
```yaml
|
||||
- name: scan-subdomains
|
||||
type: foreach
|
||||
log: "Scanning each subdomain"
|
||||
input: "{{Output}}/subdomains.txt"
|
||||
variable: subdomain
|
||||
step:
|
||||
name: scan-subdomain
|
||||
type: bash
|
||||
command: httpx -u [[subdomain]]
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `name` | string | Yes | Unique step name |
|
||||
| `type` | string | Yes | Must be `foreach` |
|
||||
| `input` | string | Yes | File path or array to iterate over |
|
||||
| `variable` | string | Yes | Loop variable name (use `[[variable]]`) |
|
||||
| `step` | object | Yes | Step to execute for each item |
|
||||
| `parallel` | int | No | Number of parallel iterations |
|
||||
|
||||
---
|
||||
|
||||
### remote-bash
|
||||
|
||||
Execute commands in Docker containers or via SSH.
|
||||
|
||||
```yaml
|
||||
- name: docker-scan
|
||||
type: remote-bash
|
||||
log: "Running scan in Docker"
|
||||
step_runner: docker
|
||||
step_runner_config:
|
||||
image: "projectdiscovery/nuclei:latest"
|
||||
volumes:
|
||||
- "{{Output}}:/output"
|
||||
command: nuclei -u {{Target}} -o /output/nuclei.txt
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `name` | string | Yes | Unique step name |
|
||||
| `type` | string | Yes | Must be `remote-bash` |
|
||||
| `step_runner` | string | Yes | `docker` or `ssh` |
|
||||
| `step_runner_config` | object | No | Runner-specific configuration |
|
||||
| `command` | string | No* | Command to execute |
|
||||
| `commands` | array | No* | Sequential commands |
|
||||
|
||||
**Docker Configuration:**
|
||||
```yaml
|
||||
step_runner_config:
|
||||
image: "image:tag"
|
||||
volumes: ["host:container"]
|
||||
env:
|
||||
KEY: value
|
||||
network: "host"
|
||||
workdir: "/app"
|
||||
```
|
||||
|
||||
**SSH Configuration:**
|
||||
```yaml
|
||||
step_runner_config:
|
||||
host: "worker.example.com"
|
||||
user: "ubuntu"
|
||||
key_file: "~/.ssh/id_rsa"
|
||||
port: 22
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### http
|
||||
|
||||
Make HTTP requests and capture responses.
|
||||
|
||||
```yaml
|
||||
- name: api-request
|
||||
type: http
|
||||
log: "Calling API"
|
||||
url: "https://api.example.com/endpoint"
|
||||
method: POST
|
||||
headers:
|
||||
Content-Type: "application/json"
|
||||
Authorization: "Bearer {{api_token}}"
|
||||
request_body: '{"domain": "{{Target}}"}'
|
||||
timeout: 30
|
||||
exports:
|
||||
api_response: "{{api_request_body}}"
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `name` | string | Yes | Unique step name |
|
||||
| `type` | string | Yes | Must be `http` |
|
||||
| `url` | string | Yes | Request URL |
|
||||
| `method` | string | No | HTTP method (default: GET) |
|
||||
| `headers` | map | No | Request headers |
|
||||
| `request_body` | string | No | Request body for POST/PUT |
|
||||
| `timeout` | int | No | Timeout in seconds |
|
||||
|
||||
**Auto-Exports:**
|
||||
- `<step_name>_status_code` - HTTP status code
|
||||
- `<step_name>_body` - Response body
|
||||
- `<step_name>_headers` - Response headers
|
||||
|
||||
---
|
||||
|
||||
### llm
|
||||
|
||||
Execute LLM (Large Language Model) API calls for AI-powered analysis.
|
||||
|
||||
```yaml
|
||||
- name: analyze-target
|
||||
type: llm
|
||||
log: "Analyzing target with LLM"
|
||||
messages:
|
||||
- role: system
|
||||
content: "You are a security analyst."
|
||||
- role: user
|
||||
content: "Analyze the security of {{Target}}"
|
||||
llm_config:
|
||||
max_tokens: 1000
|
||||
temperature: 0.7
|
||||
timeout: 60
|
||||
exports:
|
||||
analysis: "{{analyze_target_content}}"
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `name` | string | Yes | Unique step name |
|
||||
| `type` | string | Yes | Must be `llm` |
|
||||
| `messages` | array | No* | Chat messages (role + content) |
|
||||
| `is_embedding` | bool | No | Set to true for embeddings |
|
||||
| `embedding_input` | array | No* | Strings to embed |
|
||||
| `llm_config` | object | No | Step-level LLM configuration |
|
||||
| `tools` | array | No | Tool definitions for function calling |
|
||||
| `tool_choice` | string | No | Tool selection (`auto`, `none`, etc.) |
|
||||
| `extra_llm_parameters` | map | No | Additional provider-specific params |
|
||||
| `timeout` | int | No | Timeout in seconds |
|
||||
|
||||
*Either `messages` or `embedding_input` (with `is_embedding: true`) is required.
|
||||
|
||||
**Message Format:**
|
||||
```yaml
|
||||
messages:
|
||||
- role: system
|
||||
content: "System prompt"
|
||||
- role: user
|
||||
content: "User message with {{variables}}"
|
||||
```
|
||||
|
||||
**Multimodal Messages (with images):**
|
||||
```yaml
|
||||
messages:
|
||||
- role: user
|
||||
content:
|
||||
- type: text
|
||||
text: "What do you see in this screenshot?"
|
||||
- type: image_url
|
||||
image_url:
|
||||
url: "data:image/png;base64,{{screenshot_base64}}"
|
||||
```
|
||||
|
||||
**LLM Configuration Override:**
|
||||
```yaml
|
||||
llm_config:
|
||||
model: "gpt-4"
|
||||
max_tokens: 2000
|
||||
temperature: 0.3
|
||||
response_format:
|
||||
type: json_object
|
||||
```
|
||||
|
||||
**Embeddings:**
|
||||
```yaml
|
||||
- name: generate-embeddings
|
||||
type: llm
|
||||
is_embedding: true
|
||||
embedding_input:
|
||||
- "{{Target}} security analysis"
|
||||
- "vulnerability assessment"
|
||||
exports:
|
||||
embeddings: "{{generate_embeddings_llm_resp}}"
|
||||
```
|
||||
|
||||
**Tool Calling:**
|
||||
```yaml
|
||||
- name: with-tools
|
||||
type: llm
|
||||
messages:
|
||||
- role: user
|
||||
content: "What DNS records exist for {{Target}}?"
|
||||
tools:
|
||||
- type: function
|
||||
function:
|
||||
name: dns_lookup
|
||||
description: "Look up DNS records"
|
||||
parameters:
|
||||
type: object
|
||||
properties:
|
||||
domain:
|
||||
type: string
|
||||
required: [domain]
|
||||
tool_choice: auto
|
||||
```
|
||||
|
||||
**Auto-Exports:**
|
||||
- `<step_name>_llm_resp` - Full response object (id, model, usage, content, tool_calls)
|
||||
- `<step_name>_content` - Just the content string for easy access
|
||||
@@ -0,0 +1,352 @@
|
||||
# Runs (Scans)
|
||||
|
||||
## Create a New Scan
|
||||
|
||||
Execute a workflow against a target.
|
||||
|
||||
**Basic scan with flow workflow:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/runs \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"flow": "subdomain-enum",
|
||||
"target": "example.com"
|
||||
}'
|
||||
```
|
||||
|
||||
**Basic scan with module workflow:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/runs \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"module": "port-scan",
|
||||
"target": "example.com"
|
||||
}'
|
||||
```
|
||||
|
||||
**Scan with custom parameters:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/runs \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"flow": "subdomain-enum",
|
||||
"target": "example.com",
|
||||
"params": {
|
||||
"threads": "50",
|
||||
"timeout": "30"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
**Scan with priority and timeout:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/runs \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"flow": "subdomain-enum",
|
||||
"target": "example.com",
|
||||
"priority": "high",
|
||||
"timeout": 60
|
||||
}'
|
||||
```
|
||||
|
||||
**Scan with Docker runner:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/runs \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"flow": "subdomain-enum",
|
||||
"target": "example.com",
|
||||
"runner_type": "docker",
|
||||
"docker_image": "osmedeus/osmedeus:latest"
|
||||
}'
|
||||
```
|
||||
|
||||
**Scan with SSH runner:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/runs \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"flow": "subdomain-enum",
|
||||
"target": "example.com",
|
||||
"runner_type": "ssh",
|
||||
"ssh_host": "worker1.example.com"
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Run started",
|
||||
"workflow": "subdomain-enum",
|
||||
"kind": "flow",
|
||||
"target": "example.com",
|
||||
"target_count": 1,
|
||||
"priority": "high",
|
||||
"runner_type": "docker",
|
||||
"timeout": 60
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Multi-Target Scanning
|
||||
|
||||
Scan multiple targets with concurrency control:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/runs \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"flow": "subdomain-enum",
|
||||
"targets": ["example.com", "test.com", "demo.com"],
|
||||
"concurrency": 3
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Scan started",
|
||||
"workflow": "subdomain-enum",
|
||||
"kind": "flow",
|
||||
"target_count": 3,
|
||||
"targets": ["example.com", "test.com", "demo.com"],
|
||||
"concurrency": 3,
|
||||
"priority": "medium"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Scan from Uploaded Target File
|
||||
|
||||
Use an uploaded target file (from `/osm/api/upload-file`) for running:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/runs \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"module": "port-scan",
|
||||
"target_file": "/home/user/osmedeus-base/data/uploads/targets.txt",
|
||||
"concurrency": 5
|
||||
}'
|
||||
```
|
||||
|
||||
This is similar to CLI's `-T` flag: `osmedeus run -m port-scan -T targets.txt`
|
||||
|
||||
---
|
||||
|
||||
## List Runs
|
||||
|
||||
Get a paginated list of all runs.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/runs \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `status` | string | - | Filter by status: `pending`, `running`, `completed`, `failed` |
|
||||
| `workflow_name` | string | - | Filter by workflow name |
|
||||
| `target` | string | - | Filter by target |
|
||||
| `offset` | int | 0 | Pagination offset |
|
||||
| `limit` | int | 20 | Maximum records to return |
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "run-abc123",
|
||||
"run_id": "run-2025-01-15-subdomain-enum-example.com",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"workflow_kind": "flow",
|
||||
"target": "example.com",
|
||||
"params": {"threads": "50"},
|
||||
"status": "running",
|
||||
"workspace_path": "/home/user/osmedeus-base/workspaces/example.com",
|
||||
"started_at": "2025-01-15T10:00:00Z",
|
||||
"completed_at": null,
|
||||
"total_steps": 10,
|
||||
"completed_steps": 3,
|
||||
"trigger_type": "manual",
|
||||
"created_at": "2025-01-15T10:00:00Z",
|
||||
"updated_at": "2025-01-15T10:03:00Z"
|
||||
}
|
||||
],
|
||||
"pagination": {
|
||||
"total": 50,
|
||||
"offset": 0,
|
||||
"limit": 20
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Get Run Details
|
||||
|
||||
Get details of a specific run by ID.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/runs/run-abc123 \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"id": "run-abc123",
|
||||
"run_id": "run-2025-01-15-subdomain-enum-example.com",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"workflow_kind": "flow",
|
||||
"target": "example.com",
|
||||
"params": {"threads": "50"},
|
||||
"status": "completed",
|
||||
"workspace_path": "/home/user/osmedeus-base/workspaces/example.com",
|
||||
"started_at": "2025-01-15T10:00:00Z",
|
||||
"completed_at": "2025-01-15T10:30:00Z",
|
||||
"error_message": "",
|
||||
"schedule_id": "",
|
||||
"trigger_type": "manual",
|
||||
"trigger_name": "",
|
||||
"total_steps": 10,
|
||||
"completed_steps": 10,
|
||||
"created_at": "2025-01-15T10:00:00Z",
|
||||
"updated_at": "2025-01-15T10:30:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Cancel Run
|
||||
|
||||
Cancel a running workflow execution.
|
||||
|
||||
```bash
|
||||
curl -X DELETE http://localhost:8002/osm/api/runs/run-abc123 \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Run cancellation requested",
|
||||
"id": "run-abc123"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Get Run Steps
|
||||
|
||||
Get all step results for a specific run.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/runs/run-abc123/steps \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "step-xyz789",
|
||||
"run_id": "run-abc123",
|
||||
"step_name": "run-subfinder",
|
||||
"step_type": "bash",
|
||||
"status": "completed",
|
||||
"command": "subfinder -d example.com -o subdomains.txt",
|
||||
"output": "Found 150 subdomains",
|
||||
"error_message": "",
|
||||
"exports": {"subdomains_file": "subdomains.txt"},
|
||||
"duration_ms": 45000,
|
||||
"log_file": "/workspaces/example.com/logs/run-subfinder.log",
|
||||
"started_at": "2025-01-15T10:01:00Z",
|
||||
"completed_at": "2025-01-15T10:01:45Z",
|
||||
"created_at": "2025-01-15T10:01:00Z"
|
||||
},
|
||||
{
|
||||
"id": "step-def456",
|
||||
"run_id": "run-abc123",
|
||||
"step_name": "run-httpx",
|
||||
"step_type": "bash",
|
||||
"status": "completed",
|
||||
"command": "httpx -l subdomains.txt -o alive.txt",
|
||||
"output": "Probed 150 hosts, 89 alive",
|
||||
"error_message": "",
|
||||
"exports": {"alive_file": "alive.txt"},
|
||||
"duration_ms": 120000,
|
||||
"log_file": "/workspaces/example.com/logs/run-httpx.log",
|
||||
"started_at": "2025-01-15T10:01:45Z",
|
||||
"completed_at": "2025-01-15T10:03:45Z",
|
||||
"created_at": "2025-01-15T10:01:45Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Get Run Artifacts
|
||||
|
||||
Get all output artifacts for a specific run.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/runs/run-abc123/artifacts \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "artifact-001",
|
||||
"run_id": "run-abc123",
|
||||
"name": "subdomains.txt",
|
||||
"path": "/workspaces/example.com/subdomains.txt",
|
||||
"type": "text",
|
||||
"size_bytes": 4523,
|
||||
"line_count": 150,
|
||||
"description": "Discovered subdomains",
|
||||
"created_at": "2025-01-15T10:01:45Z"
|
||||
},
|
||||
{
|
||||
"id": "artifact-002",
|
||||
"run_id": "run-abc123",
|
||||
"name": "alive.txt",
|
||||
"path": "/workspaces/example.com/alive.txt",
|
||||
"type": "text",
|
||||
"size_bytes": 2890,
|
||||
"line_count": 89,
|
||||
"description": "Alive HTTP endpoints",
|
||||
"created_at": "2025-01-15T10:03:45Z"
|
||||
},
|
||||
{
|
||||
"id": "artifact-003",
|
||||
"run_id": "run-abc123",
|
||||
"name": "nuclei-results.json",
|
||||
"path": "/workspaces/example.com/nuclei-results.json",
|
||||
"type": "json",
|
||||
"size_bytes": 15234,
|
||||
"line_count": 45,
|
||||
"description": "Nuclei vulnerability scan results",
|
||||
"created_at": "2025-01-15T10:15:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,273 @@
|
||||
# Schedules
|
||||
|
||||
## List Schedules
|
||||
|
||||
Get a paginated list of all scheduled workflows.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/schedules \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**With pagination:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/schedules?offset=0&limit=50" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "sch_1234567890",
|
||||
"name": "daily-scan",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"workflow_path": "/home/user/osmedeus-base/workflows/flows/subdomain-enum.yaml",
|
||||
"trigger_name": "daily-scan-trigger",
|
||||
"trigger_type": "cron",
|
||||
"schedule": "0 2 * * *",
|
||||
"event_topic": "",
|
||||
"watch_path": "",
|
||||
"input_config": {
|
||||
"target": "example.com",
|
||||
"threads": "50"
|
||||
},
|
||||
"is_enabled": true,
|
||||
"last_run": "2025-01-15T02:00:00Z",
|
||||
"next_run": "2025-01-16T02:00:00Z",
|
||||
"run_count": 30,
|
||||
"created_at": "2025-01-01T00:00:00Z",
|
||||
"updated_at": "2025-01-15T02:00:00Z"
|
||||
},
|
||||
{
|
||||
"id": "sch_0987654321",
|
||||
"name": "weekly-full-recon",
|
||||
"workflow_name": "full-recon",
|
||||
"workflow_path": "/home/user/osmedeus-base/workflows/flows/full-recon.yaml",
|
||||
"trigger_name": "weekly-trigger",
|
||||
"trigger_type": "cron",
|
||||
"schedule": "0 0 * * 0",
|
||||
"event_topic": "",
|
||||
"watch_path": "",
|
||||
"input_config": {
|
||||
"target": "example.com",
|
||||
"threads": "100",
|
||||
"runner_type": "docker"
|
||||
},
|
||||
"is_enabled": true,
|
||||
"last_run": "2025-01-12T00:00:00Z",
|
||||
"next_run": "2025-01-19T00:00:00Z",
|
||||
"run_count": 5,
|
||||
"created_at": "2025-01-01T00:00:00Z",
|
||||
"updated_at": "2025-01-12T00:00:00Z"
|
||||
}
|
||||
],
|
||||
"pagination": {
|
||||
"total": 5,
|
||||
"offset": 0,
|
||||
"limit": 20
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Create Schedule
|
||||
|
||||
Create a new scheduled workflow execution.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/schedules \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "daily-scan",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"workflow_kind": "flow",
|
||||
"target": "example.com",
|
||||
"schedule": "0 2 * * *",
|
||||
"enabled": true
|
||||
}'
|
||||
```
|
||||
|
||||
**With additional parameters:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/schedules \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "weekly-full-scan",
|
||||
"workflow_name": "full-recon",
|
||||
"workflow_kind": "flow",
|
||||
"target": "example.com",
|
||||
"schedule": "0 0 * * 0",
|
||||
"enabled": true,
|
||||
"params": {
|
||||
"threads": "100"
|
||||
},
|
||||
"runner_type": "docker"
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Schedule created",
|
||||
"data": {
|
||||
"id": "sch_1234567890",
|
||||
"name": "daily-scan",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"schedule": "0 2 * * *",
|
||||
"is_enabled": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Get Schedule
|
||||
|
||||
Get details of a specific schedule.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/schedules/sch_1234567890 \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"id": "sch_1234567890",
|
||||
"name": "daily-scan",
|
||||
"workflow_name": "subdomain-enum",
|
||||
"workflow_path": "/home/user/osmedeus-base/workflows/flows/subdomain-enum.yaml",
|
||||
"trigger_name": "daily-scan-trigger",
|
||||
"trigger_type": "cron",
|
||||
"schedule": "0 2 * * *",
|
||||
"event_topic": "",
|
||||
"watch_path": "",
|
||||
"input_config": {
|
||||
"target": "example.com",
|
||||
"threads": "50"
|
||||
},
|
||||
"is_enabled": true,
|
||||
"last_run": "2025-01-15T02:00:00Z",
|
||||
"next_run": "2025-01-16T02:00:00Z",
|
||||
"run_count": 30,
|
||||
"created_at": "2025-01-01T00:00:00Z",
|
||||
"updated_at": "2025-01-15T02:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Update Schedule
|
||||
|
||||
Update an existing schedule.
|
||||
|
||||
```bash
|
||||
curl -X PUT http://localhost:8002/osm/api/schedules/sch_1234567890 \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "updated-daily-scan",
|
||||
"schedule": "0 3 * * *"
|
||||
}'
|
||||
```
|
||||
|
||||
**Update only the schedule:**
|
||||
```bash
|
||||
curl -X PUT http://localhost:8002/osm/api/schedules/sch_1234567890 \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"schedule": "0 4 * * *"
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Schedule updated",
|
||||
"data": {
|
||||
"id": "sch_1234567890",
|
||||
"name": "updated-daily-scan",
|
||||
"schedule": "0 3 * * *"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Delete Schedule
|
||||
|
||||
Delete a schedule.
|
||||
|
||||
```bash
|
||||
curl -X DELETE http://localhost:8002/osm/api/schedules/sch_1234567890 \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Schedule deleted"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Enable Schedule
|
||||
|
||||
Enable a disabled schedule.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/schedules/sch_1234567890/enable \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Schedule enabled"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Disable Schedule
|
||||
|
||||
Disable an enabled schedule.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/schedules/sch_1234567890/disable \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Schedule disabled"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Trigger Schedule
|
||||
|
||||
Manually trigger a scheduled workflow execution.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/schedules/sch_1234567890/trigger \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Schedule triggered",
|
||||
"schedule": "daily-scan",
|
||||
"workflow": "subdomain-enum"
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,72 @@
|
||||
# Settings
|
||||
|
||||
Manage server configuration settings.
|
||||
|
||||
## Get YAML Configuration
|
||||
|
||||
Get the entire YAML configuration file with sensitive fields redacted. Fields containing `_key`, `secret`, `password`, `username`, or `_token` are replaced with `[REDACTED]`.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/settings/yaml \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:** (text/yaml)
|
||||
```yaml
|
||||
# =============================================================================
|
||||
# Osmedeus Configuration File
|
||||
# =============================================================================
|
||||
base_folder: ~/osmedeus-base
|
||||
|
||||
environments:
|
||||
binaries_path: "{{base_folder}}/binaries"
|
||||
# ... more config
|
||||
|
||||
server:
|
||||
host: "0.0.0.0"
|
||||
port: 8002
|
||||
workspace_prefix_key: "[REDACTED]"
|
||||
simple_user_map_key: "[REDACTED]"
|
||||
jwt:
|
||||
secret_signing_key: "[REDACTED]"
|
||||
expiration_minutes: 180
|
||||
|
||||
database:
|
||||
host: ""
|
||||
port: 5432
|
||||
username: "[REDACTED]"
|
||||
password: "[REDACTED]"
|
||||
# ... more config
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Update YAML Configuration
|
||||
|
||||
Replace the entire YAML configuration file with new content. A backup of the existing configuration is created before overwriting.
|
||||
|
||||
```bash
|
||||
curl -X PUT http://localhost:8002/osm/api/settings/yaml \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: text/yaml" \
|
||||
--data-binary @new-config.yaml
|
||||
```
|
||||
|
||||
**Request Body:** Raw YAML configuration content
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Configuration updated successfully",
|
||||
"path": "/home/user/osmedeus-base/osm-settings.yaml",
|
||||
"backup": "/home/user/osmedeus-base/osm-settings.yaml.backup"
|
||||
}
|
||||
```
|
||||
|
||||
**Error Response (Invalid YAML):**
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Invalid YAML configuration: yaml: unmarshal errors: ..."
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,206 @@
|
||||
# Snapshots
|
||||
|
||||
## List Snapshots
|
||||
|
||||
Get a list of available snapshot files in the snapshot directory.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/snapshots \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"name": "example.com_1704067200.zip",
|
||||
"path": "/home/user/osmedeus-base/snapshot/example.com_1704067200.zip",
|
||||
"size": 15728640,
|
||||
"created_at": "2025-01-01T12:00:00Z"
|
||||
},
|
||||
{
|
||||
"name": "test.com_1704153600.zip",
|
||||
"path": "/home/user/osmedeus-base/snapshot/test.com_1704153600.zip",
|
||||
"size": 8388608,
|
||||
"created_at": "2025-01-02T12:00:00Z"
|
||||
}
|
||||
],
|
||||
"count": 2,
|
||||
"path": "/home/user/osmedeus-base/snapshot"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Export Workspace Snapshot
|
||||
|
||||
Export a workspace to a compressed zip archive and download it.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/snapshots/export \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"workspace": "example.com"}' \
|
||||
--output example.com_snapshot.zip
|
||||
```
|
||||
|
||||
**Request Body:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `workspace` | string | Yes | Name of the workspace to export |
|
||||
|
||||
**Response:**
|
||||
- On success: Returns the zip file as a binary download
|
||||
- Response headers include:
|
||||
- `Content-Disposition: attachment; filename=<workspace>_<timestamp>.zip`
|
||||
- `Content-Type: application/zip`
|
||||
- `X-Snapshot-Size: <size_in_bytes>`
|
||||
|
||||
**Error Response (404):**
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Workspace not found: example.com"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Import Workspace Snapshot
|
||||
|
||||
Import a workspace from an uploaded zip file or URL.
|
||||
|
||||
**Import from file upload:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/snapshots/import \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-F "file=@example.com_1704067200.zip"
|
||||
```
|
||||
|
||||
**Import from URL:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/snapshots/import \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-F "url=https://example.com/snapshots/workspace.zip"
|
||||
```
|
||||
|
||||
**Import with force overwrite:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/snapshots/import \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-F "file=@example.com_snapshot.zip" \
|
||||
-F "force=true"
|
||||
```
|
||||
|
||||
**Import files only (skip database):**
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/snapshots/import \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-F "file=@example.com_snapshot.zip" \
|
||||
-F "skip_db=true"
|
||||
```
|
||||
|
||||
**Form Parameters:**
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|------|----------|-------------|
|
||||
| `file` | file | No* | Snapshot zip file to import |
|
||||
| `url` | string | No* | URL of snapshot to download and import |
|
||||
| `force` | bool | No | Overwrite existing workspace if present (default: false) |
|
||||
| `skip_db` | bool | No | Skip database import, extract files only (default: false) |
|
||||
|
||||
*Either `file` or `url` is required.
|
||||
|
||||
**Response (200):**
|
||||
```json
|
||||
{
|
||||
"message": "Workspace imported successfully",
|
||||
"workspace": "example.com",
|
||||
"local_path": "/home/user/workspaces-osmedeus/example.com",
|
||||
"data_source": "imported",
|
||||
"files_count": 1523,
|
||||
"warning": "Imported workspace database state may be unstable. Only import from trusted sources."
|
||||
}
|
||||
```
|
||||
|
||||
**Error Response (400):**
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Either file or url is required"
|
||||
}
|
||||
```
|
||||
|
||||
**Error Response (500 - workspace exists):**
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Failed to import snapshot: workspace already exists: /home/user/workspaces-osmedeus/example.com (use --force to overwrite)"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Delete Snapshot
|
||||
|
||||
Delete a snapshot file by name.
|
||||
|
||||
```bash
|
||||
curl -X DELETE http://localhost:8002/osm/api/snapshots/example.com_1704067200.zip \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Snapshot deleted successfully",
|
||||
"name": "example.com_1704067200.zip"
|
||||
}
|
||||
```
|
||||
|
||||
**Error Response (404):**
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Snapshot not found: example.com_1704067200.zip"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Legacy Endpoint
|
||||
|
||||
The legacy snapshot download endpoint is still available for backward compatibility:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/snapshot-download/example.com \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
--output snapshot.zip
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Data Source Values
|
||||
|
||||
When a workspace is imported, its `data_source` field is set to indicate how it was created:
|
||||
|
||||
| Value | Description |
|
||||
|-------|-------------|
|
||||
| `local` | Created locally via scan (default) |
|
||||
| `cloud` | Synced from cloud storage |
|
||||
| `imported` | Imported from snapshot file |
|
||||
|
||||
---
|
||||
|
||||
## Security Considerations
|
||||
|
||||
**Warning:** Only import snapshots from trusted sources!
|
||||
|
||||
Imported workspace data may contain:
|
||||
- Database records that could conflict with existing data
|
||||
- File paths that reference external resources
|
||||
- Configuration that may not be compatible
|
||||
|
||||
The imported workspace database state may be unstable. Use the `skip_db=true` parameter if you only need the files without database import.
|
||||
@@ -0,0 +1,67 @@
|
||||
# System Statistics
|
||||
|
||||
## Get System Stats
|
||||
|
||||
Get aggregated system statistics including workflows, runs, workspaces, assets, vulnerabilities, and schedules.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/stats \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"workflows": {
|
||||
"total": 25,
|
||||
"flows": 10,
|
||||
"modules": 15
|
||||
},
|
||||
"runs": {
|
||||
"total": 150,
|
||||
"completed": 120,
|
||||
"running": 5,
|
||||
"failed": 10,
|
||||
"pending": 15
|
||||
},
|
||||
"workspaces": {
|
||||
"total": 50
|
||||
},
|
||||
"assets": {
|
||||
"total": 5000
|
||||
},
|
||||
"vulnerabilities": {
|
||||
"total": 150,
|
||||
"critical": 10,
|
||||
"high": 25,
|
||||
"medium": 50,
|
||||
"low": 65
|
||||
},
|
||||
"schedules": {
|
||||
"total": 8,
|
||||
"enabled": 5
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Statistics Fields:**
|
||||
|
||||
| Category | Field | Description |
|
||||
|----------|-------|-------------|
|
||||
| workflows.total | int | Total number of workflows (flows + modules) |
|
||||
| workflows.flows | int | Number of flow-type workflows |
|
||||
| workflows.modules | int | Number of module-type workflows |
|
||||
| runs.total | int | Total number of runs |
|
||||
| runs.completed | int | Successfully completed runs |
|
||||
| runs.running | int | Currently running workflows |
|
||||
| runs.failed | int | Failed runs |
|
||||
| runs.pending | int | Pending runs waiting to start |
|
||||
| workspaces.total | int | Total number of scan workspaces |
|
||||
| assets.total | int | Total discovered assets across all workspaces |
|
||||
| vulnerabilities.total | int | Total vulnerabilities (sum of all severities) |
|
||||
| vulnerabilities.critical | int | Critical severity vulnerabilities |
|
||||
| vulnerabilities.high | int | High severity vulnerabilities |
|
||||
| vulnerabilities.medium | int | Medium severity vulnerabilities |
|
||||
| vulnerabilities.low | int | Low severity vulnerabilities |
|
||||
| schedules.total | int | Total configured schedules |
|
||||
| schedules.enabled | int | Currently enabled schedules |
|
||||
@@ -0,0 +1,49 @@
|
||||
# File Uploads
|
||||
|
||||
## Upload Input File
|
||||
|
||||
Upload a file containing a list of inputs (targets, URLs, etc.) for later use in runs.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/upload-file \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-F "file=@targets.txt"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "File uploaded",
|
||||
"filename": "1704326400000000000_targets.txt",
|
||||
"path": "/home/user/osmedeus-base/data/uploads/1704326400000000000_targets.txt",
|
||||
"size": 1024,
|
||||
"lines": 50
|
||||
}
|
||||
```
|
||||
|
||||
The returned `path` can be used as a target in subsequent run requests.
|
||||
|
||||
---
|
||||
|
||||
## Upload Workflow
|
||||
|
||||
Upload a raw YAML workflow file and save it to the workflows directory.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/workflow-upload \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-F "file=@my-custom-workflow.yaml"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Workflow uploaded",
|
||||
"name": "my-custom-workflow",
|
||||
"kind": "module",
|
||||
"description": "A custom security workflow",
|
||||
"path": "/home/user/osmedeus-base/workflows/modules/my-custom-workflow.yaml"
|
||||
}
|
||||
```
|
||||
|
||||
The workflow file must be a valid YAML with `.yaml` or `.yml` extension. It will be saved to either the `flows/` or `modules/` subdirectory based on the workflow kind.
|
||||
@@ -0,0 +1,317 @@
|
||||
# Vulnerabilities
|
||||
|
||||
## List Vulnerabilities
|
||||
|
||||
Get a paginated list of vulnerabilities with optional filtering by workspace, severity, confidence, or asset value.
|
||||
|
||||
**List all vulnerabilities:**
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/vulnerabilities \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**List vulnerabilities with pagination:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/vulnerabilities?offset=0&limit=100" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Filter by workspace:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/vulnerabilities?workspace=example.com" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Filter by severity:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/vulnerabilities?severity=critical" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Filter by confidence:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/vulnerabilities?confidence=Certain" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Filter by asset value (partial match):**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/vulnerabilities?asset_value=api.example" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Combine filters:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/vulnerabilities?workspace=example.com&severity=high&offset=0&limit=50" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `workspace` | string | - | Filter by workspace name |
|
||||
| `severity` | string | - | Filter by severity (critical, high, medium, low, info) |
|
||||
| `confidence` | string | - | Filter by confidence (Certain, Firm, Tentative, Manual Review Required) |
|
||||
| `asset_value` | string | - | Filter by asset value (partial match) |
|
||||
| `offset` | int | 0 | Number of records to skip |
|
||||
| `limit` | int | 20 | Maximum records to return (max 10000) |
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": 1,
|
||||
"workspace": "example.com",
|
||||
"vuln_info": "CVE-2024-1234",
|
||||
"vuln_title": "SQL Injection in Login Form",
|
||||
"vuln_desc": "The login form is vulnerable to SQL injection via the username parameter.",
|
||||
"vuln_poc": "username=' OR '1'='1' --&password=test",
|
||||
"severity": "critical",
|
||||
"confidence": "Certain",
|
||||
"asset_type": "web",
|
||||
"asset_value": "https://example.com/login",
|
||||
"tags": ["sqli", "owasp-top10", "authentication"],
|
||||
"detail_http_request": "POST /login HTTP/1.1\nHost: example.com\n...",
|
||||
"detail_http_response": "HTTP/1.1 200 OK\n...",
|
||||
"raw_vuln_json": "{\"template\":\"sqli-login.yaml\",...}",
|
||||
"created_at": "2025-01-15T10:30:00Z",
|
||||
"updated_at": "2025-01-15T10:30:00Z"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"workspace": "example.com",
|
||||
"vuln_info": "CVE-2024-5678",
|
||||
"vuln_title": "Cross-Site Scripting (XSS) in Search",
|
||||
"vuln_desc": "Reflected XSS vulnerability in the search functionality.",
|
||||
"vuln_poc": "<script>alert('XSS')</script>",
|
||||
"severity": "high",
|
||||
"confidence": "Firm",
|
||||
"asset_type": "web",
|
||||
"asset_value": "https://example.com/search",
|
||||
"tags": ["xss", "owasp-top10"],
|
||||
"detail_http_request": "GET /search?q=<script>alert(1)</script> HTTP/1.1\n...",
|
||||
"detail_http_response": "HTTP/1.1 200 OK\n...",
|
||||
"raw_vuln_json": "{\"template\":\"xss-reflected.yaml\",...}",
|
||||
"created_at": "2025-01-15T10:31:00Z",
|
||||
"updated_at": "2025-01-15T10:31:00Z"
|
||||
}
|
||||
],
|
||||
"pagination": {
|
||||
"total": 15,
|
||||
"offset": 0,
|
||||
"limit": 20
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Get Vulnerability Summary
|
||||
|
||||
Get a summary of vulnerabilities grouped by severity, optionally filtered by workspace.
|
||||
|
||||
**Get summary for all workspaces:**
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/vulnerabilities/summary \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Get summary for a specific workspace:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/vulnerabilities/summary?workspace=example.com" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `workspace` | string | - | Filter by workspace name |
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"by_severity": {
|
||||
"critical": 2,
|
||||
"high": 5,
|
||||
"medium": 8,
|
||||
"low": 12,
|
||||
"info": 3
|
||||
},
|
||||
"total": 30,
|
||||
"workspace": "example.com"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Get Vulnerability by ID
|
||||
|
||||
Retrieve a single vulnerability by its ID.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/vulnerabilities/1 \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"id": 1,
|
||||
"workspace": "example.com",
|
||||
"vuln_info": "CVE-2024-1234",
|
||||
"vuln_title": "SQL Injection in Login Form",
|
||||
"vuln_desc": "The login form is vulnerable to SQL injection via the username parameter.",
|
||||
"vuln_poc": "username=' OR '1'='1' --&password=test",
|
||||
"severity": "critical",
|
||||
"confidence": "Certain",
|
||||
"asset_type": "web",
|
||||
"asset_value": "https://example.com/login",
|
||||
"tags": ["sqli", "owasp-top10", "authentication"],
|
||||
"detail_http_request": "POST /login HTTP/1.1\nHost: example.com\n...",
|
||||
"detail_http_response": "HTTP/1.1 200 OK\n...",
|
||||
"raw_vuln_json": "{\"template\":\"sqli-login.yaml\",...}",
|
||||
"created_at": "2025-01-15T10:30:00Z",
|
||||
"updated_at": "2025-01-15T10:30:00Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Error Response (404):**
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Vulnerability not found"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Create Vulnerability
|
||||
|
||||
Create a new vulnerability record.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/vulnerabilities \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace": "example.com",
|
||||
"vuln_info": "CVE-2024-9999",
|
||||
"vuln_title": "Remote Code Execution",
|
||||
"vuln_desc": "Critical RCE vulnerability in admin panel.",
|
||||
"vuln_poc": "curl -X POST /admin/exec -d \"cmd=id\"",
|
||||
"severity": "critical",
|
||||
"asset_type": "web",
|
||||
"asset_value": "https://example.com/admin",
|
||||
"tags": ["rce", "critical", "admin"],
|
||||
"detail_http_request": "POST /admin/exec HTTP/1.1\n...",
|
||||
"detail_http_response": "HTTP/1.1 200 OK\nuid=0(root)...",
|
||||
"raw_vuln_json": "{\"template\":\"rce-admin.yaml\"}"
|
||||
}'
|
||||
```
|
||||
|
||||
**Request Body:**
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `workspace` | string | Yes | Workspace/target name |
|
||||
| `vuln_info` | string | No | CVE or vulnerability identifier |
|
||||
| `vuln_title` | string | No | Short title for the vulnerability |
|
||||
| `vuln_desc` | string | No | Detailed description |
|
||||
| `vuln_poc` | string | No | Proof of concept |
|
||||
| `severity` | string | No | Severity level (critical, high, medium, low, info) |
|
||||
| `confidence` | string | No | Confidence level (Certain, Firm, Tentative, Manual Review Required) |
|
||||
| `asset_type` | string | No | Type of asset (web, api, network, etc.) |
|
||||
| `asset_value` | string | No | Affected asset URL or identifier |
|
||||
| `tags` | array | No | Tags for categorization |
|
||||
| `detail_http_request` | string | No | Raw HTTP request |
|
||||
| `detail_http_response` | string | No | Raw HTTP response |
|
||||
| `raw_vuln_json` | string | No | Raw JSON from scanner (nuclei, etc.) |
|
||||
|
||||
**Response (201 Created):**
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"id": 15,
|
||||
"workspace": "example.com",
|
||||
"vuln_info": "CVE-2024-9999",
|
||||
"vuln_title": "Remote Code Execution",
|
||||
"vuln_desc": "Critical RCE vulnerability in admin panel.",
|
||||
"vuln_poc": "curl -X POST /admin/exec -d \"cmd=id\"",
|
||||
"severity": "critical",
|
||||
"confidence": "Certain",
|
||||
"asset_type": "web",
|
||||
"asset_value": "https://example.com/admin",
|
||||
"tags": ["rce", "critical", "admin"],
|
||||
"detail_http_request": "POST /admin/exec HTTP/1.1\n...",
|
||||
"detail_http_response": "HTTP/1.1 200 OK\nuid=0(root)...",
|
||||
"raw_vuln_json": "{\"template\":\"rce-admin.yaml\"}",
|
||||
"created_at": "2025-01-15T14:25:00Z",
|
||||
"updated_at": "2025-01-15T14:25:00Z"
|
||||
},
|
||||
"message": "Vulnerability created successfully"
|
||||
}
|
||||
```
|
||||
|
||||
**Error Response (400):**
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Workspace is required"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Delete Vulnerability
|
||||
|
||||
Delete a vulnerability by ID.
|
||||
|
||||
```bash
|
||||
curl -X DELETE http://localhost:8002/osm/api/vulnerabilities/15 \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Vulnerability deleted successfully"
|
||||
}
|
||||
```
|
||||
|
||||
**Error Response (404):**
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"message": "Vulnerability not found"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Vulnerability Fields Reference
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `id` | int | Unique vulnerability identifier |
|
||||
| `workspace` | string | Workspace/scan target name |
|
||||
| `vuln_info` | string | CVE or vulnerability identifier |
|
||||
| `vuln_title` | string | Short descriptive title |
|
||||
| `vuln_desc` | string | Detailed vulnerability description |
|
||||
| `vuln_poc` | string | Proof of concept exploit |
|
||||
| `severity` | string | Severity level (critical, high, medium, low, info) |
|
||||
| `confidence` | string | Confidence level (Certain, Firm, Tentative, Manual Review Required) |
|
||||
| `asset_type` | string | Type of affected asset |
|
||||
| `asset_value` | string | Affected asset URL or identifier |
|
||||
| `tags` | array | Categorization tags |
|
||||
| `detail_http_request` | string | Raw HTTP request that triggered the vulnerability |
|
||||
| `detail_http_response` | string | Raw HTTP response from the vulnerable endpoint |
|
||||
| `raw_vuln_json` | string | Raw JSON output from vulnerability scanner |
|
||||
| `created_at` | timestamp | Record creation timestamp |
|
||||
| `updated_at` | timestamp | Last update timestamp |
|
||||
@@ -0,0 +1,251 @@
|
||||
# Workflows
|
||||
|
||||
## List All Workflows
|
||||
|
||||
Get a paginated list of all available workflows with filtering support.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/workflows \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `source` | string | `db` | Data source: `db` (database) or `filesystem` (direct file scan) |
|
||||
| `tags` | string | - | Comma-separated list of tags to filter by |
|
||||
| `kind` | string | - | Filter by workflow kind: `flow` or `module` |
|
||||
| `search` | string | - | Search in workflow name and description |
|
||||
| `offset` | int | 0 | Pagination offset |
|
||||
| `limit` | int | 50 | Maximum records to return |
|
||||
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
# Filter by tags
|
||||
curl "http://localhost:8002/osm/api/workflows?tags=recon,subdomain" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
|
||||
# Filter by kind
|
||||
curl "http://localhost:8002/osm/api/workflows?kind=module" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
|
||||
# Search workflows
|
||||
curl "http://localhost:8002/osm/api/workflows?search=enum" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
|
||||
# Load directly from filesystem (bypasses database)
|
||||
curl "http://localhost:8002/osm/api/workflows?source=filesystem" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
|
||||
# Pagination
|
||||
curl "http://localhost:8002/osm/api/workflows?offset=10&limit=20" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"name": "subdomain-enum",
|
||||
"kind": "flow",
|
||||
"description": "Comprehensive subdomain enumeration and probing workflow",
|
||||
"tags": ["recon", "subdomain", "httpx"],
|
||||
"file_path": "/home/user/osmedeus-base/workflows/flows/subdomain-enum.yaml",
|
||||
"params": [
|
||||
{"name": "target", "required": true, "default": "", "generator": ""},
|
||||
{"name": "threads", "required": false, "default": "50", "generator": ""},
|
||||
{"name": "timeout", "required": false, "default": "30", "generator": ""},
|
||||
{"name": "wordlist", "required": false, "default": "", "generator": "default_wordlist"}
|
||||
],
|
||||
"required_params": ["target"],
|
||||
"step_count": 8,
|
||||
"module_count": 3,
|
||||
"checksum": "sha256:abc123...",
|
||||
"indexed_at": "2025-01-15T08:00:00Z"
|
||||
},
|
||||
{
|
||||
"name": "port-scan",
|
||||
"kind": "module",
|
||||
"description": "Port scanning module using nmap and masscan",
|
||||
"tags": ["recon", "portscan", "nmap"],
|
||||
"file_path": "/home/user/osmedeus-base/workflows/modules/port-scan.yaml",
|
||||
"params": [
|
||||
{"name": "target", "required": true, "default": "", "generator": ""},
|
||||
{"name": "ports", "required": false, "default": "top-1000", "generator": ""},
|
||||
{"name": "rate", "required": false, "default": "1000", "generator": ""}
|
||||
],
|
||||
"required_params": ["target"],
|
||||
"step_count": 4,
|
||||
"module_count": 0,
|
||||
"checksum": "sha256:def456...",
|
||||
"indexed_at": "2025-01-15T08:00:00Z"
|
||||
}
|
||||
],
|
||||
"pagination": {
|
||||
"total": 25,
|
||||
"offset": 0,
|
||||
"limit": 50
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Get Workflow Tags
|
||||
|
||||
Get all unique tags from indexed workflows.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/workflows/tags \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"tags": ["recon", "subdomain", "portscan", "vulnerability", "nuclei"],
|
||||
"count": 5
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Refresh Workflow Index
|
||||
|
||||
Re-index all workflows from filesystem to database. Use this after adding or modifying workflow files.
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8002/osm/api/workflows/refresh \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `force` | bool | false | Force re-index all workflows regardless of checksum |
|
||||
|
||||
**Force re-index all:**
|
||||
```bash
|
||||
curl -X POST "http://localhost:8002/osm/api/workflows/refresh?force=true" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"message": "Workflows indexed successfully",
|
||||
"added": 5,
|
||||
"updated": 2,
|
||||
"removed": 1,
|
||||
"errors": []
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Get Workflow Details
|
||||
|
||||
Get workflow content. Returns raw YAML by default, or JSON with full parsed details.
|
||||
|
||||
```bash
|
||||
# Get raw YAML content (default)
|
||||
curl http://localhost:8002/osm/api/workflows/subdomain-enum \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
```bash
|
||||
# Get workflow details as JSON
|
||||
curl "http://localhost:8002/osm/api/workflows/subdomain-enum?json=true" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `json` | bool | false | Return JSON with parsed details instead of raw YAML |
|
||||
|
||||
**Response (YAML - default):**
|
||||
```yaml
|
||||
name: subdomain-enum
|
||||
kind: flow
|
||||
description: Subdomain enumeration
|
||||
params:
|
||||
- name: target
|
||||
required: true
|
||||
steps:
|
||||
- name: run-subfinder
|
||||
command: subfinder -d {{target}}
|
||||
...
|
||||
```
|
||||
|
||||
**Response (JSON with `?json=true`):**
|
||||
```json
|
||||
{
|
||||
"name": "subdomain-enum",
|
||||
"kind": "flow",
|
||||
"description": "Comprehensive subdomain enumeration and probing workflow",
|
||||
"file_path": "/home/user/osmedeus-base/workflows/flows/subdomain-enum.yaml",
|
||||
"params": [
|
||||
{"name": "target", "required": true, "default": "", "generator": ""},
|
||||
{"name": "threads", "required": false, "default": "50", "generator": ""},
|
||||
{"name": "output_dir", "required": false, "default": "{{Workspace}}", "generator": "workspace_path"},
|
||||
{"name": "wordlist", "required": false, "default": "", "generator": "default_wordlist"}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"index": 0,
|
||||
"name": "run-subfinder",
|
||||
"type": "bash",
|
||||
"command": "subfinder -d {{target}} -t {{threads}} -o {{output_dir}}/subdomains-subfinder.txt",
|
||||
"timeout": "30m",
|
||||
"pre_condition": "",
|
||||
"exports": {"subfinder_output": "{{output_dir}}/subdomains-subfinder.txt"}
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"name": "run-amass",
|
||||
"type": "bash",
|
||||
"command": "amass enum -passive -d {{target}} -o {{output_dir}}/subdomains-amass.txt",
|
||||
"timeout": "60m",
|
||||
"pre_condition": "commandExists('amass')",
|
||||
"exports": {"amass_output": "{{output_dir}}/subdomains-amass.txt"}
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"name": "merge-subdomains",
|
||||
"type": "function",
|
||||
"command": "mergeFiles('{{output_dir}}/subdomains-*.txt', '{{output_dir}}/all-subdomains.txt')",
|
||||
"timeout": "",
|
||||
"pre_condition": "",
|
||||
"exports": {"all_subdomains": "{{output_dir}}/all-subdomains.txt"}
|
||||
},
|
||||
{
|
||||
"index": 3,
|
||||
"name": "run-httpx",
|
||||
"type": "bash",
|
||||
"command": "httpx -l {{all_subdomains}} -t {{threads}} -o {{output_dir}}/alive.txt -json -o {{output_dir}}/httpx.json",
|
||||
"timeout": "60m",
|
||||
"pre_condition": "fileLength('{{all_subdomains}}') > 0",
|
||||
"exports": {"alive_hosts": "{{output_dir}}/alive.txt", "httpx_json": "{{output_dir}}/httpx.json"}
|
||||
}
|
||||
],
|
||||
"modules": [
|
||||
{"index": 0, "name": "port-scan", "path": "modules/port-scan.yaml", "depends_on": [], "condition": ""},
|
||||
{"index": 1, "name": "nuclei-scan", "path": "modules/nuclei-scan.yaml", "depends_on": ["port-scan"], "condition": "fileLength('{{alive_hosts}}') > 0"},
|
||||
{"index": 2, "name": "screenshot", "path": "modules/screenshot.yaml", "depends_on": ["port-scan"], "condition": ""}
|
||||
],
|
||||
"triggers": [
|
||||
{"name": "daily-scan", "on": "cron", "schedule": "0 2 * * *", "enabled": true},
|
||||
{"name": "on-new-asset", "on": "event", "topic": "asset.discovered", "enabled": false}
|
||||
],
|
||||
"dependencies": {
|
||||
"commands": ["subfinder", "amass", "httpx", "nuclei", "nmap"],
|
||||
"files": ["{{wordlist}}"]
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,159 @@
|
||||
# Workspaces
|
||||
|
||||
## List Workspaces
|
||||
|
||||
Get a list of all run workspaces.
|
||||
|
||||
**List workspaces from database (default):**
|
||||
```bash
|
||||
curl http://localhost:8002/osm/api/workspaces \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**List workspaces with pagination:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/workspaces?offset=0&limit=50" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**List workspaces from filesystem/assets:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/workspaces?filesystem=true" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Combine pagination with filesystem mode:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/workspaces?filesystem=true&offset=20&limit=10" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response (database mode):**
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "example.com",
|
||||
"local_path": "/home/user/osmedeus-base/workspaces/example.com",
|
||||
"total_assets": 150,
|
||||
"total_subdomains": 120,
|
||||
"total_urls": 500,
|
||||
"total_vulns": 12,
|
||||
"vuln_critical": 2,
|
||||
"vuln_high": 3,
|
||||
"vuln_medium": 4,
|
||||
"vuln_low": 3,
|
||||
"vuln_potential": 0,
|
||||
"risk_score": 7.5,
|
||||
"tags": ["production", "priority"],
|
||||
"last_run": "2025-01-15T10:30:00Z",
|
||||
"run_workflow": "subdomain-enum",
|
||||
"created_at": "2025-01-10T08:00:00Z",
|
||||
"updated_at": "2025-01-15T10:30:00Z"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "test.com",
|
||||
"local_path": "/home/user/osmedeus-base/workspaces/test.com",
|
||||
"total_assets": 50,
|
||||
"total_subdomains": 35,
|
||||
"total_urls": 120,
|
||||
"total_vulns": 3,
|
||||
"vuln_critical": 0,
|
||||
"vuln_high": 1,
|
||||
"vuln_medium": 2,
|
||||
"vuln_low": 0,
|
||||
"vuln_potential": 5,
|
||||
"risk_score": 4.2,
|
||||
"tags": ["staging"],
|
||||
"last_run": "2025-01-14T15:00:00Z",
|
||||
"run_workflow": "port-scan",
|
||||
"created_at": "2025-01-12T12:00:00Z",
|
||||
"updated_at": "2025-01-14T15:00:00Z"
|
||||
}
|
||||
],
|
||||
"pagination": {
|
||||
"total": 100,
|
||||
"offset": 0,
|
||||
"limit": 20
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Workspace Fields Reference:**
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `id` | int | Unique workspace identifier |
|
||||
| `name` | string | Workspace name (usually the target domain) |
|
||||
| `local_path` | string | Full path to workspace directory |
|
||||
| `total_assets` | int | Total discovered assets |
|
||||
| `total_subdomains` | int | Total discovered subdomains |
|
||||
| `total_urls` | int | Total discovered URLs |
|
||||
| `total_vulns` | int | Total vulnerabilities found |
|
||||
| `vuln_critical` | int | Critical severity vulnerabilities |
|
||||
| `vuln_high` | int | High severity vulnerabilities |
|
||||
| `vuln_medium` | int | Medium severity vulnerabilities |
|
||||
| `vuln_low` | int | Low severity vulnerabilities |
|
||||
| `vuln_potential` | int | Potential/informational findings |
|
||||
| `risk_score` | float | Calculated risk score (0-10) |
|
||||
| `tags` | array | Custom tags for organization |
|
||||
| `last_run` | timestamp | Last workflow run timestamp |
|
||||
| `run_workflow` | string | Name of last executed workflow |
|
||||
| `state_execution_log` | string | Path to execution log file |
|
||||
| `state_completed_file` | string | Path to completed marker file |
|
||||
| `state_workflow_file` | string | Path to workflow YAML file |
|
||||
| `state_workflow_folder` | string | Path to workflow folder |
|
||||
| `created_at` | timestamp | Workspace creation timestamp |
|
||||
| `updated_at` | timestamp | Last update timestamp |
|
||||
|
||||
---
|
||||
|
||||
## Get Workspace State File
|
||||
|
||||
Retrieve the content of a workspace state file. This endpoint provides access to execution logs, completion markers, and workflow files associated with a workspace.
|
||||
|
||||
**Get execution log:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/workspaces/example.com/state-file?state_file=execution_log" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Get completed file:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/workspaces/example.com/state-file?state_file=completed_file" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Get workflow file:**
|
||||
```bash
|
||||
curl "http://localhost:8002/osm/api/workspaces/example.com/state-file?state_file=workflow_file" \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"workspace": "example.com",
|
||||
"state_file": "execution_log",
|
||||
"file_path": "/home/user/osmedeus-base/workspaces/example.com/log/execution.log",
|
||||
"content": "2025-01-15 10:30:00 [INFO] Starting workflow...\n2025-01-15 10:30:05 [INFO] Step 1 completed...\n..."
|
||||
}
|
||||
```
|
||||
|
||||
**State File Types:**
|
||||
|
||||
| Type | Description |
|
||||
|------|-------------|
|
||||
| `execution_log` | Detailed execution log with timestamps and step output |
|
||||
| `completed_file` | Marker file indicating workflow completion status |
|
||||
| `workflow_file` | The YAML workflow definition that was executed |
|
||||
|
||||
**Error Responses:**
|
||||
|
||||
| Status | Description |
|
||||
|--------|-------------|
|
||||
| 400 | Invalid workspace name or missing state_file parameter |
|
||||
| 403 | Path traversal attempt detected |
|
||||
| 404 | Workspace or state file not found |
|
||||
@@ -0,0 +1,161 @@
|
||||
module github.com/j3ssie/osmedeus/v5
|
||||
|
||||
go 1.25.4
|
||||
|
||||
require (
|
||||
github.com/Masterminds/semver/v3 v3.4.0
|
||||
github.com/alecthomas/chroma/v2 v2.21.1
|
||||
github.com/charmbracelet/bubbles v0.21.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/glamour v0.10.0
|
||||
github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834
|
||||
github.com/creativeprojects/go-selfupdate v1.5.2
|
||||
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3
|
||||
github.com/flosch/pongo2/v6 v6.0.0
|
||||
github.com/fsnotify/fsnotify v1.9.0
|
||||
github.com/go-co-op/gocron/v2 v2.12.4
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1
|
||||
github.com/goccy/go-yaml v1.19.1
|
||||
github.com/gofiber/adaptor/v2 v2.2.1
|
||||
github.com/gofiber/fiber/v2 v2.52.10
|
||||
github.com/gofiber/swagger v1.1.1
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hashicorp/go-getter/v2 v2.2.3
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/itchyny/gojq v0.12.18
|
||||
github.com/mattn/go-sqlite3 v1.14.32
|
||||
github.com/minio/minio-go/v7 v7.0.97
|
||||
github.com/olekukonko/tablewriter v0.0.5
|
||||
github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44
|
||||
github.com/pkg/sftp v1.13.9
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/redis/rueidis v1.0.70
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/swaggo/swag v1.16.6
|
||||
github.com/uptrace/bun v1.2.16
|
||||
github.com/uptrace/bun/dialect/pgdialect v1.2.16
|
||||
github.com/uptrace/bun/dialect/sqlitedialect v1.2.16
|
||||
github.com/uptrace/bun/driver/pgdriver v1.2.8
|
||||
github.com/uptrace/bun/driver/sqliteshim v1.2.16
|
||||
github.com/valyala/fastjson v1.6.7
|
||||
go.uber.org/zap v1.27.1
|
||||
golang.org/x/crypto v0.46.0
|
||||
golang.org/x/net v0.48.0
|
||||
golang.org/x/term v0.38.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
code.gitea.io/sdk/gitea v0.22.1 // indirect
|
||||
github.com/42wim/httpsig v1.2.3 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/PuerkitoBio/purell v1.1.1 // indirect
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
|
||||
github.com/charmbracelet/harmonica v0.2.0 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.10.1 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13 // indirect
|
||||
github.com/charmbracelet/x/exp/slice v0.0.0-20250327172914-2fdc97757edf // indirect
|
||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/davidmz/go-pageant v1.0.2 // indirect
|
||||
github.com/dlclark/regexp2 v1.11.5 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/go-fed/httpsig v1.1.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||
github.com/go-openapi/jsonreference v0.19.6 // indirect
|
||||
github.com/go-openapi/spec v0.20.4 // indirect
|
||||
github.com/go-openapi/swag v0.19.15 // indirect
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
|
||||
github.com/google/go-github/v74 v74.0.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.0 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
|
||||
github.com/hashicorp/go-safetemp v1.0.0 // indirect
|
||||
github.com/hashicorp/go-version v1.8.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/itchyny/timefmt-go v0.1.7 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jonboulle/clockwork v0.5.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
|
||||
github.com/minio/crc64nvme v1.1.0 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/mitchellh/go-homedir v1.0.0 // indirect
|
||||
github.com/mitchellh/go-testing-interface v1.0.0 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/reflow v0.3.0 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/sahilm/fuzzy v0.1.1 // indirect
|
||||
github.com/spf13/pflag v1.0.9 // indirect
|
||||
github.com/swaggo/files/v2 v2.0.2 // indirect
|
||||
github.com/tinylib/msgp v1.3.0 // indirect
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.52.0 // indirect
|
||||
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/yuin/goldmark v1.7.8 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.5 // indirect
|
||||
gitlab.com/gitlab-org/api/client-go v1.9.1 // indirect
|
||||
go.uber.org/multierr v1.10.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 // indirect
|
||||
golang.org/x/mod v0.30.0 // indirect
|
||||
golang.org/x/oauth2 v0.34.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
golang.org/x/tools v0.39.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
mellium.im/sasl v0.3.2 // indirect
|
||||
modernc.org/libc v1.67.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.42.2 // indirect
|
||||
)
|
||||
@@ -0,0 +1,477 @@
|
||||
code.gitea.io/sdk/gitea v0.22.1 h1:7K05KjRORyTcTYULQ/AwvlVS6pawLcWyXZcTr7gHFyA=
|
||||
code.gitea.io/sdk/gitea v0.22.1/go.mod h1:yyF5+GhljqvA30sRDreoyHILruNiy4ASufugzYg0VHM=
|
||||
github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs=
|
||||
github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM=
|
||||
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
|
||||
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
|
||||
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
|
||||
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
|
||||
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
|
||||
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
|
||||
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
|
||||
github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
|
||||
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/chroma/v2 v2.21.1 h1:FaSDrp6N+3pphkNKU6HPCiYLgm8dbe5UXIXcoBhZSWA=
|
||||
github.com/alecthomas/chroma/v2 v2.21.1/go.mod h1:NqVhfBR0lte5Ouh3DcthuUCTUpDC9cxBOfyMbMQPs3o=
|
||||
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
|
||||
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
|
||||
github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
|
||||
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
|
||||
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
|
||||
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
|
||||
github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
|
||||
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs=
|
||||
github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg=
|
||||
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
|
||||
github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs=
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk=
|
||||
github.com/charmbracelet/glamour v0.10.0 h1:MtZvfwsYCx8jEPFJm3rIBFIMZUfUJ765oX8V6kXldcY=
|
||||
github.com/charmbracelet/glamour v0.10.0/go.mod h1:f+uf+I/ChNmqo087elLnVdCiVgjSKWuXa/l6NU2ndYk=
|
||||
github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ=
|
||||
github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao=
|
||||
github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 h1:ZR7e0ro+SZZiIZD7msJyA+NjkCNNavuiPBLgerbOziE=
|
||||
github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834/go.mod h1:aKC/t2arECF6rNOnaKaVU6y4t4ZeHQzqfxedE/VkVhA=
|
||||
github.com/charmbracelet/x/ansi v0.10.1 h1:rL3Koar5XvX0pHGfovN03f5cxLbCF2YvLeyz7D2jVDQ=
|
||||
github.com/charmbracelet/x/ansi v0.10.1/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs=
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ=
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
|
||||
github.com/charmbracelet/x/exp/slice v0.0.0-20250327172914-2fdc97757edf h1:rLG0Yb6MQSDKdB52aGX55JT1oi0P0Kuaj7wi1bLUpnI=
|
||||
github.com/charmbracelet/x/exp/slice v0.0.0-20250327172914-2fdc97757edf/go.mod h1:B3UgsnsBZS/eX42BlaNiJkD1pPOUa+oF1IYC6Yd2CEU=
|
||||
github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
|
||||
github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
|
||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
||||
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/creativeprojects/go-selfupdate v1.5.2 h1:3KR3JLrq70oplb9yZzbmJ89qRP78D1AN/9u+l3k0LJ4=
|
||||
github.com/creativeprojects/go-selfupdate v1.5.2/go.mod h1:BCOuwIl1dRRCmPNRPH0amULeZqayhKyY2mH/h4va7Dk=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
|
||||
github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
|
||||
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
|
||||
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3 h1:bVp3yUzvSAJzu9GqID+Z96P+eu5TKnIMJSV4QaZMauM=
|
||||
github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
|
||||
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
|
||||
github.com/flosch/pongo2/v6 v6.0.0 h1:lsGru8IAzHgIAw6H2m4PCyleO58I40ow6apih0WprMU=
|
||||
github.com/flosch/pongo2/v6 v6.0.0/go.mod h1:CuDpFm47R0uGGE7z13/tTlt1Y6zdxvr2RLT5LJhsHEU=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-co-op/gocron/v2 v2.12.4 h1:h1HWApo3T+61UrZqEY2qG1LUpDnB7tkYITxf6YIK354=
|
||||
github.com/go-co-op/gocron/v2 v2.12.4/go.mod h1:xY7bJxGazKam1cz04EebrlP4S9q4iWdiAylMGP3jY9w=
|
||||
github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
|
||||
github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=
|
||||
github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
|
||||
github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=
|
||||
github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
|
||||
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
|
||||
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 h1:wG8n/XJQ07TmjbITcGiUaOtXxdrINDz1b0J1w0SzqDc=
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8=
|
||||
github.com/goccy/go-yaml v1.19.1 h1:3rG3+v8pkhRqoQ/88NYNMHYVGYztCOCIZ7UQhu7H+NE=
|
||||
github.com/goccy/go-yaml v1.19.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/gofiber/adaptor/v2 v2.2.1 h1:givE7iViQWlsTR4Jh7tB4iXzrlKBgiraB/yTdHs9Lv4=
|
||||
github.com/gofiber/adaptor/v2 v2.2.1/go.mod h1:AhR16dEqs25W2FY/l8gSj1b51Azg5dtPDmm+pruNOrc=
|
||||
github.com/gofiber/fiber/v2 v2.52.10 h1:jRHROi2BuNti6NYXmZ6gbNSfT3zj/8c0xy94GOU5elY=
|
||||
github.com/gofiber/fiber/v2 v2.52.10/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
|
||||
github.com/gofiber/swagger v1.1.1 h1:FZVhVQQ9s1ZKLHL/O0loLh49bYB5l1HEAgxDlcTtkRA=
|
||||
github.com/gofiber/swagger v1.1.1/go.mod h1:vtvY/sQAMc/lGTUCg0lqmBL7Ht9O7uzChpbvJeJQINw=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-github/v74 v74.0.0 h1:yZcddTUn8DPbj11GxnMrNiAnXH14gNs559AsUpNpPgM=
|
||||
github.com/google/go-github/v74 v74.0.0/go.mod h1:ubn/YdyftV80VPSI26nSJvaEsTOnsjrxG3o9kJhcyak=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
|
||||
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
|
||||
github.com/hashicorp/go-getter/v2 v2.2.3 h1:6CVzhT0KJQHqd9b0pK3xSP0CM/Cv+bVhk+jcaRJ2pGk=
|
||||
github.com/hashicorp/go-getter/v2 v2.2.3/go.mod h1:hp5Yy0GMQvwWVUmwLs3ygivz1JSLI323hdIE9J9m7TY=
|
||||
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
|
||||
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
|
||||
github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
|
||||
github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
|
||||
github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
|
||||
github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
|
||||
github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4=
|
||||
github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/itchyny/gojq v0.12.18 h1:gFGHyt/MLbG9n6dqnvlliiya2TaMMh6FFaR2b1H6Drc=
|
||||
github.com/itchyny/gojq v0.12.18/go.mod h1:4hPoZ/3lN9fDL1D+aK7DY1f39XZpY9+1Xpjz8atrEkg=
|
||||
github.com/itchyny/timefmt-go v0.1.7 h1:xyftit9Tbw+Dc/huSSPJaEmX1TVL8lw5vxjJLK4GMMA=
|
||||
github.com/itchyny/timefmt-go v0.1.7/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
|
||||
github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
|
||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
|
||||
github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
|
||||
github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q=
|
||||
github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ=
|
||||
github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk=
|
||||
github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
|
||||
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
|
||||
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
|
||||
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
|
||||
github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM=
|
||||
github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4=
|
||||
github.com/orivej/e v0.0.0-20180728214217-ac3492690fda h1:fqLgbcmo9qKecZOH8lByuxi9XXoIhNYBpRJEo4rDEUQ=
|
||||
github.com/orivej/e v0.0.0-20180728214217-ac3492690fda/go.mod h1:eOxOguJBxQH6q/o7CZvmR+fh5v1LHH1sfohtgISSSFA=
|
||||
github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44 h1:XDJpMiCKWt8CIT2LE1QrF4DdrvI1WciSNUrnYtNewPo=
|
||||
github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44/go.mod h1:4SkaXpoQ0tQ0OIkGqU8ByPLANmTTTU1iWPDz7YXatSA=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||
github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw=
|
||||
github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg=
|
||||
github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
|
||||
github.com/redis/rueidis v1.0.70 h1:O01v0Mt27/qXV9mKU/zahgxHdC8piHzIepqW4Nyzn/I=
|
||||
github.com/redis/rueidis v1.0.70/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA=
|
||||
github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/swaggo/files/v2 v2.0.2 h1:Bq4tgS/yxLB/3nwOMcul5oLEUKa877Ykgz3CJMVbQKU=
|
||||
github.com/swaggo/files/v2 v2.0.2/go.mod h1:TVqetIzZsO9OhHX1Am9sRf9LdrFZqoK49N37KON/jr0=
|
||||
github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
|
||||
github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
|
||||
github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
|
||||
github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
|
||||
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
|
||||
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/uptrace/bun v1.2.16 h1:QlObi6ZIK5Ao7kAALnh91HWYNZUBbVwye52fmlQM9kc=
|
||||
github.com/uptrace/bun v1.2.16/go.mod h1:jMoNg2n56ckaawi/O/J92BHaECmrz6IRjuMWqlMaMTM=
|
||||
github.com/uptrace/bun/dialect/pgdialect v1.2.16 h1:KFNZ0LxAyczKNfK/IJWMyaleO6eI9/Z5tUv3DE1NVL4=
|
||||
github.com/uptrace/bun/dialect/pgdialect v1.2.16/go.mod h1:IJdMeV4sLfh0LDUZl7TIxLI0LipF1vwTK3hBC7p5qLo=
|
||||
github.com/uptrace/bun/dialect/sqlitedialect v1.2.16 h1:6wVAiYLj1pMibRthGwy4wDLa3D5AQo32Y8rvwPd8CQ0=
|
||||
github.com/uptrace/bun/dialect/sqlitedialect v1.2.16/go.mod h1:Z7+5qK8CGZkDQiPMu+LSdVuDuR1I5jcwtkB1Pi3F82E=
|
||||
github.com/uptrace/bun/driver/pgdriver v1.2.8 h1:5XrNn/9enSrWhhrUpz+6PY9S1vcg/jhCQPJu+ZmsKX4=
|
||||
github.com/uptrace/bun/driver/pgdriver v1.2.8/go.mod h1:cwRRwqabgePwYBiLlXtbeNmPD7LGJnqP21J2ZKP4ah8=
|
||||
github.com/uptrace/bun/driver/sqliteshim v1.2.16 h1:M6Dh5kkDWFbUWBrOsIE1g1zdZ5JbSytTD4piFRBOUAI=
|
||||
github.com/uptrace/bun/driver/sqliteshim v1.2.16/go.mod h1:iKdJ06P3XS+pwKcONjSIK07bbhksH3lWsw3mpfr0+bY=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
|
||||
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
|
||||
github.com/valyala/fastjson v1.6.7 h1:ZE4tRy0CIkh+qDc5McjatheGX2czdn8slQjomexVpBM=
|
||||
github.com/valyala/fastjson v1.6.7/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
|
||||
github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic=
|
||||
github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
|
||||
github.com/yuin/goldmark-emoji v1.0.5 h1:EMVWyCGPlXJfUXBXpuMu+ii3TIaxbVBnEX9uaDC4cIk=
|
||||
github.com/yuin/goldmark-emoji v1.0.5/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U=
|
||||
gitlab.com/gitlab-org/api/client-go v1.9.1 h1:tZm+URa36sVy8UCEHQyGGJ8COngV4YqMHpM6k9O5tK8=
|
||||
gitlab.com/gitlab-org/api/client-go v1.9.1/go.mod h1:71yTJk1lnHCWcZLvM5kPAXzeJ2fn5GjaoV8gTOPd4ME=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
|
||||
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
|
||||
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
|
||||
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 h1:zfMcR1Cs4KNuomFFgGefv5N0czO2XZpUbxGUy8i8ug0=
|
||||
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6/go.mod h1:46edojNIoXTNOhySWIWdix628clX9ODXwPsQuG6hsK0=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
|
||||
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
|
||||
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
|
||||
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
|
||||
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
mellium.im/sasl v0.3.2 h1:PT6Xp7ccn9XaXAnJ03FcEjmAn7kK1x7aoXV6F+Vmrl0=
|
||||
mellium.im/sasl v0.3.2/go.mod h1:NKXDi1zkr+BlMHLQjY3ofYuU4KSPFxknb8mfEu6SveY=
|
||||
modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
|
||||
modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
|
||||
modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc=
|
||||
modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM=
|
||||
modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
|
||||
modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE=
|
||||
modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.67.0 h1:QzL4IrKab2OFmxA3/vRYl0tLXrIamwrhD6CKD4WBVjQ=
|
||||
modernc.org/libc v1.67.0/go.mod h1:QvvnnJ5P7aitu0ReNpVIEyesuhmDLQ8kaEoyMjIFZJA=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
|
||||
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.42.2 h1:7hkZUNJvJFN2PgfUdjni9Kbvd4ef4mNLOu0B9FGxM74=
|
||||
modernc.org/sqlite v1.42.2/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
@@ -0,0 +1,940 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// exampleConfigYAML contains the default configuration template
|
||||
// Source of truth: public/presets/osm-settings.example.yaml
|
||||
var exampleConfigYAML = []byte(`# Osmedeus Configuration File
|
||||
# This file contains all available configuration options for osmedeus.
|
||||
# Copy this file to ~/osmedeus-base/osm-settings.yaml and customize as needed.
|
||||
|
||||
# =============================================================================
|
||||
# Base Folder
|
||||
# =============================================================================
|
||||
# Root directory for all osmedeus data (workflows, binaries, data, etc.)
|
||||
# Environment variables like $HOME are automatically expanded
|
||||
base_folder: $HOME/osmedeus-base
|
||||
|
||||
# =============================================================================
|
||||
# Environment Paths
|
||||
# =============================================================================
|
||||
# Directory paths for various osmedeus components
|
||||
# Use {{base_folder}} to reference the base_folder value above
|
||||
environments:
|
||||
# Path to binary executables (tools like nmap, ffuf, etc.)
|
||||
external_binaries_path: "{{base_folder}}/external-binaries"
|
||||
|
||||
# Data directory for storing assets, wordlists, etc.
|
||||
external_data: "{{base_folder}}/external-data"
|
||||
|
||||
# External configuration files (nuclei templates, etc.)
|
||||
external_configs: "{{base_folder}}/external-configs"
|
||||
|
||||
# Output directory for scan workspaces
|
||||
# Each target gets its own subdirectory here
|
||||
workspaces: "$HOME/workspaces-osmedeus"
|
||||
|
||||
# Directory containing workflow YAML files
|
||||
# Subdirectories: flows/, modules/
|
||||
workflows: "{{base_folder}}/workflows"
|
||||
|
||||
# Directory for workspace snapshots (zip archives)
|
||||
# Used by the snapshot-download API endpoint
|
||||
snapshot: "{{base_folder}}/snapshot"
|
||||
|
||||
# Directory for markdown report templates
|
||||
# Used by render_markdown_report() function
|
||||
markdown_report_templates: "{{base_folder}}/markdown-report-templates"
|
||||
|
||||
# Directory for external agent configurations
|
||||
# Used for LLM Agent commands, skills, and related configurations
|
||||
external_agent_configs: "{{base_folder}}/external-agent-configs"
|
||||
|
||||
# Directory for external utility scripts
|
||||
# Used for storing custom scripts and utilities
|
||||
external_scripts: "{{base_folder}}/external-scripts"
|
||||
|
||||
# =============================================================================
|
||||
# Database Configuration
|
||||
# =============================================================================
|
||||
# Osmedeus supports SQLite (default) and PostgreSQL
|
||||
database:
|
||||
# Database engine: "sqlite" or "postgresql"
|
||||
db_engine: sqlite
|
||||
|
||||
# SQLite: Path to the database file
|
||||
# Ignored when using PostgreSQL
|
||||
db_path: "{{base_folder}}/database-osm.sqlite"
|
||||
|
||||
# PostgreSQL connection settings
|
||||
# Only used when db_engine is "postgresql"
|
||||
host: localhost
|
||||
port: 5432
|
||||
username: osmedeus
|
||||
password: osmedeus
|
||||
db_name: osmedeus
|
||||
|
||||
# Connection timeout in seconds
|
||||
connection_timeout: 60
|
||||
|
||||
# PostgreSQL SSL mode: disable, require, verify-ca, verify-full
|
||||
ssl_mode: disable
|
||||
|
||||
# =============================================================================
|
||||
# Server Configuration
|
||||
# =============================================================================
|
||||
# REST API server settings for the web interface
|
||||
server:
|
||||
# Host to bind the server to
|
||||
# Use "0.0.0.0" to listen on all interfaces
|
||||
# Use "127.0.0.1" to listen only on localhost
|
||||
host: "0.0.0.0"
|
||||
|
||||
# Port number for the API server
|
||||
port: 8002
|
||||
|
||||
# Path to serve static UI files
|
||||
# Default: {{base_folder}}/ui/ - if this directory exists, it will be served at /ui
|
||||
# Set to empty string to disable UI serving
|
||||
ui_path: "{{base_folder}}/ui/"
|
||||
|
||||
# Random prefix for workspace static files (auto-generated 16 chars if empty)
|
||||
# Used as URL path segment for direct access to workspaces folder
|
||||
workspace_prefix_key: ""
|
||||
|
||||
# Authentication credentials (map of username:password)
|
||||
# Supports multiple users
|
||||
simple_user_map_key:
|
||||
osmedeus: osmedeus-admin
|
||||
|
||||
# JWT (JSON Web Token) settings
|
||||
jwt:
|
||||
# Secret key for signing JWT tokens
|
||||
# IMPORTANT: Use a strong, unique secret in production!
|
||||
secret_signing_key: change-this-secret-in-production
|
||||
|
||||
# Token expiration time in minutes
|
||||
expiration_minutes: 60
|
||||
|
||||
# License type shown in HTTP Server header and /server-info endpoint
|
||||
license: "open-source"
|
||||
|
||||
# =============================================================================
|
||||
# Scan Tactic Configuration
|
||||
# =============================================================================
|
||||
# Thread counts for different scan intensity levels
|
||||
# Higher values = faster but more aggressive scans
|
||||
# Lower values = slower but gentler on target systems
|
||||
scan_tactic:
|
||||
# Aggressive/fast mode - maximum parallelism
|
||||
# Used with: osmedeus scan -t target --tactic aggressive
|
||||
aggressive: 40
|
||||
|
||||
# Default/normal mode - balanced approach
|
||||
# Used when no tactic is specified
|
||||
default: 10
|
||||
|
||||
# Gentle/thorough mode - minimal parallelism
|
||||
# Used with: osmedeus scan -t target --tactic gently
|
||||
gently: 5
|
||||
|
||||
# =============================================================================
|
||||
# Redis Configuration (Optional)
|
||||
# =============================================================================
|
||||
# Redis is required for distributed scanning mode
|
||||
# Leave host empty to disable Redis
|
||||
redis:
|
||||
# Redis server hostname
|
||||
# Leave empty to disable distributed mode
|
||||
host: ""
|
||||
|
||||
# Redis server port
|
||||
port: 6379
|
||||
|
||||
# Redis authentication (if required)
|
||||
username: ""
|
||||
password: ""
|
||||
|
||||
# Redis database number (0-15)
|
||||
db: 0
|
||||
|
||||
# Connection timeout in seconds
|
||||
connection_timeout: 60
|
||||
|
||||
# =============================================================================
|
||||
# Global Variables
|
||||
# =============================================================================
|
||||
# User-defined variables available in workflows via {{VARIABLE_NAME}}
|
||||
# Variables can optionally be exported to environment variables
|
||||
# Use _API_KEY suffix for secrets to indicate sensitive values
|
||||
#
|
||||
# Format:
|
||||
# VARIABLE_NAME:
|
||||
# value: "the-value"
|
||||
# as_env: true # Optional: export as env var (default: true)
|
||||
#
|
||||
# Example usage in workflows:
|
||||
# - bash: "echo {{GITHUB_API_KEY}}"
|
||||
# - bash: "shodan search $SHODAN_API_KEY" # Uses env var
|
||||
global_vars:
|
||||
# GitHub personal access token for API access
|
||||
GITHUB_API_KEY:
|
||||
value: ""
|
||||
as_env: true # Exports as GITHUB_API_KEY
|
||||
|
||||
# Shodan API key for passive reconnaissance
|
||||
SHODAN_API_KEY:
|
||||
value: ""
|
||||
as_env: true # Exports as SHODAN_API_KEY
|
||||
|
||||
# Censys API key for certificate/host search
|
||||
CENSYS_API_KEY:
|
||||
value: ""
|
||||
as_env: true # Exports as CENSYS_API_KEY
|
||||
|
||||
# PassiveTotal API key for passive DNS/WHOIS
|
||||
PASSIVETOTAL_API_KEY:
|
||||
value: ""
|
||||
as_env: true # Exports as PASSIVETOTAL_API_KEY
|
||||
|
||||
# Add more API keys as needed (use _API_KEY suffix for secrets)
|
||||
|
||||
# =============================================================================
|
||||
# Notification Configuration
|
||||
# =============================================================================
|
||||
# Send notifications when scans complete or find interesting results
|
||||
notification:
|
||||
# Notification provider: "telegram" (future: slack, discord, webhook)
|
||||
provider: telegram
|
||||
|
||||
# Master switch to enable/disable all notifications
|
||||
enabled: false
|
||||
|
||||
# Telegram bot settings
|
||||
# Create a bot via @BotFather and get the token
|
||||
# Get your chat ID by messaging @userinfobot
|
||||
telegram:
|
||||
# Bot token from @BotFather
|
||||
bot_token: ""
|
||||
|
||||
# Chat ID to send messages to (can be user or group)
|
||||
chat_id: 0
|
||||
|
||||
# Enable Telegram notifications
|
||||
enabled: false
|
||||
|
||||
# =============================================================================
|
||||
# Cloud Storage Configuration (Optional)
|
||||
# =============================================================================
|
||||
# S3-compatible storage for backing up scan results
|
||||
# Supports AWS S3, MinIO, Google Cloud Storage, DigitalOcean Spaces, etc.
|
||||
storage:
|
||||
# Storage provider: "s3", "minio", "gcs", "spaces", etc.
|
||||
provider: s3
|
||||
|
||||
# Storage endpoint URL
|
||||
# AWS S3: Leave empty or use region-specific endpoint
|
||||
# MinIO: "http://localhost:9000"
|
||||
# DigitalOcean: "https://nyc3.digitaloceanspaces.com"
|
||||
endpoint: ""
|
||||
|
||||
# Access credentials
|
||||
access_key_id: ""
|
||||
secret_access_key: ""
|
||||
|
||||
# Bucket name for storing results
|
||||
bucket: ""
|
||||
|
||||
# Cloud region (e.g., us-east-1, eu-west-1)
|
||||
region: us-east-1
|
||||
|
||||
# Use SSL/TLS for connections
|
||||
use_ssl: true
|
||||
|
||||
# Enable cloud storage uploads
|
||||
enabled: false
|
||||
|
||||
# =============================================================================
|
||||
# LLM Configuration (Optional)
|
||||
# =============================================================================
|
||||
# Large Language Model settings for AI-powered features
|
||||
# Supports providers like Ollama, OpenAI, Anthropic, etc.
|
||||
# Multiple providers can be configured for automatic rotation on error/rate limit
|
||||
llm_config:
|
||||
# List of LLM providers (rotates to next on error/rate limit)
|
||||
llm_providers:
|
||||
# Primary provider (used first)
|
||||
- provider: ollama
|
||||
base_url: "http://localhost:11434/v1/chat/completions"
|
||||
auth_token: ""
|
||||
model: "gpt-oss:120b-cloud"
|
||||
# Backup provider example (uncomment to enable rotation)
|
||||
# - provider: openai
|
||||
# base_url: "https://api.openai.com/v1/chat/completions"
|
||||
# auth_token: "sk-your-api-key"
|
||||
# model: "gpt-4"
|
||||
|
||||
# Enable LLM tool call features
|
||||
enabled_tool_call: false
|
||||
|
||||
# Maximum number of tokens to generate
|
||||
max_tokens: 1000
|
||||
|
||||
# Temperature for sampling
|
||||
temperature: 0.7
|
||||
|
||||
# Top-k sampling
|
||||
top_k: 50
|
||||
|
||||
# Top-p sampling
|
||||
top_p: 0.9
|
||||
|
||||
# Number of completions to generate
|
||||
n: 1
|
||||
|
||||
# Maximum number of retries for failed requests
|
||||
max_retries: 3
|
||||
|
||||
# Timeout for API requests
|
||||
timeout: 120s
|
||||
|
||||
# Enable streaming responses
|
||||
stream: false
|
||||
|
||||
# Enable structured JSON output format
|
||||
structured_json_format: false
|
||||
|
||||
# System prompt for the LLM
|
||||
system_prompt: ""
|
||||
|
||||
# Custom headers for API requests
|
||||
custom_headers: ""
|
||||
`)
|
||||
|
||||
// generateRandomString generates a random alphanumeric string of the given length
|
||||
func generateRandomString(length int) string {
|
||||
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
b := make([]byte, length)
|
||||
for i := range b {
|
||||
n, _ := rand.Int(rand.Reader, big.NewInt(int64(len(charset))))
|
||||
b[i] = charset[n.Int64()]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// GlobalVar represents a single global variable with optional env export
|
||||
type GlobalVar struct {
|
||||
Value string `yaml:"value"`
|
||||
AsEnv *bool `yaml:"as_env,omitempty"` // pointer to distinguish unset (defaults true) from false
|
||||
}
|
||||
|
||||
// IsAsEnv returns true if this variable should be exported to environment
|
||||
// Defaults to true if not explicitly set
|
||||
func (g GlobalVar) IsAsEnv() bool {
|
||||
if g.AsEnv == nil {
|
||||
return true // default
|
||||
}
|
||||
return *g.AsEnv
|
||||
}
|
||||
|
||||
// GlobalVarsConfig holds all global variables
|
||||
type GlobalVarsConfig map[string]GlobalVar
|
||||
|
||||
// NotificationConfig for multi-provider notifications
|
||||
type NotificationConfig struct {
|
||||
Provider string `yaml:"provider"` // telegram, webhook
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Telegram TelegramConfig `yaml:"telegram,omitempty"`
|
||||
Webhooks []WebhookConfig `yaml:"webhooks,omitempty"` // Multiple webhook endpoints
|
||||
}
|
||||
|
||||
var (
|
||||
globalConfig *Config
|
||||
configMu sync.RWMutex
|
||||
)
|
||||
|
||||
// Config holds the complete application configuration
|
||||
type Config struct {
|
||||
BaseFolder string `yaml:"base_folder"`
|
||||
Environments EnvironmentConfig `yaml:"environments"`
|
||||
Database DatabaseConfig `yaml:"database"`
|
||||
Server ServerConfig `yaml:"server"`
|
||||
ScanTactic ScanTacticConfig `yaml:"scan_tactic"`
|
||||
Redis RedisConfig `yaml:"redis"`
|
||||
GlobalVars GlobalVarsConfig `yaml:"global_vars"`
|
||||
Notification NotificationConfig `yaml:"notification"`
|
||||
Storage StorageConfig `yaml:"storage"`
|
||||
LLM LLMConfig `yaml:"llm_config"`
|
||||
|
||||
// Runtime paths (resolved from templates)
|
||||
BinariesPath string `yaml:"-"`
|
||||
DataPath string `yaml:"-"`
|
||||
ConfigsPath string `yaml:"-"`
|
||||
WorkspacesPath string `yaml:"-"`
|
||||
WorkflowsPath string `yaml:"-"`
|
||||
UIPath string `yaml:"-"` // Resolved UI static files path
|
||||
SnapshotPath string `yaml:"-"` // Resolved snapshot directory path
|
||||
MarkdownReportTemplatesPath string `yaml:"-"` // Resolved markdown report templates path
|
||||
ExternalAgentConfigsPath string `yaml:"-"` // Resolved external agent configs path
|
||||
ExternalScriptsPath string `yaml:"-"` // Resolved external scripts path
|
||||
}
|
||||
|
||||
// EnvironmentConfig holds environment path configurations
|
||||
type EnvironmentConfig struct {
|
||||
ExternalBinariesPath string `yaml:"external_binaries_path"`
|
||||
ExternalData string `yaml:"external_data"`
|
||||
ExternalConfigs string `yaml:"external_configs"`
|
||||
Workspaces string `yaml:"workspaces"`
|
||||
Workflows string `yaml:"workflows"`
|
||||
Snapshot string `yaml:"snapshot"`
|
||||
MarkdownReportTemplates string `yaml:"markdown_report_templates"`
|
||||
ExternalAgentConfigs string `yaml:"external_agent_configs"`
|
||||
ExternalScripts string `yaml:"external_scripts"`
|
||||
}
|
||||
|
||||
// DatabaseConfig holds database connection settings
|
||||
type DatabaseConfig struct {
|
||||
DBEngine string `yaml:"db_engine"` // sqlite, postgresql
|
||||
DBPath string `yaml:"db_path"` // SQLite file path
|
||||
Host string `yaml:"host"`
|
||||
Port int `yaml:"port"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
DBName string `yaml:"db_name"`
|
||||
ConnectionTimeout int `yaml:"connection_timeout"`
|
||||
SSLMode string `yaml:"ssl_mode"`
|
||||
}
|
||||
|
||||
// ServerConfig holds API server settings
|
||||
type ServerConfig struct {
|
||||
Host string `yaml:"host"`
|
||||
Port int `yaml:"port"`
|
||||
UIPath string `yaml:"ui_path"` // Path to serve static UI files
|
||||
WorkspacePrefixKey string `yaml:"workspace_prefix_key"` // Random prefix for workspace static files (16 chars)
|
||||
SimpleUserMapKey map[string]string `yaml:"simple_user_map_key"` // Map of username:password for authentication
|
||||
JWT JWTConfig `yaml:"jwt"` // JWT settings
|
||||
License string `yaml:"license"` // License type shown in ServerHeader and /server-info
|
||||
EnabledAuthAPI bool `yaml:"enabled_auth_api"` // Enable API key authentication (default: false)
|
||||
AuthAPIKey string `yaml:"auth_api_key"` // API key for x-osm-api-key header authentication
|
||||
}
|
||||
|
||||
// ScanTacticConfig holds scan aggressiveness levels
|
||||
type ScanTacticConfig struct {
|
||||
Aggressive int `yaml:"aggressive"`
|
||||
Default int `yaml:"default"`
|
||||
Gently int `yaml:"gently"`
|
||||
}
|
||||
|
||||
// JWTConfig holds JWT settings
|
||||
type JWTConfig struct {
|
||||
SecretSigningKey string `yaml:"secret_signing_key"`
|
||||
ExpirationMinutes int `yaml:"expiration_minutes"`
|
||||
}
|
||||
|
||||
// RedisConfig holds Redis connection settings for distributed mode
|
||||
type RedisConfig struct {
|
||||
Host string `yaml:"host"`
|
||||
Port int `yaml:"port"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
DB int `yaml:"db"`
|
||||
ConnectionTimeout int `yaml:"connection_timeout"`
|
||||
}
|
||||
|
||||
// TelegramConfig holds Telegram bot settings
|
||||
type TelegramConfig struct {
|
||||
BotToken string `yaml:"bot_token"`
|
||||
ChatID int64 `yaml:"chat_id"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// WebhookConfig holds configuration for a single webhook endpoint
|
||||
type WebhookConfig struct {
|
||||
URL string `yaml:"url"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Headers map[string]string `yaml:"headers,omitempty"`
|
||||
Timeout int `yaml:"timeout,omitempty"` // seconds, default 30
|
||||
RetryCount int `yaml:"retry_count,omitempty"` // default 3
|
||||
SkipTLSVerify bool `yaml:"skip_tls_verify,omitempty"` // default false
|
||||
Events []string `yaml:"events,omitempty"` // scan_complete, scan_failed, step_failed, etc.
|
||||
}
|
||||
|
||||
// StorageConfig holds cloud storage settings (S3-compatible)
|
||||
type StorageConfig struct {
|
||||
Provider string `yaml:"provider"` // s3, minio, gcs, etc.
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
AccessKeyID string `yaml:"access_key_id"`
|
||||
SecretAccessKey string `yaml:"secret_access_key"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
Region string `yaml:"region"`
|
||||
UseSSL bool `yaml:"use_ssl"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// LLMProvider holds configuration for a single LLM provider endpoint
|
||||
type LLMProvider struct {
|
||||
Provider string `yaml:"provider"` // ollama, openai, anthropic, custom, etc.
|
||||
BaseURL string `yaml:"base_url"` // API endpoint URL
|
||||
AuthToken string `yaml:"auth_token"` // Authentication token (can be blank for local Ollama)
|
||||
Model string `yaml:"model"` // Model name/ID
|
||||
}
|
||||
|
||||
// LLMConfig holds LLM (Large Language Model) settings
|
||||
type LLMConfig struct {
|
||||
LLMProviders []LLMProvider `yaml:"llm_providers"` // List of LLM providers for rotation
|
||||
EnabledToolCall bool `yaml:"enabled_tool_call"` // Enable LLM tool call features
|
||||
MaxTokens int `yaml:"max_tokens"` // Maximum number of tokens to generate
|
||||
Temperature float64 `yaml:"temperature"` // Temperature for sampling
|
||||
TopK int `yaml:"top_k"` // Top-k sampling
|
||||
TopP float64 `yaml:"top_p"` // Top-p sampling
|
||||
N int `yaml:"n"` // Number of completions to generate
|
||||
MaxRetries int `yaml:"max_retries"` // Maximum number of retries for failed requests
|
||||
Timeout string `yaml:"timeout"` // Timeout for API requests
|
||||
Stream bool `yaml:"stream"` // Enable streaming responses
|
||||
StructuredJSONFormat bool `yaml:"structured_json_format"` // Enable structured JSON output format
|
||||
SystemPrompt string `yaml:"system_prompt"` // System prompt for the LLM
|
||||
CustomHeaders string `yaml:"custom_headers"` // Custom headers for API requests
|
||||
|
||||
// Internal fields for provider rotation (not serialized)
|
||||
currentIndex int // Current provider index
|
||||
mu sync.Mutex // Mutex for thread-safe rotation
|
||||
}
|
||||
|
||||
// Load loads configuration from the specified base folder
|
||||
func Load(baseFolder string) (*Config, error) {
|
||||
settingsPath := filepath.Join(baseFolder, "osm-settings.yaml")
|
||||
cfg, err := LoadFromFile(settingsPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Override base folder if provided
|
||||
if baseFolder != "" {
|
||||
cfg.BaseFolder = baseFolder
|
||||
}
|
||||
|
||||
// Resolve environment paths
|
||||
cfg.ResolvePaths()
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// LoadFromFile loads configuration from a specific file
|
||||
func LoadFromFile(path string) (*Config, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ParseConfig(data)
|
||||
}
|
||||
|
||||
// LoadFromBytes loads configuration from raw YAML bytes
|
||||
func LoadFromBytes(data []byte) (*Config, error) {
|
||||
return ParseConfig(data)
|
||||
}
|
||||
|
||||
// ResolvePaths resolves template variables in environment paths.
|
||||
// This method should be called after changing BaseFolder to recalculate all derived paths.
|
||||
func (c *Config) ResolvePaths() {
|
||||
baseFolder := c.resolveEnvVars(c.BaseFolder)
|
||||
c.BaseFolder = baseFolder
|
||||
|
||||
// Resolve each path with base_folder substitution
|
||||
c.BinariesPath = c.resolvePath(c.Environments.ExternalBinariesPath, baseFolder)
|
||||
c.DataPath = c.resolvePath(c.Environments.ExternalData, baseFolder)
|
||||
c.ConfigsPath = c.resolvePath(c.Environments.ExternalConfigs, baseFolder)
|
||||
c.WorkspacesPath = c.resolvePath(c.Environments.Workspaces, baseFolder)
|
||||
c.WorkflowsPath = c.resolvePath(c.Environments.Workflows, baseFolder)
|
||||
|
||||
// Resolve server paths
|
||||
c.UIPath = c.resolvePath(c.Server.UIPath, baseFolder)
|
||||
|
||||
// Resolve snapshot path
|
||||
c.SnapshotPath = c.resolvePath(c.Environments.Snapshot, baseFolder)
|
||||
|
||||
// Resolve markdown report templates path
|
||||
c.MarkdownReportTemplatesPath = c.resolvePath(c.Environments.MarkdownReportTemplates, baseFolder)
|
||||
|
||||
// Resolve external agent configs path
|
||||
c.ExternalAgentConfigsPath = c.resolvePath(c.Environments.ExternalAgentConfigs, baseFolder)
|
||||
|
||||
// Resolve external scripts path
|
||||
c.ExternalScriptsPath = c.resolvePath(c.Environments.ExternalScripts, baseFolder)
|
||||
}
|
||||
|
||||
// resolvePath resolves a single path with variable substitution
|
||||
func (c *Config) resolvePath(path, baseFolder string) string {
|
||||
if path == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Replace {{base_folder}} template variable
|
||||
resolved := strings.ReplaceAll(path, "{{base_folder}}", baseFolder)
|
||||
|
||||
// Resolve environment variables like $HOME
|
||||
resolved = c.resolveEnvVars(resolved)
|
||||
|
||||
return resolved
|
||||
}
|
||||
|
||||
// resolveEnvVars resolves environment variables in a string
|
||||
func (c *Config) resolveEnvVars(s string) string {
|
||||
return os.ExpandEnv(s)
|
||||
}
|
||||
|
||||
// GetWorkflowsDir returns the workflows directory path
|
||||
func (c *Config) GetWorkflowsDir() string {
|
||||
return c.WorkflowsPath
|
||||
}
|
||||
|
||||
// GetModulesDir returns the modules directory path
|
||||
func (c *Config) GetModulesDir() string {
|
||||
return filepath.Join(c.WorkflowsPath, "modules")
|
||||
}
|
||||
|
||||
// GetWorkspacesDir returns the workspaces directory path
|
||||
func (c *Config) GetWorkspacesDir() string {
|
||||
return c.WorkspacesPath
|
||||
}
|
||||
|
||||
// GetDBPath returns the resolved database file path for SQLite
|
||||
func (c *Config) GetDBPath() string {
|
||||
if c.Database.DBPath == "" {
|
||||
return filepath.Join(c.BaseFolder, "database-osm.sqlite")
|
||||
}
|
||||
return c.resolvePath(c.Database.DBPath, c.BaseFolder)
|
||||
}
|
||||
|
||||
// IsSQLite returns true if the database engine is SQLite
|
||||
func (c *Config) IsSQLite() bool {
|
||||
return c.Database.DBEngine == "" || c.Database.DBEngine == "sqlite"
|
||||
}
|
||||
|
||||
// IsPostgres returns true if the database engine is PostgreSQL
|
||||
func (c *Config) IsPostgres() bool {
|
||||
return c.Database.DBEngine == "postgresql" || c.Database.DBEngine == "postgres"
|
||||
}
|
||||
|
||||
// IsRedisConfigured returns true if Redis is configured
|
||||
func (c *Config) IsRedisConfigured() bool {
|
||||
return c.Redis.Host != "" && c.Redis.Port > 0
|
||||
}
|
||||
|
||||
// GetRedisAddr returns the Redis address in host:port format
|
||||
func (c *Config) GetRedisAddr() string {
|
||||
return fmt.Sprintf("%s:%d", c.Redis.Host, c.Redis.Port)
|
||||
}
|
||||
|
||||
// GetDSN returns the PostgreSQL connection string
|
||||
func (c *Config) GetDSN() string {
|
||||
sslMode := c.Database.SSLMode
|
||||
if sslMode == "" {
|
||||
sslMode = "disable"
|
||||
}
|
||||
return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s",
|
||||
c.Database.Username, c.Database.Password,
|
||||
c.Database.Host, c.Database.Port,
|
||||
c.Database.DBName, sslMode)
|
||||
}
|
||||
|
||||
// IsNotificationConfigured returns true if any notification provider is configured and enabled
|
||||
func (c *Config) IsNotificationConfigured() bool {
|
||||
if !c.Notification.Enabled {
|
||||
return false
|
||||
}
|
||||
switch c.Notification.Provider {
|
||||
case "telegram":
|
||||
return c.Notification.Telegram.BotToken != "" && c.Notification.Telegram.ChatID != 0
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// IsTelegramConfigured returns true if Telegram is configured and enabled
|
||||
// Deprecated: Use IsNotificationConfigured instead
|
||||
func (c *Config) IsTelegramConfigured() bool {
|
||||
return c.Notification.Provider == "telegram" &&
|
||||
c.Notification.Enabled &&
|
||||
c.Notification.Telegram.BotToken != "" &&
|
||||
c.Notification.Telegram.ChatID != 0
|
||||
}
|
||||
|
||||
// GetGlobalVar returns a global variable value by name
|
||||
func (c *Config) GetGlobalVar(name string) (string, bool) {
|
||||
if c.GlobalVars == nil {
|
||||
return "", false
|
||||
}
|
||||
if v, ok := c.GlobalVars[name]; ok {
|
||||
return v.Value, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// ExportGlobalVarsToEnv exports variables with as_env=true to environment
|
||||
// Variable names are converted to UPPERCASE_WITH_UNDERSCORES
|
||||
func (c *Config) ExportGlobalVarsToEnv() {
|
||||
if c.GlobalVars == nil {
|
||||
return
|
||||
}
|
||||
for name, v := range c.GlobalVars {
|
||||
if v.IsAsEnv() && v.Value != "" {
|
||||
envName := strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
|
||||
_ = os.Setenv(envName, v.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetAllGlobalVars returns all global vars as map[string]string for templates
|
||||
func (c *Config) GetAllGlobalVars() map[string]string {
|
||||
result := make(map[string]string)
|
||||
if c.GlobalVars == nil {
|
||||
return result
|
||||
}
|
||||
for name, v := range c.GlobalVars {
|
||||
result[name] = v.Value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// IsStorageConfigured returns true if cloud storage is configured and enabled
|
||||
func (c *Config) IsStorageConfigured() bool {
|
||||
return c.Storage.Enabled && c.Storage.Endpoint != "" && c.Storage.Bucket != ""
|
||||
}
|
||||
|
||||
// IsLLMConfigured returns true if LLM is configured and enabled
|
||||
func (c *Config) IsLLMConfigured() bool {
|
||||
return c.LLM.EnabledToolCall && len(c.LLM.LLMProviders) > 0
|
||||
}
|
||||
|
||||
// GetCurrentProvider returns the current active LLM provider (thread-safe)
|
||||
// Returns nil if no providers are configured
|
||||
func (l *LLMConfig) GetCurrentProvider() *LLMProvider {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if len(l.LLMProviders) == 0 {
|
||||
return nil
|
||||
}
|
||||
return &l.LLMProviders[l.currentIndex]
|
||||
}
|
||||
|
||||
// RotateProvider advances to the next LLM provider (thread-safe, wraps around)
|
||||
// Returns the new current provider, or nil if no providers are configured
|
||||
func (l *LLMConfig) RotateProvider() *LLMProvider {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if len(l.LLMProviders) == 0 {
|
||||
return nil
|
||||
}
|
||||
l.currentIndex = (l.currentIndex + 1) % len(l.LLMProviders)
|
||||
return &l.LLMProviders[l.currentIndex]
|
||||
}
|
||||
|
||||
// ResetProviderIndex resets the current provider to the first one (thread-safe)
|
||||
func (l *LLMConfig) ResetProviderIndex() {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.currentIndex = 0
|
||||
}
|
||||
|
||||
// GetProviderCount returns the number of configured LLM providers
|
||||
func (l *LLMConfig) GetProviderCount() int {
|
||||
return len(l.LLMProviders)
|
||||
}
|
||||
|
||||
// GetCurrentProviderIndex returns the current provider index (thread-safe)
|
||||
func (l *LLMConfig) GetCurrentProviderIndex() int {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.currentIndex
|
||||
}
|
||||
|
||||
// ResolveServerCredentials adds credentials from environment variables
|
||||
// OSM_USERNAME and OSM_PASSWORD are added to the user map if both are set
|
||||
func (c *Config) ResolveServerCredentials() {
|
||||
envUser := os.Getenv("OSM_USERNAME")
|
||||
envPass := os.Getenv("OSM_PASSWORD")
|
||||
if envUser != "" && envPass != "" {
|
||||
if c.Server.SimpleUserMapKey == nil {
|
||||
c.Server.SimpleUserMapKey = make(map[string]string)
|
||||
}
|
||||
c.Server.SimpleUserMapKey[envUser] = envPass
|
||||
}
|
||||
}
|
||||
|
||||
// GetThreads returns thread counts for the given scan tactic
|
||||
// Returns (threads, baseThreads) where baseThreads is half of threads
|
||||
func (c *Config) GetThreads(tactic string) (int, int) {
|
||||
var threads int
|
||||
switch tactic {
|
||||
case "aggressive", "fast":
|
||||
threads = c.ScanTactic.Aggressive
|
||||
case "gently", "thorough":
|
||||
threads = c.ScanTactic.Gently
|
||||
default: // normal, default
|
||||
threads = c.ScanTactic.Default
|
||||
}
|
||||
if threads <= 0 {
|
||||
threads = 10 // fallback default
|
||||
}
|
||||
baseThreads := threads / 2
|
||||
if baseThreads < 1 {
|
||||
baseThreads = 1
|
||||
}
|
||||
return threads, baseThreads
|
||||
}
|
||||
|
||||
// Set sets the global configuration
|
||||
func Set(cfg *Config) {
|
||||
configMu.Lock()
|
||||
defer configMu.Unlock()
|
||||
globalConfig = cfg
|
||||
}
|
||||
|
||||
// Get returns the global configuration
|
||||
func Get() *Config {
|
||||
configMu.RLock()
|
||||
defer configMu.RUnlock()
|
||||
return globalConfig
|
||||
}
|
||||
|
||||
// DefaultConfig returns a default configuration
|
||||
func DefaultConfig() *Config {
|
||||
homeDir, _ := os.UserHomeDir()
|
||||
baseFolder := filepath.Join(homeDir, "osmedeus-base")
|
||||
|
||||
return &Config{
|
||||
BaseFolder: baseFolder,
|
||||
Environments: EnvironmentConfig{
|
||||
ExternalBinariesPath: "{{base_folder}}/external-binaries",
|
||||
ExternalData: "{{base_folder}}/external-data",
|
||||
ExternalConfigs: "{{base_folder}}/external-configs",
|
||||
Workspaces: "{{base_folder}}/workspaces",
|
||||
Workflows: "{{base_folder}}/workflows",
|
||||
Snapshot: "{{base_folder}}/snapshot",
|
||||
ExternalScripts: "{{base_folder}}/external-scripts",
|
||||
},
|
||||
Database: DatabaseConfig{
|
||||
DBEngine: "sqlite",
|
||||
DBPath: "{{base_folder}}/database-osm.sqlite",
|
||||
Host: "localhost",
|
||||
Port: 5432,
|
||||
Username: "osmedeus",
|
||||
Password: "osmedeus",
|
||||
DBName: "osmedeus",
|
||||
ConnectionTimeout: 60,
|
||||
SSLMode: "disable",
|
||||
},
|
||||
Server: ServerConfig{
|
||||
Host: "0.0.0.0",
|
||||
Port: 8002,
|
||||
UIPath: "{{base_folder}}/ui/",
|
||||
WorkspacePrefixKey: generateRandomString(16),
|
||||
SimpleUserMapKey: map[string]string{
|
||||
"osmedeus": "osmedeus-admin",
|
||||
},
|
||||
JWT: JWTConfig{
|
||||
SecretSigningKey: "change-this-secret-in-production",
|
||||
ExpirationMinutes: 60,
|
||||
},
|
||||
License: "open-source",
|
||||
},
|
||||
ScanTactic: ScanTacticConfig{
|
||||
Aggressive: 40,
|
||||
Default: 10,
|
||||
Gently: 5,
|
||||
},
|
||||
Redis: RedisConfig{
|
||||
Host: "",
|
||||
Port: 6379,
|
||||
Username: "",
|
||||
Password: "",
|
||||
DB: 0,
|
||||
ConnectionTimeout: 60,
|
||||
},
|
||||
GlobalVars: GlobalVarsConfig{
|
||||
"GITHUB_API_KEY": {Value: ""},
|
||||
"SHODAN_API_KEY": {Value: ""},
|
||||
"CENSYS_API_KEY": {Value: ""},
|
||||
"PASSIVETOTAL_API_KEY": {Value: ""},
|
||||
// Add more default placeholders as needed
|
||||
},
|
||||
Notification: NotificationConfig{
|
||||
Provider: "telegram",
|
||||
Enabled: false,
|
||||
Telegram: TelegramConfig{
|
||||
BotToken: "",
|
||||
ChatID: 0,
|
||||
Enabled: false,
|
||||
},
|
||||
},
|
||||
Storage: StorageConfig{
|
||||
Provider: "s3",
|
||||
Endpoint: "",
|
||||
AccessKeyID: "",
|
||||
SecretAccessKey: "",
|
||||
Bucket: "",
|
||||
Region: "us-east-1",
|
||||
UseSSL: true,
|
||||
Enabled: false,
|
||||
},
|
||||
LLM: LLMConfig{
|
||||
LLMProviders: []LLMProvider{
|
||||
{
|
||||
Provider: "ollama",
|
||||
BaseURL: "http://localhost:11434/v1/chat/completions",
|
||||
AuthToken: "",
|
||||
Model: "gpt-oss:120b-cloud",
|
||||
},
|
||||
},
|
||||
EnabledToolCall: false,
|
||||
MaxTokens: 1000,
|
||||
Temperature: 0.7,
|
||||
TopK: 50,
|
||||
TopP: 0.9,
|
||||
N: 1,
|
||||
MaxRetries: 3,
|
||||
Timeout: "120s",
|
||||
Stream: false,
|
||||
StructuredJSONFormat: false,
|
||||
SystemPrompt: "",
|
||||
CustomHeaders: "",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureConfigExists creates osm-settings.yaml if it doesn't exist
|
||||
// Uses the embedded example configuration file as template
|
||||
func EnsureConfigExists(baseFolder string) error {
|
||||
settingsPath := filepath.Join(baseFolder, "osm-settings.yaml")
|
||||
|
||||
// Check if file already exists
|
||||
if _, err := os.Stat(settingsPath); err == nil {
|
||||
return nil // File exists, nothing to do
|
||||
}
|
||||
|
||||
// Create base folder if needed
|
||||
if err := os.MkdirAll(baseFolder, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Generate random workspace_prefix_key and replace blank value in template
|
||||
configContent := string(exampleConfigYAML)
|
||||
configContent = strings.Replace(configContent,
|
||||
"workspace_prefix_key: \"\"",
|
||||
fmt.Sprintf("workspace_prefix_key: \"%s\"", generateRandomString(16)),
|
||||
1)
|
||||
|
||||
// Write the config file with generated values
|
||||
return os.WriteFile(settingsPath, []byte(configContent), 0644)
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"github.com/goccy/go-yaml"
|
||||
)
|
||||
|
||||
// ParseConfig parses configuration from YAML bytes
|
||||
func ParseConfig(data []byte) (*Config, error) {
|
||||
var cfg Config
|
||||
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// ParseConfigStrict parses configuration with strict validation
|
||||
func ParseConfigStrict(data []byte) (*Config, error) {
|
||||
var cfg Config
|
||||
if err := yaml.UnmarshalWithOptions(data, &cfg, yaml.Strict()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// ToYAML serializes the config to YAML bytes
|
||||
func (c *Config) ToYAML() ([]byte, error) {
|
||||
return yaml.Marshal(c)
|
||||
}
|
||||
|
||||
// Validate validates the configuration
|
||||
func (c *Config) Validate() error {
|
||||
// Validate required fields
|
||||
if c.BaseFolder == "" {
|
||||
return &ConfigError{Field: "base_folder", Message: "base_folder is required"}
|
||||
}
|
||||
|
||||
// Validate database config if server mode is expected
|
||||
if c.Database.Host == "" {
|
||||
c.Database.Host = "localhost"
|
||||
}
|
||||
if c.Database.Port == 0 {
|
||||
c.Database.Port = 5432
|
||||
}
|
||||
|
||||
// Validate server config
|
||||
if c.Server.Port == 0 {
|
||||
c.Server.Port = 8002
|
||||
}
|
||||
|
||||
// Set defaults for scan tactics
|
||||
if c.ScanTactic.Default == 0 {
|
||||
c.ScanTactic.Default = 10
|
||||
}
|
||||
if c.ScanTactic.Aggressive == 0 {
|
||||
c.ScanTactic.Aggressive = 40
|
||||
}
|
||||
if c.ScanTactic.Gently == 0 {
|
||||
c.ScanTactic.Gently = 5
|
||||
}
|
||||
|
||||
// Set defaults for JWT
|
||||
if c.Server.JWT.ExpirationMinutes == 0 {
|
||||
c.Server.JWT.ExpirationMinutes = 60
|
||||
}
|
||||
|
||||
// Set default for snapshot path
|
||||
if c.Environments.Snapshot == "" {
|
||||
c.Environments.Snapshot = "{{base_folder}}/snapshot"
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConfigError represents a configuration validation error
|
||||
type ConfigError struct {
|
||||
Field string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *ConfigError) Error() string {
|
||||
return "config error: " + e.Field + " - " + e.Message
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Capture manages console output capture to file while maintaining terminal display
|
||||
type Capture struct {
|
||||
mu sync.Mutex
|
||||
file *os.File
|
||||
originalOut *os.File
|
||||
originalErr *os.File
|
||||
outWriter *os.File
|
||||
errWriter *os.File
|
||||
outReader *os.File
|
||||
errReader *os.File
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// StartCapture begins capturing stdout/stderr to the specified file
|
||||
func StartCapture(filePath string) (*Capture, error) {
|
||||
// Ensure directory exists
|
||||
if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Open file for writing
|
||||
file, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c := &Capture{
|
||||
file: file,
|
||||
originalOut: os.Stdout,
|
||||
originalErr: os.Stderr,
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
|
||||
// Create pipes for stdout and stderr
|
||||
outReader, outWriter, err := os.Pipe()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
errReader, errWriter, err := os.Pipe()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
outReader.Close()
|
||||
outWriter.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.outWriter = outWriter
|
||||
c.errWriter = errWriter
|
||||
c.outReader = outReader
|
||||
c.errReader = errReader
|
||||
|
||||
// Replace stdout/stderr
|
||||
os.Stdout = outWriter
|
||||
os.Stderr = errWriter
|
||||
|
||||
// Tee goroutines - write to both terminal and file
|
||||
c.wg.Add(2)
|
||||
go c.tee(outReader, c.originalOut)
|
||||
go c.tee(errReader, c.originalErr)
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Capture) tee(reader *os.File, terminal *os.File) {
|
||||
defer c.wg.Done()
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
select {
|
||||
case <-c.done:
|
||||
// Drain any remaining data
|
||||
c.drainReader(reader, terminal, buf)
|
||||
return
|
||||
default:
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
data := buf[:n]
|
||||
_, _ = terminal.Write(data)
|
||||
c.mu.Lock()
|
||||
if c.file != nil {
|
||||
_, _ = c.file.Write(data)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Capture) drainReader(reader *os.File, terminal *os.File, buf []byte) {
|
||||
for {
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
data := buf[:n]
|
||||
_, _ = terminal.Write(data)
|
||||
c.mu.Lock()
|
||||
if c.file != nil {
|
||||
_, _ = c.file.Write(data)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WriteToFile writes content directly to the capture file without printing to terminal
|
||||
// This is useful for writing verbose output that should only appear in the log file
|
||||
func (c *Capture) WriteToFile(content string) {
|
||||
if content == "" {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.file != nil {
|
||||
_, _ = c.file.WriteString(content)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop restores original stdout/stderr and closes the file
|
||||
func (c *Capture) Stop() error {
|
||||
// Guard against nil receiver
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Signal done to tee goroutines
|
||||
close(c.done)
|
||||
|
||||
// Close pipe writers to signal EOF to tee goroutines
|
||||
if c.outWriter != nil {
|
||||
c.outWriter.Close()
|
||||
}
|
||||
if c.errWriter != nil {
|
||||
c.errWriter.Close()
|
||||
}
|
||||
|
||||
// Restore original stdout/stderr immediately
|
||||
os.Stdout = c.originalOut
|
||||
os.Stderr = c.originalErr
|
||||
|
||||
// Wait for tee goroutines to finish
|
||||
c.wg.Wait()
|
||||
|
||||
// Close readers
|
||||
if c.outReader != nil {
|
||||
c.outReader.Close()
|
||||
}
|
||||
if c.errReader != nil {
|
||||
c.errReader.Close()
|
||||
}
|
||||
|
||||
// Close file
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.file != nil {
|
||||
_ = c.file.Sync()
|
||||
c.file.Close()
|
||||
c.file = nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package core
|
||||
|
||||
// Project metadata constants
|
||||
const (
|
||||
// VERSION of this project
|
||||
VERSION = "v5.0.0-beta"
|
||||
// DESC description of the tool
|
||||
DESC = "A Modern Orchestration Engine for Security"
|
||||
// BINARY name of osmedeus
|
||||
BINARY = "osmedeus"
|
||||
// SNAPSHOT binary name of osmedeus
|
||||
SNAPSHOT = "osm"
|
||||
// AUTHOR of this
|
||||
AUTHOR = "@j3ssie"
|
||||
// DOCS private document
|
||||
DOCS = "https://docs.osmedeus.org"
|
||||
// DOCS private document
|
||||
LICENSE = "open-source"
|
||||
// REPO_URL private document
|
||||
REPO_URL = "https://github.com/j3ssie/osmedeus"
|
||||
// DEFAULT_BASE_REPO default repository for base folder
|
||||
DEFAULT_BASE_REPO = "https://github.com/osmedeus/osmedeus-base.git"
|
||||
// DEFAULT_WORKFLOW_REPO default repository for workflows
|
||||
DEFAULT_WORKFLOW_REPO = "https://github.com/osmedeus/osmedeus-workflow.git"
|
||||
// METADATA domain for checking update
|
||||
METADATA = "https://metadata.osmedeus.org"
|
||||
// INSTALL default install script
|
||||
INSTALL = "https://raw.githubusercontent.com/osmedeus/osmedeus-base/master/install.sh"
|
||||
// DefaultUA is the default User-Agent for HTTP clients
|
||||
DefaultUA = "Mozilla/5.0 (compatible; Osmedeus/" + VERSION + "; +" + REPO_URL + ")"
|
||||
)
|
||||
@@ -0,0 +1,225 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// ExecutionContext holds runtime state for workflow execution
|
||||
type ExecutionContext struct {
|
||||
WorkflowName string
|
||||
WorkflowKind WorkflowKind
|
||||
RunID string
|
||||
Target string
|
||||
WorkspacePath string
|
||||
BaseFolder string
|
||||
|
||||
// Params are the input parameters (immutable after init)
|
||||
Params map[string]interface{}
|
||||
|
||||
// Exports are variables exported by steps (mutable)
|
||||
Exports map[string]interface{}
|
||||
|
||||
// Variables combines Params and Exports for template rendering
|
||||
Variables map[string]interface{}
|
||||
|
||||
// Logger for this execution
|
||||
Logger *zap.Logger
|
||||
|
||||
// StepIndex tracks the current step number (for display purposes)
|
||||
StepIndex int
|
||||
|
||||
// WorkspaceName is the workspace identifier for database operations
|
||||
WorkspaceName string
|
||||
|
||||
// mu protects concurrent access to Exports and Variables
|
||||
mu sync.RWMutex
|
||||
|
||||
// variablesSnapshot provides O(1) read access for GetVariables()
|
||||
// Updated atomically on SetVariable/SetExport/MergeExports/SetParam
|
||||
variablesSnapshot atomic.Value // map[string]interface{}
|
||||
}
|
||||
|
||||
// NewExecutionContext creates a new execution context
|
||||
func NewExecutionContext(workflowName string, kind WorkflowKind, runID, target string) *ExecutionContext {
|
||||
return &ExecutionContext{
|
||||
WorkflowName: workflowName,
|
||||
WorkflowKind: kind,
|
||||
RunID: runID,
|
||||
Target: target,
|
||||
Params: make(map[string]interface{}),
|
||||
Exports: make(map[string]interface{}),
|
||||
Variables: make(map[string]interface{}),
|
||||
}
|
||||
}
|
||||
|
||||
// updateSnapshot creates an immutable copy of Variables for fast reads
|
||||
// Must be called with c.mu held (Lock, not RLock)
|
||||
func (c *ExecutionContext) updateSnapshot() {
|
||||
snapshot := make(map[string]interface{}, len(c.Variables))
|
||||
for k, v := range c.Variables {
|
||||
snapshot[k] = v
|
||||
}
|
||||
c.variablesSnapshot.Store(snapshot)
|
||||
}
|
||||
|
||||
// SetParam sets a parameter value
|
||||
func (c *ExecutionContext) SetParam(key string, value interface{}) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.Params[key] = value
|
||||
c.Variables[key] = value
|
||||
c.updateSnapshot()
|
||||
}
|
||||
|
||||
// GetParam gets a parameter value
|
||||
func (c *ExecutionContext) GetParam(key string) (interface{}, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
v, ok := c.Params[key]
|
||||
return v, ok
|
||||
}
|
||||
|
||||
// SetExport sets an exported variable
|
||||
func (c *ExecutionContext) SetExport(key string, value interface{}) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.Exports[key] = value
|
||||
c.Variables[key] = value
|
||||
c.updateSnapshot()
|
||||
}
|
||||
|
||||
// GetExport gets an exported variable
|
||||
func (c *ExecutionContext) GetExport(key string) (interface{}, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
v, ok := c.Exports[key]
|
||||
return v, ok
|
||||
}
|
||||
|
||||
// GetVariable gets a variable (param or export)
|
||||
func (c *ExecutionContext) GetVariable(key string) (interface{}, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
v, ok := c.Variables[key]
|
||||
return v, ok
|
||||
}
|
||||
|
||||
// SetVariable sets a variable
|
||||
func (c *ExecutionContext) SetVariable(key string, value interface{}) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.Variables[key] = value
|
||||
c.updateSnapshot()
|
||||
}
|
||||
|
||||
// GetVariables returns all variables for template rendering.
|
||||
// Uses atomic snapshot for O(1) read performance.
|
||||
func (c *ExecutionContext) GetVariables() map[string]interface{} {
|
||||
// Fast path: return cached snapshot (no lock needed)
|
||||
if snapshot := c.variablesSnapshot.Load(); snapshot != nil {
|
||||
return snapshot.(map[string]interface{})
|
||||
}
|
||||
// Fallback for uninitialized contexts (shouldn't happen in normal use)
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
vars := make(map[string]interface{}, len(c.Variables))
|
||||
for k, v := range c.Variables {
|
||||
vars[k] = v
|
||||
}
|
||||
return vars
|
||||
}
|
||||
|
||||
// MergeExports merges exports from a step result
|
||||
func (c *ExecutionContext) MergeExports(exports map[string]interface{}) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for k, v := range exports {
|
||||
c.Exports[k] = v
|
||||
c.Variables[k] = v
|
||||
}
|
||||
c.updateSnapshot()
|
||||
}
|
||||
|
||||
// Clone creates a shallow copy of the context for child execution
|
||||
func (c *ExecutionContext) Clone() *ExecutionContext {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
clone := &ExecutionContext{
|
||||
WorkflowName: c.WorkflowName,
|
||||
WorkflowKind: c.WorkflowKind,
|
||||
RunID: c.RunID,
|
||||
Target: c.Target,
|
||||
WorkspacePath: c.WorkspacePath,
|
||||
BaseFolder: c.BaseFolder,
|
||||
WorkspaceName: c.WorkspaceName,
|
||||
Params: make(map[string]interface{}, len(c.Params)),
|
||||
Exports: make(map[string]interface{}, len(c.Exports)),
|
||||
Variables: make(map[string]interface{}, len(c.Variables)),
|
||||
Logger: c.Logger,
|
||||
}
|
||||
|
||||
for k, v := range c.Params {
|
||||
clone.Params[k] = v
|
||||
}
|
||||
for k, v := range c.Exports {
|
||||
clone.Exports[k] = v
|
||||
}
|
||||
for k, v := range c.Variables {
|
||||
clone.Variables[k] = v
|
||||
}
|
||||
|
||||
// Initialize snapshot for fast GetVariables() reads
|
||||
clone.updateSnapshot()
|
||||
|
||||
return clone
|
||||
}
|
||||
|
||||
// CloneForLoop creates an optimized clone for foreach/parallel iterations.
|
||||
// Key optimizations:
|
||||
// - Shares Params reference (documented as immutable after init)
|
||||
// - Pre-sets loop variables to avoid separate SetVariable calls
|
||||
// - Reduces map copy overhead by ~33% (skips Params copy)
|
||||
func (c *ExecutionContext) CloneForLoop(loopVar string, loopValue interface{}, iterID int) *ExecutionContext {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
// Estimate capacity: parent variables + 2 loop variables
|
||||
varCapacity := len(c.Variables) + 2
|
||||
|
||||
clone := &ExecutionContext{
|
||||
WorkflowName: c.WorkflowName,
|
||||
WorkflowKind: c.WorkflowKind,
|
||||
RunID: c.RunID,
|
||||
Target: c.Target,
|
||||
WorkspacePath: c.WorkspacePath,
|
||||
BaseFolder: c.BaseFolder,
|
||||
WorkspaceName: c.WorkspaceName,
|
||||
Logger: c.Logger,
|
||||
// Share immutable Params reference (no copy needed)
|
||||
Params: c.Params,
|
||||
// Fresh exports map for this iteration
|
||||
Exports: make(map[string]interface{}, 4),
|
||||
// Variables map with pre-allocated capacity
|
||||
Variables: make(map[string]interface{}, varCapacity),
|
||||
}
|
||||
|
||||
// Copy parent Variables for template rendering
|
||||
for k, v := range c.Variables {
|
||||
clone.Variables[k] = v
|
||||
}
|
||||
|
||||
// Pre-set loop variables (avoids separate SetVariable calls)
|
||||
if loopVar != "" {
|
||||
clone.Variables[loopVar] = loopValue
|
||||
}
|
||||
clone.Variables["_id_"] = iterID
|
||||
|
||||
// Initialize snapshot for fast GetVariables() reads
|
||||
clone.updateSnapshot()
|
||||
|
||||
return clone
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Dependencies defines workflow requirements
|
||||
type Dependencies struct {
|
||||
Commands []string `yaml:"commands"`
|
||||
Files []string `yaml:"files"`
|
||||
Variables []VariableDep `yaml:"variables"`
|
||||
TargetTypes []TargetType `yaml:"target_types"`
|
||||
FunctionsConditions []string `yaml:"functions_conditions"`
|
||||
}
|
||||
|
||||
// VariableDep defines variable requirements
|
||||
type VariableDep struct {
|
||||
Name string `yaml:"name"`
|
||||
Type VariableType `yaml:"type"`
|
||||
Required bool `yaml:"required"`
|
||||
}
|
||||
|
||||
// HasCommandDeps returns true if there are command dependencies
|
||||
func (d *Dependencies) HasCommandDeps() bool {
|
||||
return d != nil && len(d.Commands) > 0
|
||||
}
|
||||
|
||||
// HasFileDeps returns true if there are file dependencies
|
||||
func (d *Dependencies) HasFileDeps() bool {
|
||||
return d != nil && len(d.Files) > 0
|
||||
}
|
||||
|
||||
// HasVariableDeps returns true if there are variable dependencies
|
||||
func (d *Dependencies) HasVariableDeps() bool {
|
||||
return d != nil && len(d.Variables) > 0
|
||||
}
|
||||
|
||||
// HasFunctionConditions returns true if there are function-based condition dependencies
|
||||
func (d *Dependencies) HasFunctionConditions() bool {
|
||||
return d != nil && len(d.FunctionsConditions) > 0
|
||||
}
|
||||
|
||||
// GetRequiredVariables returns all required variable dependencies
|
||||
func (d *Dependencies) GetRequiredVariables() []VariableDep {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
var required []VariableDep
|
||||
for _, v := range d.Variables {
|
||||
if v.Required {
|
||||
required = append(required, v)
|
||||
}
|
||||
}
|
||||
return required
|
||||
}
|
||||
|
||||
var domainRegex = regexp.MustCompile(`^([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$`)
|
||||
var numericRegex = regexp.MustCompile(`^-?\d+(\.\d+)?$`)
|
||||
var simpleRepoRegex = regexp.MustCompile(`^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$`)
|
||||
var hostedRepoRegex = regexp.MustCompile(`^(?:(?:https?://)?(?:www\.)?(github\.com|gitlab\.com)/)([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+?)(?:\.git)?/?$`)
|
||||
var sshRepoRegex = regexp.MustCompile(`^git@[^:]+:([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+?)(\.git)?$`)
|
||||
|
||||
func MatchesVariableType(value string, varType VariableType) (bool, error) {
|
||||
switch varType {
|
||||
case VarTypeDomain:
|
||||
return domainRegex.MatchString(value), nil
|
||||
case VarTypeSubdomain:
|
||||
if !domainRegex.MatchString(value) {
|
||||
return false, nil
|
||||
}
|
||||
return strings.Count(value, ".") >= 2, nil
|
||||
case VarTypeURL:
|
||||
u, err := url.Parse(value)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
if u.Scheme == "http" || u.Scheme == "https" {
|
||||
return u.Host != "", nil
|
||||
}
|
||||
return false, nil
|
||||
case VarTypeCIDR:
|
||||
_, _, err := net.ParseCIDR(value)
|
||||
return err == nil, nil
|
||||
case VarTypeRepo:
|
||||
return isRepo(value), nil
|
||||
case VarTypePath, VarTypeFile, VarTypeFolder:
|
||||
return value != "", nil
|
||||
case VarTypeNumber:
|
||||
return numericRegex.MatchString(value), nil
|
||||
case VarTypeString:
|
||||
return true, nil
|
||||
default:
|
||||
return false, fmt.Errorf("unknown variable type: %s", varType)
|
||||
}
|
||||
}
|
||||
|
||||
func MatchesTargetType(target string, targetType TargetType) (bool, error) {
|
||||
switch targetType {
|
||||
case TargetTypeDomain:
|
||||
return MatchesVariableType(target, VarTypeDomain)
|
||||
case TargetTypeSubdomain:
|
||||
return MatchesVariableType(target, VarTypeSubdomain)
|
||||
case TargetTypeURL:
|
||||
return MatchesVariableType(target, VarTypeURL)
|
||||
case TargetTypeCIDR:
|
||||
return MatchesVariableType(target, VarTypeCIDR)
|
||||
case TargetTypeRepo:
|
||||
return MatchesVariableType(target, VarTypeRepo)
|
||||
case TargetTypePath:
|
||||
return MatchesVariableType(target, VarTypePath)
|
||||
case TargetTypeNumber:
|
||||
return MatchesVariableType(target, VarTypeNumber)
|
||||
case TargetTypeString:
|
||||
return true, nil
|
||||
case TargetTypeFile:
|
||||
info, err := os.Stat(target)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
return !info.IsDir(), nil
|
||||
case TargetTypeFolder:
|
||||
info, err := os.Stat(target)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
return info.IsDir(), nil
|
||||
default:
|
||||
return false, fmt.Errorf("unknown target type: %s", targetType)
|
||||
}
|
||||
}
|
||||
|
||||
func isRepo(value string) bool {
|
||||
if simpleRepoRegex.MatchString(value) {
|
||||
return true
|
||||
}
|
||||
if hostedRepoRegex.MatchString(value) {
|
||||
return true
|
||||
}
|
||||
if sshRepoRegex.MatchString(value) {
|
||||
return true
|
||||
}
|
||||
u, err := url.Parse(value)
|
||||
if err == nil && (u.Scheme == "http" || u.Scheme == "https" || u.Scheme == "ssh" || u.Scheme == "git") {
|
||||
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||
if len(parts) >= 2 {
|
||||
owner := parts[0]
|
||||
repo := strings.TrimSuffix(parts[1], ".git")
|
||||
if owner != "" && repo != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package core
|
||||
|
||||
import "context"
|
||||
|
||||
// WorkflowParser parses workflow files
|
||||
type WorkflowParser interface {
|
||||
// Parse parses a workflow file and returns the workflow
|
||||
Parse(path string) (*Workflow, error)
|
||||
|
||||
// ParseContent parses workflow content from bytes
|
||||
ParseContent(content []byte) (*Workflow, error)
|
||||
|
||||
// Validate validates a parsed workflow
|
||||
Validate(w *Workflow) error
|
||||
}
|
||||
|
||||
// TemplateEngine renders templates with variable substitution
|
||||
type TemplateEngine interface {
|
||||
// Render renders a template string with the given context
|
||||
Render(template string, ctx map[string]interface{}) (string, error)
|
||||
|
||||
// RenderStep renders all template fields in a step
|
||||
RenderStep(step *Step, ctx map[string]interface{}) (*Step, error)
|
||||
|
||||
// ExecuteGenerator executes a generator function and returns the result
|
||||
ExecuteGenerator(expr string) (string, error)
|
||||
}
|
||||
|
||||
// FunctionRegistry manages and executes utility functions
|
||||
type FunctionRegistry interface {
|
||||
// Register registers a function with the given name
|
||||
Register(name string, fn interface{}) error
|
||||
|
||||
// Execute executes a function expression and returns the result
|
||||
Execute(expr string, ctx map[string]interface{}) (interface{}, error)
|
||||
|
||||
// EvaluateCondition evaluates a condition expression and returns true/false
|
||||
EvaluateCondition(condition string, ctx map[string]interface{}) (bool, error)
|
||||
|
||||
// EvaluateExports evaluates export expressions and returns the results
|
||||
EvaluateExports(exports map[string]string, ctx map[string]interface{}) (map[string]interface{}, error)
|
||||
}
|
||||
|
||||
// StepExecutor executes individual steps
|
||||
type StepExecutor interface {
|
||||
// Execute executes a step and returns the result
|
||||
Execute(ctx context.Context, step *Step, execCtx *ExecutionContext) (*StepResult, error)
|
||||
|
||||
// CanHandle returns true if this executor can handle the given step type
|
||||
CanHandle(stepType StepType) bool
|
||||
}
|
||||
|
||||
// WorkflowExecutor executes complete workflows
|
||||
type WorkflowExecutor interface {
|
||||
// ExecuteModule executes a module workflow
|
||||
ExecuteModule(ctx context.Context, module *Workflow, params map[string]string) (*WorkflowResult, error)
|
||||
|
||||
// ExecuteFlow executes a flow workflow
|
||||
ExecuteFlow(ctx context.Context, flow *Workflow, params map[string]string) (*WorkflowResult, error)
|
||||
}
|
||||
|
||||
// Scheduler manages workflow triggers and scheduling
|
||||
type Scheduler interface {
|
||||
// RegisterTrigger registers a workflow trigger
|
||||
RegisterTrigger(workflow *Workflow, trigger *Trigger) error
|
||||
|
||||
// UnregisterTrigger removes a trigger by name
|
||||
UnregisterTrigger(name string) error
|
||||
|
||||
// Start starts the scheduler
|
||||
Start() error
|
||||
|
||||
// Stop stops the scheduler
|
||||
Stop() error
|
||||
|
||||
// EmitEvent emits a named event with payload
|
||||
EmitEvent(name string, payload map[string]interface{}) error
|
||||
}
|
||||
|
||||
// WorkflowLoader loads workflows from disk
|
||||
type WorkflowLoader interface {
|
||||
// LoadWorkflow loads a single workflow by name
|
||||
LoadWorkflow(name string) (*Workflow, error)
|
||||
|
||||
// LoadAllWorkflows loads all workflows from the configured directory
|
||||
LoadAllWorkflows() ([]*Workflow, error)
|
||||
|
||||
// ReloadWorkflows reloads all workflows from disk
|
||||
ReloadWorkflows() error
|
||||
|
||||
// GetWorkflow returns a cached workflow by name
|
||||
GetWorkflow(name string) (*Workflow, bool)
|
||||
}
|
||||
|
||||
// DependencyChecker validates workflow dependencies
|
||||
type DependencyChecker interface {
|
||||
// CheckCommands checks if required commands are available
|
||||
CheckCommands(commands []string) error
|
||||
|
||||
// CheckFiles checks if required files exist
|
||||
CheckFiles(files []string, ctx map[string]interface{}) error
|
||||
|
||||
// CheckVariables validates required variables are present
|
||||
CheckVariables(deps []VariableDep, ctx map[string]interface{}) error
|
||||
|
||||
// CheckAll performs all dependency checks
|
||||
CheckAll(deps *Dependencies, ctx map[string]interface{}) error
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package core
|
||||
|
||||
// LLMMessageRole represents the role of a message sender
|
||||
type LLMMessageRole string
|
||||
|
||||
const (
|
||||
LLMRoleSystem LLMMessageRole = "system"
|
||||
LLMRoleUser LLMMessageRole = "user"
|
||||
LLMRoleAssistant LLMMessageRole = "assistant"
|
||||
LLMRoleTool LLMMessageRole = "tool"
|
||||
)
|
||||
|
||||
// LLMContentType represents the type of content in a message part
|
||||
type LLMContentType string
|
||||
|
||||
const (
|
||||
LLMContentTypeText LLMContentType = "text"
|
||||
LLMContentTypeImageURL LLMContentType = "image_url"
|
||||
)
|
||||
|
||||
// LLMImageURL represents an image URL with optional detail level
|
||||
type LLMImageURL struct {
|
||||
URL string `yaml:"url" json:"url"`
|
||||
Detail string `yaml:"detail,omitempty" json:"detail,omitempty"` // "low", "high", "auto"
|
||||
}
|
||||
|
||||
// LLMContentPart represents a single content part (text or image)
|
||||
type LLMContentPart struct {
|
||||
Type LLMContentType `yaml:"type" json:"type"`
|
||||
Text string `yaml:"text,omitempty" json:"text,omitempty"`
|
||||
ImageURL *LLMImageURL `yaml:"image_url,omitempty" json:"image_url,omitempty"`
|
||||
}
|
||||
|
||||
// LLMMessage represents a single message in the conversation
|
||||
// Content can be either a string or []LLMContentPart for multimodal
|
||||
type LLMMessage struct {
|
||||
Role LLMMessageRole `yaml:"role" json:"role"`
|
||||
Content interface{} `yaml:"content" json:"content"` // string or []LLMContentPart
|
||||
Name string `yaml:"name,omitempty" json:"name,omitempty"`
|
||||
ToolCallID string `yaml:"tool_call_id,omitempty" json:"tool_call_id,omitempty"`
|
||||
ToolCalls []LLMToolCall `yaml:"tool_calls,omitempty" json:"tool_calls,omitempty"`
|
||||
}
|
||||
|
||||
// LLMToolFunction defines a function that the LLM can call
|
||||
type LLMToolFunction struct {
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Description string `yaml:"description,omitempty" json:"description,omitempty"`
|
||||
Parameters map[string]interface{} `yaml:"parameters,omitempty" json:"parameters,omitempty"`
|
||||
}
|
||||
|
||||
// LLMTool represents a tool available to the LLM
|
||||
type LLMTool struct {
|
||||
Type string `yaml:"type" json:"type"` // "function"
|
||||
Function LLMToolFunction `yaml:"function" json:"function"`
|
||||
}
|
||||
|
||||
// LLMToolCallFunction represents the function details in a tool call
|
||||
type LLMToolCallFunction struct {
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Arguments string `yaml:"arguments" json:"arguments"` // JSON string
|
||||
}
|
||||
|
||||
// LLMToolCall represents a tool call made by the LLM in a response
|
||||
type LLMToolCall struct {
|
||||
ID string `yaml:"id" json:"id"`
|
||||
Type string `yaml:"type" json:"type"` // "function"
|
||||
Function LLMToolCallFunction `yaml:"function" json:"function"`
|
||||
}
|
||||
|
||||
// LLMResponseFormat specifies the output format
|
||||
type LLMResponseFormat struct {
|
||||
Type string `yaml:"type" json:"type"` // "text", "json_object", "json_schema"
|
||||
JSONSchema map[string]interface{} `yaml:"json_schema,omitempty" json:"json_schema,omitempty"`
|
||||
}
|
||||
|
||||
// LLMStepConfig holds step-level LLM configuration overrides
|
||||
// These override the global llm_config settings
|
||||
type LLMStepConfig struct {
|
||||
// Provider override (use specific provider by name instead of rotation)
|
||||
Provider string `yaml:"provider,omitempty"`
|
||||
|
||||
// Model override
|
||||
Model string `yaml:"model,omitempty"`
|
||||
|
||||
// Generation parameters (using pointers to distinguish between unset and zero values)
|
||||
MaxTokens *int `yaml:"max_tokens,omitempty"`
|
||||
Temperature *float64 `yaml:"temperature,omitempty"`
|
||||
TopK *int `yaml:"top_k,omitempty"`
|
||||
TopP *float64 `yaml:"top_p,omitempty"`
|
||||
N *int `yaml:"n,omitempty"`
|
||||
|
||||
// Request settings
|
||||
Timeout string `yaml:"timeout,omitempty"`
|
||||
MaxRetries *int `yaml:"max_retries,omitempty"`
|
||||
Stream *bool `yaml:"stream,omitempty"`
|
||||
|
||||
// Response format
|
||||
ResponseFormat *LLMResponseFormat `yaml:"response_format,omitempty"`
|
||||
|
||||
// Custom headers (merged with global)
|
||||
CustomHeaders map[string]string `yaml:"custom_headers,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package core
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Param represents a workflow parameter
|
||||
type Param struct {
|
||||
Name string `yaml:"name"`
|
||||
Type string `yaml:"type"` // "string", "bool", "int" (default: "string")
|
||||
Default any `yaml:"default"` // Supports string, bool, int from YAML
|
||||
Required bool `yaml:"required"`
|
||||
Generator string `yaml:"generator"` // e.g., uuid(), currentDate(), getEnvVar("KEY")
|
||||
}
|
||||
|
||||
// HasDefault returns true if the parameter has a default value
|
||||
func (p *Param) HasDefault() bool {
|
||||
return p.Default != nil && p.DefaultString() != ""
|
||||
}
|
||||
|
||||
// HasGenerator returns true if the parameter has a generator function
|
||||
func (p *Param) HasGenerator() bool {
|
||||
return p.Generator != ""
|
||||
}
|
||||
|
||||
// IsRequired returns true if the parameter is required
|
||||
func (p *Param) IsRequired() bool {
|
||||
return p.Required
|
||||
}
|
||||
|
||||
// IsBool returns true if the parameter type is bool
|
||||
func (p *Param) IsBool() bool {
|
||||
return p.Type == "bool"
|
||||
}
|
||||
|
||||
// IsInt returns true if the parameter type is int
|
||||
func (p *Param) IsInt() bool {
|
||||
return p.Type == "int"
|
||||
}
|
||||
|
||||
// DefaultString returns the default value as a string
|
||||
func (p *Param) DefaultString() string {
|
||||
if p.Default == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%v", p.Default)
|
||||
}
|
||||
|
||||
// DefaultBool returns the default value as a bool
|
||||
func (p *Param) DefaultBool() bool {
|
||||
if p.Default == nil {
|
||||
return false
|
||||
}
|
||||
switch v := p.Default.(type) {
|
||||
case bool:
|
||||
return v
|
||||
case string:
|
||||
return v == "true" || v == "1"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package core
|
||||
|
||||
// Preferences defines workflow-level execution preferences.
|
||||
// All fields are pointers to distinguish "not set" (nil) from "explicitly false".
|
||||
// When a preference is set in a workflow, it serves as a default that can be
|
||||
// overridden by explicit CLI flags.
|
||||
type Preferences struct {
|
||||
// DisableNotifications turns off all notifications (--disable-notification)
|
||||
DisableNotifications *bool `yaml:"disable_notifications,omitempty"`
|
||||
|
||||
// DisableLogging turns off all logging output (--disable-logging)
|
||||
DisableLogging *bool `yaml:"disable_logging,omitempty"`
|
||||
|
||||
// HeuristicsCheck sets the heuristics check level: "none", "basic", "advanced" (--heuristics-check)
|
||||
HeuristicsCheck *string `yaml:"heuristics_check,omitempty"`
|
||||
|
||||
// CIOutputFormat outputs results in JSON format for CI pipelines (--ci-output-format)
|
||||
CIOutputFormat *bool `yaml:"ci_output_format,omitempty"`
|
||||
|
||||
// Silent suppresses all output except errors (--silent)
|
||||
Silent *bool `yaml:"silent,omitempty"`
|
||||
|
||||
// Repeat enables repeat mode after completion (--repeat)
|
||||
Repeat *bool `yaml:"repeat,omitempty"`
|
||||
|
||||
// RepeatWaitTime sets wait time between repeats, e.g., "60s", "1h" (--repeat-wait-time)
|
||||
RepeatWaitTime *string `yaml:"repeat_wait_time,omitempty"`
|
||||
}
|
||||
|
||||
// Helper functions to safely get values with defaults
|
||||
|
||||
// GetDisableNotifications returns the disable_notifications preference or the default value
|
||||
func (p *Preferences) GetDisableNotifications(defaultVal bool) bool {
|
||||
if p == nil || p.DisableNotifications == nil {
|
||||
return defaultVal
|
||||
}
|
||||
return *p.DisableNotifications
|
||||
}
|
||||
|
||||
// GetDisableLogging returns the disable_logging preference or the default value
|
||||
func (p *Preferences) GetDisableLogging(defaultVal bool) bool {
|
||||
if p == nil || p.DisableLogging == nil {
|
||||
return defaultVal
|
||||
}
|
||||
return *p.DisableLogging
|
||||
}
|
||||
|
||||
// GetHeuristicsCheck returns the heuristics_check preference or the default value
|
||||
func (p *Preferences) GetHeuristicsCheck(defaultVal string) string {
|
||||
if p == nil || p.HeuristicsCheck == nil {
|
||||
return defaultVal
|
||||
}
|
||||
return *p.HeuristicsCheck
|
||||
}
|
||||
|
||||
// GetCIOutputFormat returns the ci_output_format preference or the default value
|
||||
func (p *Preferences) GetCIOutputFormat(defaultVal bool) bool {
|
||||
if p == nil || p.CIOutputFormat == nil {
|
||||
return defaultVal
|
||||
}
|
||||
return *p.CIOutputFormat
|
||||
}
|
||||
|
||||
// GetSilent returns the silent preference or the default value
|
||||
func (p *Preferences) GetSilent(defaultVal bool) bool {
|
||||
if p == nil || p.Silent == nil {
|
||||
return defaultVal
|
||||
}
|
||||
return *p.Silent
|
||||
}
|
||||
|
||||
// GetRepeat returns the repeat preference or the default value
|
||||
func (p *Preferences) GetRepeat(defaultVal bool) bool {
|
||||
if p == nil || p.Repeat == nil {
|
||||
return defaultVal
|
||||
}
|
||||
return *p.Repeat
|
||||
}
|
||||
|
||||
// GetRepeatWaitTime returns the repeat_wait_time preference or the default value
|
||||
func (p *Preferences) GetRepeatWaitTime(defaultVal string) string {
|
||||
if p == nil || p.RepeatWaitTime == nil {
|
||||
return defaultVal
|
||||
}
|
||||
return *p.RepeatWaitTime
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package core
|
||||
|
||||
// Report defines an output file produced by a workflow
|
||||
type Report struct {
|
||||
Name string `yaml:"name"`
|
||||
Path string `yaml:"path"`
|
||||
Type string `yaml:"type"` // text, csv, json, etc.
|
||||
Description string `yaml:"description"`
|
||||
}
|
||||
|
||||
// IsTextReport returns true if this is a text report
|
||||
func (r *Report) IsTextReport() bool {
|
||||
return r.Type == "text" || r.Type == ""
|
||||
}
|
||||
|
||||
// IsCSVReport returns true if this is a CSV report
|
||||
func (r *Report) IsCSVReport() bool {
|
||||
return r.Type == "csv"
|
||||
}
|
||||
|
||||
// IsJSONReport returns true if this is a JSON report
|
||||
func (r *Report) IsJSONReport() bool {
|
||||
return r.Type == "json"
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type StepTimeout string
|
||||
|
||||
func (t *StepTimeout) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var i int
|
||||
if err := unmarshal(&i); err == nil {
|
||||
if i < 0 {
|
||||
i = 0
|
||||
}
|
||||
*t = StepTimeout(strconv.Itoa(i))
|
||||
return nil
|
||||
}
|
||||
|
||||
var s string
|
||||
if err := unmarshal(&s); err == nil {
|
||||
*t = StepTimeout(strings.TrimSpace(s))
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("invalid timeout")
|
||||
}
|
||||
|
||||
func (t StepTimeout) MarshalYAML() (interface{}, error) {
|
||||
s := strings.TrimSpace(string(t))
|
||||
if s == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if isDigits(s) {
|
||||
i, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return s, nil
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (t StepTimeout) Duration() (time.Duration, error) {
|
||||
s := strings.TrimSpace(string(t))
|
||||
if s == "" {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if isDigits(s) {
|
||||
i, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid timeout: %w", err)
|
||||
}
|
||||
if i <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return time.Duration(i) * time.Second, nil
|
||||
}
|
||||
|
||||
if strings.HasSuffix(s, "d") {
|
||||
daysStr := strings.TrimSuffix(s, "d")
|
||||
if !isDigits(daysStr) {
|
||||
return 0, fmt.Errorf("invalid timeout: %s", s)
|
||||
}
|
||||
days, err := strconv.Atoi(daysStr)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid timeout: %w", err)
|
||||
}
|
||||
if days <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return time.Duration(days) * 24 * time.Hour, nil
|
||||
}
|
||||
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid timeout: %w", err)
|
||||
}
|
||||
if d <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func isDigits(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c < '0' || c > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type StepThreads string
|
||||
|
||||
func (t *StepThreads) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var i int
|
||||
if err := unmarshal(&i); err == nil {
|
||||
if i < 0 {
|
||||
i = 0
|
||||
}
|
||||
*t = StepThreads(strconv.Itoa(i))
|
||||
return nil
|
||||
}
|
||||
|
||||
var s string
|
||||
if err := unmarshal(&s); err == nil {
|
||||
*t = StepThreads(strings.TrimSpace(s))
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("invalid threads")
|
||||
}
|
||||
|
||||
func (t StepThreads) MarshalYAML() (interface{}, error) {
|
||||
s := strings.TrimSpace(string(t))
|
||||
if s == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if isDigits(s) {
|
||||
i, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return s, nil
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (t StepThreads) Int() (int, error) {
|
||||
s := strings.TrimSpace(string(t))
|
||||
if s == "" {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
i, err := strconv.Atoi(s)
|
||||
if err == nil {
|
||||
if i <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
f, ferr := strconv.ParseFloat(s, 64)
|
||||
if ferr != nil {
|
||||
return 0, fmt.Errorf("invalid threads: %w", err)
|
||||
}
|
||||
|
||||
if f <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if f != float64(int(f)) {
|
||||
return 0, fmt.Errorf("invalid threads: %s", s)
|
||||
}
|
||||
|
||||
return int(f), nil
|
||||
}
|
||||
|
||||
// StepRunnerConfig holds per-step runner configuration for remote-bash steps
|
||||
// The runner type is specified separately in Step.StepRunner
|
||||
type StepRunnerConfig struct {
|
||||
*RunnerConfig `yaml:",inline"` // Embed all RunnerConfig fields (image, host, etc.)
|
||||
}
|
||||
|
||||
// Step represents a single execution step in a module
|
||||
type Step struct {
|
||||
Name string `yaml:"name"`
|
||||
Type StepType `yaml:"type"`
|
||||
StepRunner RunnerType `yaml:"step_runner"` // Runner for this step: local (default), docker, ssh
|
||||
PreCondition string `yaml:"pre_condition"`
|
||||
Log string `yaml:"log"`
|
||||
Timeout StepTimeout `yaml:"timeout,omitempty"`
|
||||
|
||||
// Bash step fields
|
||||
Command string `yaml:"command"`
|
||||
Commands []string `yaml:"commands"`
|
||||
ParallelCommands []string `yaml:"parallel_commands"`
|
||||
StdFile string `yaml:"std_file"` // File path to save stdout/stderr output
|
||||
|
||||
// Structured argument fields (for bash/remote-bash steps)
|
||||
// These are templated and joined with Command in order: command + speed + config + input + output
|
||||
SpeedArgs string `yaml:"speed_args"`
|
||||
ConfigArgs string `yaml:"config_args"`
|
||||
InputArgs string `yaml:"input_args"`
|
||||
OutputArgs string `yaml:"output_args"`
|
||||
|
||||
// Function step fields
|
||||
Function string `yaml:"function"`
|
||||
Functions []string `yaml:"functions"`
|
||||
ParallelFunctions []string `yaml:"parallel_functions"`
|
||||
|
||||
// Parallel step fields
|
||||
ParallelSteps []Step `yaml:"parallel_steps"`
|
||||
|
||||
// Foreach step fields
|
||||
Input string `yaml:"input"`
|
||||
Variable string `yaml:"variable"`
|
||||
Threads StepThreads `yaml:"threads,omitempty"`
|
||||
Step *Step `yaml:"step"`
|
||||
|
||||
// Remote-bash step fields
|
||||
StepRunnerConfig *StepRunnerConfig `yaml:"step_runner_config"`
|
||||
StepRemoteFile string `yaml:"step_remote_file"` // File path on remote (Docker/SSH) to copy after execution
|
||||
HostOutputFile string `yaml:"host_output_file"` // Local path to copy the remote file to
|
||||
|
||||
// HTTP step fields
|
||||
URL string `yaml:"url"`
|
||||
Method string `yaml:"method"`
|
||||
Headers map[string]string `yaml:"headers"`
|
||||
RequestBody string `yaml:"request_body"`
|
||||
|
||||
// LLM step fields
|
||||
Messages []LLMMessage `yaml:"messages"`
|
||||
Tools []LLMTool `yaml:"tools,omitempty"`
|
||||
ToolChoice interface{} `yaml:"tool_choice,omitempty"`
|
||||
LLMConfig *LLMStepConfig `yaml:"llm_config,omitempty"`
|
||||
IsEmbedding bool `yaml:"is_embedding,omitempty"`
|
||||
EmbeddingInput []string `yaml:"embedding_input,omitempty"`
|
||||
ExtraLLMParams map[string]interface{} `yaml:"extra_llm_parameters,omitempty"`
|
||||
|
||||
// Common fields
|
||||
Exports map[string]string `yaml:"exports"`
|
||||
OnSuccess []Action `yaml:"on_success"`
|
||||
OnError []Action `yaml:"on_error"`
|
||||
Decision *DecisionConfig `yaml:"decision,omitempty"`
|
||||
}
|
||||
|
||||
// DecisionCase represents a single case in switch-style decision
|
||||
type DecisionCase struct {
|
||||
Goto string `yaml:"goto"`
|
||||
}
|
||||
|
||||
// DecisionConfig supports switch/case routing for conditional workflow branching.
|
||||
//
|
||||
// Switch/case syntax:
|
||||
//
|
||||
// decision:
|
||||
// switch: "{{variable}}"
|
||||
// cases:
|
||||
// "value1": { goto: step-a }
|
||||
// "value2": { goto: step-b }
|
||||
// default:
|
||||
// goto: fallback-step
|
||||
type DecisionConfig struct {
|
||||
Switch string `yaml:"switch,omitempty"`
|
||||
Cases map[string]DecisionCase `yaml:"cases,omitempty"`
|
||||
Default *DecisionCase `yaml:"default,omitempty"`
|
||||
}
|
||||
|
||||
// Action represents on_success/on_error handler
|
||||
type Action struct {
|
||||
Action ActionType `yaml:"action"`
|
||||
Message string `yaml:"message"`
|
||||
Condition string `yaml:"condition"`
|
||||
Name string `yaml:"name"` // for export action
|
||||
Value interface{} `yaml:"value"` // for export action
|
||||
Type StepType `yaml:"type"` // for run action
|
||||
Command string `yaml:"command"` // for run bash action
|
||||
Functions []string `yaml:"functions"` // for run function action
|
||||
Export map[string]string `yaml:"export"` // for run function action
|
||||
Notify string `yaml:"notify"` // notification message
|
||||
}
|
||||
|
||||
// IsBashStep returns true if this is a bash step
|
||||
func (s *Step) IsBashStep() bool {
|
||||
return s.Type == StepTypeBash
|
||||
}
|
||||
|
||||
// IsFunctionStep returns true if this is a function step
|
||||
func (s *Step) IsFunctionStep() bool {
|
||||
return s.Type == StepTypeFunction
|
||||
}
|
||||
|
||||
// IsParallelStep returns true if this is a parallel step
|
||||
func (s *Step) IsParallelStep() bool {
|
||||
return s.Type == StepTypeParallel
|
||||
}
|
||||
|
||||
// IsForeachStep returns true if this is a foreach step
|
||||
func (s *Step) IsForeachStep() bool {
|
||||
return s.Type == StepTypeForeach
|
||||
}
|
||||
|
||||
// IsRemoteBashStep returns true if this is a remote-bash step
|
||||
func (s *Step) IsRemoteBashStep() bool {
|
||||
return s.Type == StepTypeRemoteBash
|
||||
}
|
||||
|
||||
// IsHTTPStep returns true if this is an HTTP step
|
||||
func (s *Step) IsHTTPStep() bool {
|
||||
return s.Type == StepTypeHTTP
|
||||
}
|
||||
|
||||
// IsLLMStep returns true if this is an LLM step
|
||||
func (s *Step) IsLLMStep() bool {
|
||||
return s.Type == StepTypeLLM
|
||||
}
|
||||
|
||||
// GetStepRunner returns the step runner type, defaulting to host/local
|
||||
func (s *Step) GetStepRunner() RunnerType {
|
||||
if s.StepRunner == "" {
|
||||
return RunnerTypeHost // default to local
|
||||
}
|
||||
return s.StepRunner
|
||||
}
|
||||
|
||||
// HasParallelCommands returns true if step has parallel commands
|
||||
func (s *Step) HasParallelCommands() bool {
|
||||
return len(s.ParallelCommands) > 0
|
||||
}
|
||||
|
||||
// HasParallelFunctions returns true if step has parallel functions
|
||||
func (s *Step) HasParallelFunctions() bool {
|
||||
return len(s.ParallelFunctions) > 0
|
||||
}
|
||||
|
||||
// HasDecision returns true if step has decision routing
|
||||
func (s *Step) HasDecision() bool {
|
||||
if s.Decision == nil {
|
||||
return false
|
||||
}
|
||||
return s.Decision.Switch != "" || len(s.Decision.Cases) > 0
|
||||
}
|
||||
|
||||
// HasExports returns true if step exports variables
|
||||
func (s *Step) HasExports() bool {
|
||||
return len(s.Exports) > 0
|
||||
}
|
||||
|
||||
// GetCommands returns the list of commands to execute
|
||||
// Returns single command as slice if Commands is empty
|
||||
func (s *Step) GetCommands() []string {
|
||||
if len(s.Commands) > 0 {
|
||||
return s.Commands
|
||||
}
|
||||
if s.Command != "" {
|
||||
return []string{s.Command}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFunctions returns the list of functions to execute
|
||||
// Returns single function as slice if Functions is empty
|
||||
func (s *Step) GetFunctions() []string {
|
||||
if len(s.Functions) > 0 {
|
||||
return s.Functions
|
||||
}
|
||||
if s.Function != "" {
|
||||
return []string{s.Function}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clone creates a shallow copy of the step with new slices for Commands
|
||||
func (s *Step) Clone() *Step {
|
||||
cloned := *s
|
||||
|
||||
// Deep copy slices to avoid modifying originals
|
||||
if len(s.Commands) > 0 {
|
||||
cloned.Commands = make([]string, len(s.Commands))
|
||||
copy(cloned.Commands, s.Commands)
|
||||
}
|
||||
if len(s.ParallelCommands) > 0 {
|
||||
cloned.ParallelCommands = make([]string, len(s.ParallelCommands))
|
||||
copy(cloned.ParallelCommands, s.ParallelCommands)
|
||||
}
|
||||
if len(s.Functions) > 0 {
|
||||
cloned.Functions = make([]string, len(s.Functions))
|
||||
copy(cloned.Functions, s.Functions)
|
||||
}
|
||||
if len(s.ParallelFunctions) > 0 {
|
||||
cloned.ParallelFunctions = make([]string, len(s.ParallelFunctions))
|
||||
copy(cloned.ParallelFunctions, s.ParallelFunctions)
|
||||
}
|
||||
|
||||
// Deep copy StepRunnerConfig
|
||||
if s.StepRunnerConfig != nil {
|
||||
clonedConfig := &StepRunnerConfig{}
|
||||
if s.StepRunnerConfig.RunnerConfig != nil {
|
||||
cfg := *s.StepRunnerConfig.RunnerConfig
|
||||
// Deep copy slices in RunnerConfig
|
||||
if len(cfg.Volumes) > 0 {
|
||||
cfg.Volumes = make([]string, len(s.StepRunnerConfig.Volumes))
|
||||
copy(cfg.Volumes, s.StepRunnerConfig.Volumes)
|
||||
}
|
||||
if len(cfg.Env) > 0 {
|
||||
cfg.Env = make(map[string]string, len(s.StepRunnerConfig.Env))
|
||||
for k, v := range s.StepRunnerConfig.Env {
|
||||
cfg.Env[k] = v
|
||||
}
|
||||
}
|
||||
clonedConfig.RunnerConfig = &cfg
|
||||
}
|
||||
cloned.StepRunnerConfig = clonedConfig
|
||||
}
|
||||
|
||||
// Deep copy HTTP Headers map
|
||||
if len(s.Headers) > 0 {
|
||||
cloned.Headers = make(map[string]string, len(s.Headers))
|
||||
for k, v := range s.Headers {
|
||||
cloned.Headers[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
// Deep copy LLM fields
|
||||
if len(s.Messages) > 0 {
|
||||
cloned.Messages = make([]LLMMessage, len(s.Messages))
|
||||
copy(cloned.Messages, s.Messages)
|
||||
}
|
||||
if len(s.Tools) > 0 {
|
||||
cloned.Tools = make([]LLMTool, len(s.Tools))
|
||||
copy(cloned.Tools, s.Tools)
|
||||
}
|
||||
if len(s.EmbeddingInput) > 0 {
|
||||
cloned.EmbeddingInput = make([]string, len(s.EmbeddingInput))
|
||||
copy(cloned.EmbeddingInput, s.EmbeddingInput)
|
||||
}
|
||||
if len(s.ExtraLLMParams) > 0 {
|
||||
cloned.ExtraLLMParams = make(map[string]interface{}, len(s.ExtraLLMParams))
|
||||
for k, v := range s.ExtraLLMParams {
|
||||
cloned.ExtraLLMParams[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
return &cloned
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package core
|
||||
|
||||
// Trigger defines when a workflow should execute
|
||||
type Trigger struct {
|
||||
Name string `yaml:"name"`
|
||||
On TriggerType `yaml:"on"`
|
||||
Schedule string `yaml:"schedule,omitempty"` // cron expression (for cron triggers)
|
||||
Event *EventConfig `yaml:"event,omitempty"` // event configuration (for event triggers)
|
||||
Path string `yaml:"path,omitempty"` // watch path (for watch triggers)
|
||||
Input TriggerInput `yaml:"input,omitempty"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// EventConfig holds event trigger configuration
|
||||
type EventConfig struct {
|
||||
Topic string `yaml:"topic"` // e.g., "webhook.received", "assets.new"
|
||||
Filters []string `yaml:"filters,omitempty"` // JS expressions: ["event.name == 'discovered'"]
|
||||
}
|
||||
|
||||
// TriggerInput defines the input source for trigger
|
||||
type TriggerInput struct {
|
||||
Type string `yaml:"type"` // file, event_data, function, param
|
||||
Path string `yaml:"path,omitempty"` // for file type
|
||||
Field string `yaml:"field,omitempty"` // for event_data type
|
||||
Function string `yaml:"function,omitempty"` // for function type (e.g., jq("{{event.data}}", ".url"))
|
||||
Name string `yaml:"name,omitempty"` // parameter name to set
|
||||
}
|
||||
|
||||
// IsCron returns true if this is a cron trigger
|
||||
func (t *Trigger) IsCron() bool {
|
||||
return t.On == TriggerCron
|
||||
}
|
||||
|
||||
// IsEvent returns true if this is an event trigger
|
||||
func (t *Trigger) IsEvent() bool {
|
||||
return t.On == TriggerEvent
|
||||
}
|
||||
|
||||
// IsWatch returns true if this is a file watch trigger
|
||||
func (t *Trigger) IsWatch() bool {
|
||||
return t.On == TriggerWatch
|
||||
}
|
||||
|
||||
// IsManual returns true if this is a manual trigger
|
||||
func (t *Trigger) IsManual() bool {
|
||||
return t.On == TriggerManual
|
||||
}
|
||||
|
||||
// IsEnabled returns true if the trigger is enabled
|
||||
func (t *Trigger) IsEnabled() bool {
|
||||
return t.Enabled
|
||||
}
|
||||
|
||||
// MatchesTopic checks if the trigger's event topic matches the given topic
|
||||
func (t *Trigger) MatchesTopic(topic string) bool {
|
||||
if !t.IsEvent() || t.Event == nil {
|
||||
return false
|
||||
}
|
||||
// Empty topic matches all events
|
||||
if t.Event.Topic == "" {
|
||||
return true
|
||||
}
|
||||
return t.Event.Topic == topic
|
||||
}
|
||||
|
||||
// HasFilters returns true if the event trigger has filters defined
|
||||
func (t *Trigger) HasFilters() bool {
|
||||
return t.IsEvent() && t.Event != nil && len(t.Event.Filters) > 0
|
||||
}
|
||||
|
||||
// GetFilters returns the filter expressions for the event trigger
|
||||
func (t *Trigger) GetFilters() []string {
|
||||
if t.Event == nil {
|
||||
return nil
|
||||
}
|
||||
return t.Event.Filters
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WorkflowKind represents the type of workflow
|
||||
type WorkflowKind string
|
||||
|
||||
const (
|
||||
KindModule WorkflowKind = "module"
|
||||
KindFlow WorkflowKind = "flow"
|
||||
)
|
||||
|
||||
// StepType represents the type of step
|
||||
type StepType string
|
||||
|
||||
const (
|
||||
StepTypeBash StepType = "bash"
|
||||
StepTypeFunction StepType = "function"
|
||||
StepTypeParallel StepType = "parallel-steps"
|
||||
StepTypeForeach StepType = "foreach"
|
||||
StepTypeRemoteBash StepType = "remote-bash"
|
||||
StepTypeHTTP StepType = "http"
|
||||
StepTypeLLM StepType = "llm"
|
||||
)
|
||||
|
||||
// TriggerType represents trigger types
|
||||
type TriggerType string
|
||||
|
||||
const (
|
||||
TriggerCron TriggerType = "cron"
|
||||
TriggerEvent TriggerType = "event"
|
||||
TriggerWatch TriggerType = "watch"
|
||||
TriggerManual TriggerType = "manual"
|
||||
)
|
||||
|
||||
// VariableType for dependency validation
|
||||
type VariableType string
|
||||
|
||||
const (
|
||||
VarTypeDomain VariableType = "domain"
|
||||
VarTypePath VariableType = "path"
|
||||
VarTypeNumber VariableType = "number"
|
||||
VarTypeFile VariableType = "file"
|
||||
VarTypeFolder VariableType = "folder"
|
||||
VarTypeString VariableType = "string"
|
||||
VarTypeSubdomain VariableType = "subdomain"
|
||||
VarTypeURL VariableType = "url"
|
||||
VarTypeCIDR VariableType = "cidr"
|
||||
VarTypeRepo VariableType = "repo"
|
||||
)
|
||||
|
||||
type TargetType string
|
||||
|
||||
const (
|
||||
TargetTypeDomain TargetType = "domain"
|
||||
TargetTypeSubdomain TargetType = "subdomain"
|
||||
TargetTypeURL TargetType = "url"
|
||||
TargetTypeCIDR TargetType = "cidr"
|
||||
TargetTypeRepo TargetType = "repo"
|
||||
TargetTypePath TargetType = "path"
|
||||
TargetTypeFile TargetType = "file"
|
||||
TargetTypeFolder TargetType = "folder"
|
||||
TargetTypeNumber TargetType = "number"
|
||||
TargetTypeString TargetType = "string"
|
||||
)
|
||||
|
||||
// ActionType for on_success/on_error handlers
|
||||
type ActionType string
|
||||
|
||||
const (
|
||||
ActionLog ActionType = "log"
|
||||
ActionAbort ActionType = "abort"
|
||||
ActionContinue ActionType = "continue"
|
||||
ActionExport ActionType = "export"
|
||||
ActionRun ActionType = "run"
|
||||
ActionNotify ActionType = "notify"
|
||||
)
|
||||
|
||||
// StepStatus represents the status of a step execution
|
||||
type StepStatus string
|
||||
|
||||
const (
|
||||
StepStatusPending StepStatus = "pending"
|
||||
StepStatusRunning StepStatus = "running"
|
||||
StepStatusSuccess StepStatus = "success"
|
||||
StepStatusFailed StepStatus = "failed"
|
||||
StepStatusSkipped StepStatus = "skipped"
|
||||
)
|
||||
|
||||
// RunnerType represents the execution environment for workflows
|
||||
type RunnerType string
|
||||
|
||||
const (
|
||||
RunnerTypeHost RunnerType = "host" // Execute on local machine (default)
|
||||
RunnerTypeDocker RunnerType = "docker" // Execute in Docker container
|
||||
RunnerTypeSSH RunnerType = "ssh" // Execute on remote machine via SSH
|
||||
)
|
||||
|
||||
// RunStatus represents the status of a run
|
||||
type RunStatus string
|
||||
|
||||
const (
|
||||
RunStatusPending RunStatus = "pending"
|
||||
RunStatusRunning RunStatus = "running"
|
||||
RunStatusCompleted RunStatus = "completed"
|
||||
RunStatusFailed RunStatus = "failed"
|
||||
RunStatusCancelled RunStatus = "cancelled"
|
||||
RunStatusSkipped RunStatus = "skipped"
|
||||
)
|
||||
|
||||
// StepResult holds step execution result
|
||||
type StepResult struct {
|
||||
StepName string
|
||||
Status StepStatus
|
||||
Output string
|
||||
Error error
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
Duration time.Duration
|
||||
Exports map[string]interface{}
|
||||
NextStep string // from decision routing
|
||||
LogFile string
|
||||
}
|
||||
|
||||
// WorkflowResult holds workflow execution result
|
||||
type WorkflowResult struct {
|
||||
WorkflowName string
|
||||
WorkflowKind WorkflowKind
|
||||
RunID string
|
||||
Target string
|
||||
Status RunStatus
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
Steps []*StepResult
|
||||
Artifacts []string
|
||||
Exports map[string]interface{}
|
||||
Error error
|
||||
Message string // Optional message (e.g., for skipped status)
|
||||
}
|
||||
|
||||
// Event represents a system event for triggers
|
||||
// Topics follow the format: <component>.<event_type>
|
||||
// Examples: webhook.received, assets.new, db.change, watch.files
|
||||
type Event struct {
|
||||
Topic string `json:"topic" yaml:"topic"` // e.g., "webhook.received", "assets.new"
|
||||
ID string `json:"id" yaml:"id"` // UUID of the event
|
||||
Name string `json:"name" yaml:"name"` // e.g., "vulnerability.discovered"
|
||||
Source string `json:"source" yaml:"source"` // e.g., "nuclei", "httpx"
|
||||
Data string `json:"data" yaml:"data"` // JSON string payload
|
||||
DataType string `json:"data_type" yaml:"data_type"` // e.g., "endpoint", "vulnerability"
|
||||
Timestamp time.Time `json:"timestamp" yaml:"timestamp"` // When the event occurred
|
||||
ParsedData map[string]interface{} `json:"-" yaml:"-"` // Parsed JSON for filter evaluation
|
||||
}
|
||||
|
||||
// ParseData parses the JSON data string into ParsedData map
|
||||
func (e *Event) ParseData() error {
|
||||
if e.Data == "" {
|
||||
e.ParsedData = make(map[string]interface{})
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal([]byte(e.Data), &e.ParsedData)
|
||||
}
|
||||
|
||||
// GetDataField retrieves a field from the parsed data
|
||||
func (e *Event) GetDataField(field string) interface{} {
|
||||
if e.ParsedData == nil {
|
||||
if err := e.ParseData(); err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return e.ParsedData[field]
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package core
|
||||
|
||||
import "strings"
|
||||
|
||||
// TagList is a comma-separated list of tags that parses to []string
|
||||
type TagList []string
|
||||
|
||||
// UnmarshalYAML implements custom YAML unmarshaling for comma-separated tags
|
||||
func (t *TagList) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var s string
|
||||
if err := unmarshal(&s); err != nil {
|
||||
return err
|
||||
}
|
||||
if s == "" {
|
||||
*t = []string{}
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(s, ",")
|
||||
for i := range parts {
|
||||
parts[i] = strings.TrimSpace(parts[i])
|
||||
}
|
||||
*t = parts
|
||||
return nil
|
||||
}
|
||||
|
||||
// Workflow represents either a Module or Flow
|
||||
type Workflow struct {
|
||||
Kind WorkflowKind `yaml:"kind"`
|
||||
Name string `yaml:"name"`
|
||||
Description string `yaml:"description"`
|
||||
Tags TagList `yaml:"tags,omitempty"`
|
||||
Params []Param `yaml:"params"`
|
||||
Triggers []Trigger `yaml:"trigger"`
|
||||
Dependencies *Dependencies `yaml:"dependencies"`
|
||||
Reports []Report `yaml:"reports"`
|
||||
|
||||
// Execution preferences (optional, can be overridden by CLI flags)
|
||||
Preferences *Preferences `yaml:"preferences,omitempty"`
|
||||
|
||||
// Runner configuration (module-kind only)
|
||||
Runner RunnerType `yaml:"runner,omitempty"`
|
||||
RunnerConfig *RunnerConfig `yaml:"runner_config,omitempty"`
|
||||
|
||||
// Module-specific fields
|
||||
Steps []Step `yaml:"steps,omitempty"`
|
||||
|
||||
// Flow-specific fields
|
||||
Modules []ModuleRef `yaml:"modules,omitempty"`
|
||||
|
||||
// Internal metadata
|
||||
FilePath string `yaml:"-"`
|
||||
Checksum string `yaml:"-"`
|
||||
}
|
||||
|
||||
// RunnerConfig holds configuration for different runner types
|
||||
type RunnerConfig struct {
|
||||
// Docker configuration
|
||||
Image string `yaml:"image,omitempty"` // Docker image e.g., "ubuntu:latest"
|
||||
Env map[string]string `yaml:"env,omitempty"` // Environment variables
|
||||
Volumes []string `yaml:"volumes,omitempty"` // Volume mounts e.g., "/host:/container"
|
||||
Network string `yaml:"network,omitempty"` // Network mode e.g., "host", "bridge"
|
||||
Persistent bool `yaml:"persistent,omitempty"` // true=reuse container, false=ephemeral
|
||||
|
||||
// SSH configuration
|
||||
Host string `yaml:"host,omitempty"` // SSH hostname or IP
|
||||
Port int `yaml:"port,omitempty"` // SSH port (default 22)
|
||||
User string `yaml:"user,omitempty"` // SSH username
|
||||
KeyFile string `yaml:"key_file,omitempty"` // Path to SSH private key
|
||||
Password string `yaml:"password,omitempty"` // SSH password (prefer key_file)
|
||||
|
||||
// Common configuration
|
||||
WorkDir string `yaml:"workdir,omitempty"` // Working directory on remote/container
|
||||
}
|
||||
|
||||
// ModuleRef references a module in a flow
|
||||
type ModuleRef struct {
|
||||
Name string `yaml:"name"`
|
||||
Path string `yaml:"path"`
|
||||
Params map[string]string `yaml:"params"`
|
||||
DependsOn []string `yaml:"depends_on"`
|
||||
Condition string `yaml:"condition"`
|
||||
OnSuccess []Action `yaml:"on_success"`
|
||||
OnError []Action `yaml:"on_error"`
|
||||
Decision *DecisionConfig `yaml:"decision"`
|
||||
}
|
||||
|
||||
// IsModule returns true if the workflow is a module
|
||||
func (w *Workflow) IsModule() bool {
|
||||
return w.Kind == KindModule
|
||||
}
|
||||
|
||||
// IsFlow returns true if the workflow is a flow
|
||||
func (w *Workflow) IsFlow() bool {
|
||||
return w.Kind == KindFlow
|
||||
}
|
||||
|
||||
// GetRequiredParams returns all required parameters
|
||||
func (w *Workflow) GetRequiredParams() []Param {
|
||||
var required []Param
|
||||
for _, p := range w.Params {
|
||||
if p.Required {
|
||||
required = append(required, p)
|
||||
}
|
||||
}
|
||||
return required
|
||||
}
|
||||
|
||||
// HasTriggers returns true if the workflow has any triggers defined
|
||||
func (w *Workflow) HasTriggers() bool {
|
||||
return len(w.Triggers) > 0
|
||||
}
|
||||
|
||||
// IsManualExecutionAllowed checks if manual (CLI) execution is allowed
|
||||
// Returns true if:
|
||||
// - No triggers are defined (default behavior allows manual)
|
||||
// - A manual trigger exists and is enabled
|
||||
// - No manual trigger is explicitly defined (default is enabled)
|
||||
func (w *Workflow) IsManualExecutionAllowed() bool {
|
||||
// If no triggers defined, manual is allowed (default behavior)
|
||||
if len(w.Triggers) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
// Look for explicit manual trigger
|
||||
for _, t := range w.Triggers {
|
||||
if t.On == TriggerManual {
|
||||
return t.Enabled // Use explicit setting
|
||||
}
|
||||
}
|
||||
|
||||
// No manual trigger defined among other triggers, default to true
|
||||
return true
|
||||
}
|
||||
|
||||
// GetEventTriggers returns all event-type triggers
|
||||
func (w *Workflow) GetEventTriggers() []Trigger {
|
||||
var triggers []Trigger
|
||||
for _, t := range w.Triggers {
|
||||
if t.On == TriggerEvent && t.Enabled {
|
||||
triggers = append(triggers, t)
|
||||
}
|
||||
}
|
||||
return triggers
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/config"
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/dialect/pgdialect"
|
||||
"github.com/uptrace/bun/dialect/sqlitedialect"
|
||||
"github.com/uptrace/bun/driver/pgdriver"
|
||||
"github.com/uptrace/bun/driver/sqliteshim"
|
||||
)
|
||||
|
||||
var db *bun.DB
|
||||
|
||||
// Connect establishes a database connection based on configuration
|
||||
func Connect(cfg *config.Config) (*bun.DB, error) {
|
||||
switch {
|
||||
case cfg.IsPostgres():
|
||||
return connectPostgres(cfg)
|
||||
case cfg.IsSQLite():
|
||||
return connectSQLite(cfg)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported database engine: %s", cfg.Database.DBEngine)
|
||||
}
|
||||
}
|
||||
|
||||
// connectSQLite establishes a SQLite connection
|
||||
func connectSQLite(cfg *config.Config) (*bun.DB, error) {
|
||||
dbPath := cfg.GetDBPath()
|
||||
|
||||
// Ensure directory exists
|
||||
dir := filepath.Dir(dbPath)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create database directory: %w", err)
|
||||
}
|
||||
|
||||
// Build DSN with pragmas for better performance
|
||||
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)", dbPath)
|
||||
|
||||
sqldb, err := sql.Open(sqliteshim.ShimName, dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open SQLite database: %w", err)
|
||||
}
|
||||
|
||||
// SQLite connection pooling settings
|
||||
sqldb.SetMaxOpenConns(1) // SQLite only supports one writer at a time
|
||||
sqldb.SetMaxIdleConns(1)
|
||||
|
||||
db = bun.NewDB(sqldb, sqlitedialect.New())
|
||||
|
||||
// Test connection
|
||||
if err := db.Ping(); err != nil {
|
||||
return nil, fmt.Errorf("failed to ping SQLite database: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// connectPostgres establishes a PostgreSQL connection
|
||||
func connectPostgres(cfg *config.Config) (*bun.DB, error) {
|
||||
dsn := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s",
|
||||
cfg.Database.Username,
|
||||
cfg.Database.Password,
|
||||
cfg.Database.Host,
|
||||
cfg.Database.Port,
|
||||
cfg.Database.DBName,
|
||||
getSSLMode(cfg.Database.SSLMode),
|
||||
)
|
||||
|
||||
sqldb := sql.OpenDB(pgdriver.NewConnector(pgdriver.WithDSN(dsn)))
|
||||
|
||||
// PostgreSQL connection pool settings
|
||||
sqldb.SetMaxOpenConns(25) // Limit concurrent connections
|
||||
sqldb.SetMaxIdleConns(5) // Keep some connections ready
|
||||
sqldb.SetConnMaxLifetime(time.Hour) // Recycle connections periodically
|
||||
sqldb.SetConnMaxIdleTime(10 * time.Minute) // Close idle connections
|
||||
|
||||
db = bun.NewDB(sqldb, pgdialect.New())
|
||||
|
||||
// Test connection
|
||||
if err := db.Ping(); err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to PostgreSQL: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// getSSLMode returns the SSL mode or default
|
||||
func getSSLMode(mode string) string {
|
||||
if mode == "" {
|
||||
return "disable"
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
// GetDB returns the global database instance
|
||||
func GetDB() *bun.DB {
|
||||
return db
|
||||
}
|
||||
|
||||
// SetDB sets the global database instance (for testing)
|
||||
func SetDB(newDB *bun.DB) {
|
||||
db = newDB
|
||||
}
|
||||
|
||||
// Close closes the database connection
|
||||
func Close() error {
|
||||
if db != nil {
|
||||
return db.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Migrate runs database migrations
|
||||
func Migrate(ctx context.Context) error {
|
||||
models := []interface{}{
|
||||
(*Run)(nil),
|
||||
(*StepResult)(nil),
|
||||
(*Artifact)(nil),
|
||||
(*Asset)(nil),
|
||||
(*EventLog)(nil),
|
||||
(*Schedule)(nil),
|
||||
(*Workspace)(nil),
|
||||
(*WorkflowMeta)(nil),
|
||||
(*Vulnerability)(nil),
|
||||
}
|
||||
|
||||
for _, model := range models {
|
||||
_, err := db.NewCreateTable().
|
||||
Model(model).
|
||||
IfNotExists().
|
||||
Exec(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create table: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create indexes for Asset table
|
||||
if err := createAssetIndexes(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create indexes for EventLog table
|
||||
if err := createEventLogIndexes(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create indexes for WorkflowMeta table
|
||||
if err := createWorkflowMetaIndexes(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create indexes for Vulnerability table
|
||||
if err := createVulnerabilityIndexes(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create indexes for Workspace table
|
||||
if err := createWorkspaceIndexes(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createAssetIndexes creates indexes for the assets table
|
||||
func createAssetIndexes(ctx context.Context) error {
|
||||
indexes := []string{
|
||||
"CREATE INDEX IF NOT EXISTS idx_assets_workspace ON assets(workspace)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_assets_asset_value ON assets(asset_value)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_assets_status_code ON assets(status_code)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_assets_host_ip ON assets(host_ip)",
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS idx_assets_unique ON assets(workspace, asset_value, url)",
|
||||
}
|
||||
|
||||
for _, idx := range indexes {
|
||||
if _, err := db.ExecContext(ctx, idx); err != nil {
|
||||
return fmt.Errorf("failed to create index: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createEventLogIndexes creates indexes for the event_logs table
|
||||
func createEventLogIndexes(ctx context.Context) error {
|
||||
indexes := []string{
|
||||
"CREATE INDEX IF NOT EXISTS idx_event_logs_topic ON event_logs(topic)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_event_logs_workspace ON event_logs(workspace)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_event_logs_run_id ON event_logs(run_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_event_logs_created_at ON event_logs(created_at)",
|
||||
}
|
||||
|
||||
for _, idx := range indexes {
|
||||
if _, err := db.ExecContext(ctx, idx); err != nil {
|
||||
return fmt.Errorf("failed to create index: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createWorkflowMetaIndexes creates indexes for the workflow_meta table
|
||||
func createWorkflowMetaIndexes(ctx context.Context) error {
|
||||
indexes := []string{
|
||||
"CREATE INDEX IF NOT EXISTS idx_workflow_meta_kind ON workflow_meta(kind)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_workflow_meta_checksum ON workflow_meta(checksum)",
|
||||
}
|
||||
|
||||
for _, idx := range indexes {
|
||||
if _, err := db.ExecContext(ctx, idx); err != nil {
|
||||
return fmt.Errorf("failed to create index: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createVulnerabilityIndexes creates indexes for the vulnerabilities table
|
||||
func createVulnerabilityIndexes(ctx context.Context) error {
|
||||
indexes := []string{
|
||||
"CREATE INDEX IF NOT EXISTS idx_vulnerabilities_workspace ON vulnerabilities(workspace)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_vulnerabilities_severity ON vulnerabilities(severity)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_vulnerabilities_confidence ON vulnerabilities(confidence)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_vulnerabilities_asset_value ON vulnerabilities(asset_value)",
|
||||
}
|
||||
|
||||
for _, idx := range indexes {
|
||||
if _, err := db.ExecContext(ctx, idx); err != nil {
|
||||
return fmt.Errorf("failed to create index: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createWorkspaceIndexes creates indexes for the workspaces table
|
||||
func createWorkspaceIndexes(ctx context.Context) error {
|
||||
indexes := []string{
|
||||
"CREATE INDEX IF NOT EXISTS idx_workspaces_data_source ON workspaces(data_source)",
|
||||
}
|
||||
|
||||
for _, idx := range indexes {
|
||||
if _, err := db.ExecContext(ctx, idx); err != nil {
|
||||
return fmt.Errorf("failed to create index: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Transaction wraps a function in a database transaction
|
||||
func Transaction(ctx context.Context, fn func(ctx context.Context, tx bun.Tx) error) error {
|
||||
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
return fn(ctx, tx)
|
||||
})
|
||||
}
|
||||
|
||||
// IsSQLite returns true if the current database is SQLite
|
||||
func IsSQLite() bool {
|
||||
if db == nil {
|
||||
return false
|
||||
}
|
||||
return db.Dialect().Name().String() == "sqlite"
|
||||
}
|
||||
|
||||
// IsPostgres returns true if the current database is PostgreSQL
|
||||
func IsPostgres() bool {
|
||||
if db == nil {
|
||||
return false
|
||||
}
|
||||
return db.Dialect().Name().String() == "pg"
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
// JSONLImporter handles batch import from JSONL files
|
||||
type JSONLImporter struct {
|
||||
db *bun.DB
|
||||
batchSize int
|
||||
}
|
||||
|
||||
// NewJSONLImporter creates a new JSONL importer
|
||||
func NewJSONLImporter(db *bun.DB) *JSONLImporter {
|
||||
return &JSONLImporter{
|
||||
db: db,
|
||||
batchSize: 100,
|
||||
}
|
||||
}
|
||||
|
||||
// WithBatchSize sets the batch size for imports
|
||||
func (i *JSONLImporter) WithBatchSize(size int) *JSONLImporter {
|
||||
if size > 0 {
|
||||
i.batchSize = size
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
// ImportResult holds import statistics
|
||||
type ImportResult struct {
|
||||
Total int `json:"total"`
|
||||
Imported int `json:"imported"`
|
||||
Updated int `json:"updated"`
|
||||
Failed int `json:"failed"`
|
||||
Errors []ImportError `json:"errors,omitempty"`
|
||||
Duration time.Duration `json:"duration"`
|
||||
}
|
||||
|
||||
// ImportError represents a single import error
|
||||
type ImportError struct {
|
||||
Line int `json:"line"`
|
||||
Error string `json:"error"`
|
||||
Data string `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
// ImportAssets imports assets from a JSONL file
|
||||
func (i *JSONLImporter) ImportAssets(ctx context.Context, filePath, workspace, source string) (*ImportResult, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
return i.ImportAssetsFromReader(ctx, file, workspace, source)
|
||||
}
|
||||
|
||||
// ImportAssetsFromReader imports assets from an io.Reader
|
||||
func (i *JSONLImporter) ImportAssetsFromReader(ctx context.Context, r io.Reader, workspace, source string) (*ImportResult, error) {
|
||||
startTime := time.Now()
|
||||
scanner := bufio.NewScanner(r)
|
||||
// Allow large lines (up to 10MB)
|
||||
scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
|
||||
|
||||
result := &ImportResult{}
|
||||
batch := make([]*Asset, 0, i.batchSize)
|
||||
|
||||
for scanner.Scan() {
|
||||
result.Total++
|
||||
line := scanner.Bytes()
|
||||
|
||||
// Skip empty lines
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
asset, err := ParseAssetLine(line, workspace, source)
|
||||
if err != nil {
|
||||
result.Failed++
|
||||
result.Errors = append(result.Errors, ImportError{
|
||||
Line: result.Total,
|
||||
Error: err.Error(),
|
||||
Data: truncateString(string(line), 200),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
batch = append(batch, asset)
|
||||
|
||||
if len(batch) >= i.batchSize {
|
||||
imported, err := i.insertAssetBatch(ctx, batch)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("batch insert failed at line %d: %w", result.Total, err)
|
||||
}
|
||||
result.Imported += imported
|
||||
batch = batch[:0]
|
||||
}
|
||||
}
|
||||
|
||||
// Insert remaining batch
|
||||
if len(batch) > 0 {
|
||||
imported, err := i.insertAssetBatch(ctx, batch)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("final batch insert failed: %w", err)
|
||||
}
|
||||
result.Imported += imported
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return result, fmt.Errorf("scanner error: %w", err)
|
||||
}
|
||||
|
||||
result.Duration = time.Since(startTime)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// insertAssetBatch inserts a batch of assets with upsert
|
||||
func (i *JSONLImporter) insertAssetBatch(ctx context.Context, assets []*Asset) (int, error) {
|
||||
if len(assets) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Use ON CONFLICT for upsert
|
||||
res, err := i.db.NewInsert().
|
||||
Model(&assets).
|
||||
On("CONFLICT (workspace, asset_value, url) DO UPDATE").
|
||||
Set("status_code = EXCLUDED.status_code").
|
||||
Set("title = EXCLUDED.title").
|
||||
Set("tech = EXCLUDED.tech").
|
||||
Set("content_type = EXCLUDED.content_type").
|
||||
Set("content_length = EXCLUDED.content_length").
|
||||
Set("host_ip = EXCLUDED.host_ip").
|
||||
Set("a_records = EXCLUDED.a_records").
|
||||
Set("tls = EXCLUDED.tls").
|
||||
Set("response_time = EXCLUDED.response_time").
|
||||
Set("words = EXCLUDED.words").
|
||||
Set("lines = EXCLUDED.lines").
|
||||
Set("remarks = EXCLUDED.remarks").
|
||||
Set("raw_data = EXCLUDED.raw_data").
|
||||
Set("updated_at = EXCLUDED.updated_at").
|
||||
Exec(ctx)
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
rowsAffected, _ := res.RowsAffected()
|
||||
return int(rowsAffected), nil
|
||||
}
|
||||
|
||||
// ParseAssetLine parses a single JSONL line into an Asset
|
||||
func ParseAssetLine(line []byte, defaultWorkspace, source string) (*Asset, error) {
|
||||
var raw map[string]interface{}
|
||||
if err := json.Unmarshal(line, &raw); err != nil {
|
||||
return nil, fmt.Errorf("invalid JSON: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
asset := &Asset{
|
||||
Workspace: defaultWorkspace,
|
||||
Source: source,
|
||||
RawJsonData: string(line),
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
// Map JSON fields to Asset struct
|
||||
// Required fields
|
||||
if v, ok := raw["workspace"].(string); ok && v != "" {
|
||||
asset.Workspace = v
|
||||
}
|
||||
if v, ok := raw["asset_value"].(string); ok {
|
||||
asset.AssetValue = v
|
||||
}
|
||||
|
||||
// HTTP data
|
||||
if v, ok := raw["url"].(string); ok {
|
||||
asset.URL = v
|
||||
}
|
||||
if v, ok := raw["input"].(string); ok {
|
||||
asset.Input = v
|
||||
}
|
||||
if v, ok := raw["scheme"].(string); ok {
|
||||
asset.Scheme = v
|
||||
}
|
||||
if v, ok := raw["method"].(string); ok {
|
||||
asset.Method = v
|
||||
}
|
||||
if v, ok := raw["path"].(string); ok {
|
||||
asset.Path = v
|
||||
}
|
||||
|
||||
// Response data
|
||||
if v, ok := raw["status_code"].(float64); ok {
|
||||
asset.StatusCode = int(v)
|
||||
}
|
||||
if v, ok := raw["content_type"].(string); ok {
|
||||
asset.ContentType = v
|
||||
}
|
||||
if v, ok := raw["content_length"].(float64); ok {
|
||||
asset.ContentLength = int64(v)
|
||||
}
|
||||
if v, ok := raw["title"].(string); ok {
|
||||
asset.Title = v
|
||||
}
|
||||
if v, ok := raw["words"].(float64); ok {
|
||||
asset.Words = int(v)
|
||||
}
|
||||
if v, ok := raw["lines"].(float64); ok {
|
||||
asset.Lines = int(v)
|
||||
}
|
||||
|
||||
// Network data
|
||||
if v, ok := raw["host_ip"].(string); ok {
|
||||
asset.HostIP = v
|
||||
}
|
||||
if v, ok := raw["a"].([]interface{}); ok {
|
||||
asset.DnsRecords = interfaceSliceToStringSlice(v)
|
||||
}
|
||||
if v, ok := raw["tls"].(string); ok {
|
||||
asset.TLS = v
|
||||
}
|
||||
|
||||
// Metadata
|
||||
if v, ok := raw["tech"].([]interface{}); ok {
|
||||
asset.Technologies = interfaceSliceToStringSlice(v)
|
||||
}
|
||||
if v, ok := raw["time"].(string); ok {
|
||||
asset.ResponseTime = v
|
||||
}
|
||||
if v, ok := raw["remarks"].(string); ok {
|
||||
asset.Labels = v
|
||||
}
|
||||
|
||||
// Validate required fields
|
||||
if asset.AssetValue == "" {
|
||||
return nil, fmt.Errorf("asset_value is required")
|
||||
}
|
||||
if asset.Workspace == "" {
|
||||
return nil, fmt.Errorf("workspace is required")
|
||||
}
|
||||
|
||||
return asset, nil
|
||||
}
|
||||
|
||||
// ImportEventLogs imports event logs from a JSONL file
|
||||
func (i *JSONLImporter) ImportEventLogs(ctx context.Context, filePath string) (*ImportResult, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
return i.ImportEventLogsFromReader(ctx, file)
|
||||
}
|
||||
|
||||
// ImportEventLogsFromReader imports event logs from an io.Reader
|
||||
func (i *JSONLImporter) ImportEventLogsFromReader(ctx context.Context, r io.Reader) (*ImportResult, error) {
|
||||
startTime := time.Now()
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
|
||||
|
||||
result := &ImportResult{}
|
||||
batch := make([]*EventLog, 0, i.batchSize)
|
||||
|
||||
for scanner.Scan() {
|
||||
result.Total++
|
||||
line := scanner.Bytes()
|
||||
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
event, err := ParseEventLogLine(line)
|
||||
if err != nil {
|
||||
result.Failed++
|
||||
result.Errors = append(result.Errors, ImportError{
|
||||
Line: result.Total,
|
||||
Error: err.Error(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
batch = append(batch, event)
|
||||
|
||||
if len(batch) >= i.batchSize {
|
||||
if _, err := i.db.NewInsert().Model(&batch).Exec(ctx); err != nil {
|
||||
return result, fmt.Errorf("batch insert failed: %w", err)
|
||||
}
|
||||
result.Imported += len(batch)
|
||||
batch = batch[:0]
|
||||
}
|
||||
}
|
||||
|
||||
if len(batch) > 0 {
|
||||
if _, err := i.db.NewInsert().Model(&batch).Exec(ctx); err != nil {
|
||||
return result, fmt.Errorf("final batch insert failed: %w", err)
|
||||
}
|
||||
result.Imported += len(batch)
|
||||
}
|
||||
|
||||
result.Duration = time.Since(startTime)
|
||||
return result, scanner.Err()
|
||||
}
|
||||
|
||||
// ParseEventLogLine parses a single JSONL line into an EventLog
|
||||
func ParseEventLogLine(line []byte) (*EventLog, error) {
|
||||
var event EventLog
|
||||
if err := json.Unmarshal(line, &event); err != nil {
|
||||
return nil, fmt.Errorf("invalid JSON: %w", err)
|
||||
}
|
||||
|
||||
if event.Topic == "" {
|
||||
return nil, fmt.Errorf("topic is required")
|
||||
}
|
||||
|
||||
if event.CreatedAt.IsZero() {
|
||||
event.CreatedAt = time.Now()
|
||||
}
|
||||
|
||||
return &event, nil
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func interfaceSliceToStringSlice(slice []interface{}) []string {
|
||||
result := make([]string, 0, len(slice))
|
||||
for _, v := range slice {
|
||||
if s, ok := v.(string); ok {
|
||||
result = append(result, s)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func truncateString(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
// Run represents a workflow execution
|
||||
type Run struct {
|
||||
bun.BaseModel `bun:"table:runs,alias:r"`
|
||||
|
||||
ID string `bun:"id,pk,type:text" json:"id"`
|
||||
RunID string `bun:"run_id,unique,notnull" json:"run_id"`
|
||||
WorkflowName string `bun:"workflow_name,notnull" json:"workflow_name"`
|
||||
WorkflowKind string `bun:"workflow_kind,notnull" json:"workflow_kind"`
|
||||
Target string `bun:"target,notnull" json:"target"`
|
||||
Params map[string]interface{} `bun:"params,type:json" json:"params"`
|
||||
Status string `bun:"status,notnull" json:"status"`
|
||||
WorkspacePath string `bun:"workspace_path" json:"workspace_path"`
|
||||
StartedAt *time.Time `bun:"started_at" json:"started_at"`
|
||||
CompletedAt *time.Time `bun:"completed_at" json:"completed_at"`
|
||||
ErrorMessage string `bun:"error_message" json:"error_message,omitempty"`
|
||||
CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
|
||||
UpdatedAt time.Time `bun:"updated_at,notnull,default:current_timestamp" json:"updated_at"`
|
||||
|
||||
// Scheduling context
|
||||
ScheduleID string `bun:"schedule_id" json:"schedule_id,omitempty"`
|
||||
TriggerType string `bun:"trigger_type" json:"trigger_type,omitempty"` // manual, cron, event
|
||||
TriggerName string `bun:"trigger_name" json:"trigger_name,omitempty"`
|
||||
|
||||
// Job grouping - multiple targets from same request share a JobID
|
||||
JobID string `bun:"job_id" json:"job_id,omitempty"`
|
||||
|
||||
// Progress tracking
|
||||
TotalSteps int `bun:"total_steps" json:"total_steps"`
|
||||
CompletedSteps int `bun:"completed_steps" json:"completed_steps"`
|
||||
|
||||
// Relations
|
||||
Steps []*StepResult `bun:"rel:has-many,join:id=run_id" json:"steps,omitempty"`
|
||||
Artifacts []*Artifact `bun:"rel:has-many,join:id=run_id" json:"artifacts,omitempty"`
|
||||
Events []*EventLog `bun:"rel:has-many,join:run_id=run_id" json:"events,omitempty"`
|
||||
}
|
||||
|
||||
// StepResult represents a step execution result
|
||||
type StepResult struct {
|
||||
bun.BaseModel `bun:"table:step_results,alias:sr"`
|
||||
|
||||
ID string `bun:"id,pk,type:text" json:"id"`
|
||||
RunID string `bun:"run_id,notnull,type:text" json:"run_id"`
|
||||
StepName string `bun:"step_name,notnull" json:"step_name"`
|
||||
StepType string `bun:"step_type,notnull" json:"step_type"`
|
||||
Status string `bun:"status,notnull" json:"status"`
|
||||
Command string `bun:"command" json:"command,omitempty"`
|
||||
Output string `bun:"output" json:"output,omitempty"`
|
||||
ErrorMessage string `bun:"error_message" json:"error_message,omitempty"`
|
||||
Exports map[string]interface{} `bun:"exports,type:json" json:"exports,omitempty"`
|
||||
DurationMs int64 `bun:"duration_ms" json:"duration_ms"`
|
||||
LogFile string `bun:"log_file" json:"log_file,omitempty"`
|
||||
StartedAt *time.Time `bun:"started_at" json:"started_at"`
|
||||
CompletedAt *time.Time `bun:"completed_at" json:"completed_at"`
|
||||
CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
|
||||
|
||||
// Relations
|
||||
Run *Run `bun:"rel:belongs-to,join:run_id=id" json:"run,omitempty"`
|
||||
}
|
||||
|
||||
// Artifact content type constants
|
||||
const (
|
||||
ContentTypeJSON = "json"
|
||||
ContentTypeJSONL = "jsonl"
|
||||
ContentTypeYAML = "yaml"
|
||||
ContentTypeHTML = "html"
|
||||
ContentTypeMarkdown = "md"
|
||||
ContentTypeLog = "log"
|
||||
ContentTypePDF = "pdf"
|
||||
ContentTypePNG = "png"
|
||||
ContentTypeText = "txt"
|
||||
ContentTypeZip = "zip"
|
||||
ContentTypeFolder = "folder"
|
||||
ContentTypeUnknown = "unknown"
|
||||
)
|
||||
|
||||
// Artifact type constants for categorization
|
||||
const (
|
||||
ArtifactTypeReport = "report" // Workflow reports from reports: section
|
||||
ArtifactTypeStateFile = "state_file" // State files like run-state.json
|
||||
ArtifactTypeOutput = "output" // General output files
|
||||
ArtifactTypeScreenshot = "screenshot" // Screenshots
|
||||
)
|
||||
|
||||
// Default state file names
|
||||
var DefaultStateFiles = []struct {
|
||||
Name string
|
||||
FileName string
|
||||
ContentType string
|
||||
ArtifactType string
|
||||
Description string
|
||||
}{
|
||||
{"state-execution-log", "run-execution.log", ContentTypeLog, ArtifactTypeStateFile, "Execution log file"},
|
||||
{"state-console-log", "run-console.log", ContentTypeLog, ArtifactTypeStateFile, "Console output capture"},
|
||||
{"state-completed", "run-completed.json", ContentTypeJSON, ArtifactTypeStateFile, "Completed state marker"},
|
||||
{"state-file", "run-state.json", ContentTypeJSON, ArtifactTypeStateFile, "Run state tracking file"},
|
||||
{"state-workflow", "run-workflow.yaml", ContentTypeYAML, ArtifactTypeStateFile, "Workflow definition used for the run"},
|
||||
}
|
||||
|
||||
// Artifact represents an output file from a run
|
||||
type Artifact struct {
|
||||
bun.BaseModel `bun:"table:artifacts,alias:a"`
|
||||
|
||||
ID string `bun:"id,pk,type:text" json:"id"`
|
||||
RunID string `bun:"run_id,notnull,type:text" json:"run_id"`
|
||||
Workspace string `bun:"workspace,notnull" json:"workspace"`
|
||||
Name string `bun:"name,notnull" json:"name"`
|
||||
ArtifactPath string `bun:"artifact_path,notnull" json:"artifact_path"`
|
||||
ArtifactType string `bun:"artifact_type" json:"artifact_type,omitempty"` // report, state_file, output, screenshot
|
||||
ContentType string `bun:"content_type" json:"content_type,omitempty"` // json, jsonl, yaml, html, md, log, pdf, png, txt, zip, folder, unknown
|
||||
SizeBytes int64 `bun:"size_bytes" json:"size_bytes"`
|
||||
LineCount int `bun:"line_count" json:"line_count"`
|
||||
Description string `bun:"description" json:"description,omitempty"`
|
||||
CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
|
||||
|
||||
// Relations
|
||||
Run *Run `bun:"rel:belongs-to,join:run_id=id" json:"run,omitempty"`
|
||||
}
|
||||
|
||||
// EventLog represents a system event for auditing and trigger history
|
||||
type EventLog struct {
|
||||
bun.BaseModel `bun:"table:event_logs,alias:el"`
|
||||
|
||||
ID int64 `bun:"id,pk,autoincrement" json:"id"`
|
||||
Topic string `bun:"topic,notnull" json:"topic"` // e.g., "webhook.received"
|
||||
EventID string `bun:"event_id" json:"event_id"` // UUID
|
||||
Name string `bun:"name" json:"name"` // e.g., "scan.started"
|
||||
Source string `bun:"source" json:"source"` // e.g., "scheduler", "api"
|
||||
DataType string `bun:"data_type" json:"data_type"` // e.g., "scan", "asset"
|
||||
Data string `bun:"data" json:"data"` // JSON payload
|
||||
|
||||
// Context
|
||||
Workspace string `bun:"workspace" json:"workspace,omitempty"`
|
||||
RunID string `bun:"run_id" json:"run_id,omitempty"`
|
||||
WorkflowName string `bun:"workflow_name" json:"workflow_name,omitempty"`
|
||||
|
||||
// Result
|
||||
Processed bool `bun:"processed,default:false" json:"processed"`
|
||||
ProcessedAt *time.Time `bun:"processed_at" json:"processed_at,omitempty"`
|
||||
Error string `bun:"error" json:"error,omitempty"`
|
||||
|
||||
CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
|
||||
}
|
||||
|
||||
// Schedule represents a workflow schedule
|
||||
type Schedule struct {
|
||||
bun.BaseModel `bun:"table:schedules,alias:sch"`
|
||||
|
||||
ID string `bun:"id,pk,type:text" json:"id"`
|
||||
Name string `bun:"name,notnull" json:"name"`
|
||||
WorkflowName string `bun:"workflow_name,notnull" json:"workflow_name"`
|
||||
WorkflowPath string `bun:"workflow_path,notnull" json:"workflow_path"`
|
||||
TriggerName string `bun:"trigger_name,notnull" json:"trigger_name"`
|
||||
TriggerType string `bun:"trigger_type,notnull" json:"trigger_type"`
|
||||
Schedule string `bun:"schedule" json:"schedule,omitempty"`
|
||||
EventTopic string `bun:"event_topic" json:"event_topic,omitempty"`
|
||||
WatchPath string `bun:"watch_path" json:"watch_path,omitempty"`
|
||||
InputConfig map[string]interface{} `bun:"input_config,type:json" json:"input_config,omitempty"`
|
||||
IsEnabled bool `bun:"is_enabled,default:true" json:"is_enabled"`
|
||||
LastRun *time.Time `bun:"last_run" json:"last_run,omitempty"`
|
||||
NextRun *time.Time `bun:"next_run" json:"next_run,omitempty"`
|
||||
RunCount int `bun:"run_count,default:0" json:"run_count"`
|
||||
CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
|
||||
UpdatedAt time.Time `bun:"updated_at,notnull,default:current_timestamp" json:"updated_at"`
|
||||
}
|
||||
|
||||
// Event topic constants
|
||||
const (
|
||||
TopicRunStarted = "run.started"
|
||||
TopicRunCompleted = "run.completed"
|
||||
TopicRunFailed = "run.failed"
|
||||
TopicAssetDiscovered = "asset.discovered"
|
||||
TopicAssetUpdated = "asset.updated"
|
||||
TopicWebhookReceived = "webhook.received"
|
||||
TopicScheduleTriggered = "schedule.triggered"
|
||||
TopicStepCompleted = "step.completed"
|
||||
TopicStepFailed = "step.failed"
|
||||
)
|
||||
|
||||
// Asset represents an HTTP endpoint/asset discovered during scanning
|
||||
type Asset struct {
|
||||
bun.BaseModel `bun:"table:assets,alias:as"`
|
||||
|
||||
ID int64 `bun:"id,pk,autoincrement" json:"id"`
|
||||
Workspace string `bun:"workspace,notnull" json:"workspace"`
|
||||
AssetValue string `bun:"asset_value,notnull" json:"asset_value"`
|
||||
|
||||
// HTTP data
|
||||
URL string `bun:"url" json:"url,omitempty"`
|
||||
Input string `bun:"input" json:"input,omitempty"`
|
||||
Scheme string `bun:"scheme" json:"scheme,omitempty"`
|
||||
Method string `bun:"method" json:"method,omitempty"`
|
||||
Path string `bun:"path" json:"path,omitempty"`
|
||||
|
||||
// Response data
|
||||
StatusCode int `bun:"status_code" json:"status_code,omitempty"`
|
||||
ContentType string `bun:"content_type" json:"content_type,omitempty"`
|
||||
ContentLength int64 `bun:"content_length" json:"content_length,omitempty"`
|
||||
Title string `bun:"title" json:"title,omitempty"`
|
||||
Words int `bun:"words" json:"words,omitempty"`
|
||||
Lines int `bun:"lines" json:"lines,omitempty"`
|
||||
|
||||
// Network data
|
||||
HostIP string `bun:"host_ip" json:"host_ip,omitempty"`
|
||||
DnsRecords []string `bun:"dns_records,type:json" json:"a,omitempty"`
|
||||
TLS string `bun:"tls" json:"tls,omitempty"`
|
||||
|
||||
// Metadata
|
||||
AssetType string `bun:"asset_type" json:"asset_type,omitempty"`
|
||||
Technologies []string `bun:"technologies,type:json" json:"tech,omitempty"`
|
||||
ResponseTime string `bun:"response_time" json:"time,omitempty"`
|
||||
Labels string `bun:"labels" json:"remarks,omitempty"`
|
||||
Source string `bun:"source" json:"source,omitempty"` // e.g., "httpx", "nuclei"
|
||||
RawJsonData string `bun:"raw_json_data" json:"raw_json_data,omitempty"` // Original JSON
|
||||
RawResponse string `bun:"raw_response" json:"raw_response,omitempty"`
|
||||
ScreenshotBase64Data string `bun:"screenshot_base64_data" json:"screenshot_base64_data,omitempty"`
|
||||
|
||||
// Timestamps
|
||||
CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
|
||||
UpdatedAt time.Time `bun:"updated_at,notnull,default:current_timestamp" json:"updated_at"`
|
||||
}
|
||||
|
||||
// Workspace represents a scan workspace with aggregated statistics
|
||||
type Workspace struct {
|
||||
bun.BaseModel `bun:"table:workspaces,alias:ws"`
|
||||
|
||||
ID int64 `bun:"id,pk,autoincrement" json:"id"`
|
||||
Name string `bun:"name,unique,notnull" json:"name"`
|
||||
LocalPath string `bun:"local_path" json:"local_path"`
|
||||
DataSource string `bun:"data_source,default:'local'" json:"data_source"` // local, cloud, imported
|
||||
|
||||
// Asset statistics
|
||||
TotalAssets int `bun:"total_assets,default:0" json:"total_assets"`
|
||||
TotalSubdomains int `bun:"total_subdomains,default:0" json:"total_subdomains"`
|
||||
TotalURLs int `bun:"total_urls,default:0" json:"total_urls"`
|
||||
TotalVulns int `bun:"total_vulns,default:0" json:"total_vulns"`
|
||||
TotalIPs int `bun:"total_ips,default:0" json:"total_ips"`
|
||||
TotalLinks int `bun:"total_links,default:0" json:"total_links"`
|
||||
TotalContent int `bun:"total_content,default:0" json:"total_content"`
|
||||
TotalArchive int `bun:"total_archive,default:0" json:"total_archive"`
|
||||
|
||||
// Vulnerability severity breakdown
|
||||
VulnCritical int `bun:"vuln_critical,default:0" json:"vuln_critical"`
|
||||
VulnHigh int `bun:"vuln_high,default:0" json:"vuln_high"`
|
||||
VulnMedium int `bun:"vuln_medium,default:0" json:"vuln_medium"`
|
||||
VulnLow int `bun:"vuln_low,default:0" json:"vuln_low"`
|
||||
VulnPotential int `bun:"vuln_potential,default:0" json:"vuln_potential"`
|
||||
|
||||
// Risk and metadata
|
||||
RiskScore float64 `bun:"risk_score,default:0" json:"risk_score"`
|
||||
Tags []string `bun:"tags,type:json" json:"tags"`
|
||||
|
||||
// Run info
|
||||
LastRun *time.Time `bun:"last_run" json:"last_run"`
|
||||
RunWorkflow string `bun:"run_workflow" json:"run_workflow"`
|
||||
|
||||
// State file paths
|
||||
StateExecutionLog string `bun:"state_execution_log" json:"state_execution_log,omitempty"`
|
||||
StateCompletedFile string `bun:"state_completed_file" json:"state_completed_file,omitempty"`
|
||||
StateWorkflowFile string `bun:"state_workflow_file" json:"state_workflow_file,omitempty"`
|
||||
StateWorkflowFolder string `bun:"state_workflow_folder" json:"state_workflow_folder,omitempty"`
|
||||
|
||||
// Timestamps
|
||||
CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
|
||||
UpdatedAt time.Time `bun:"updated_at,notnull,default:current_timestamp" json:"updated_at"`
|
||||
}
|
||||
|
||||
// WorkflowMeta stores workflow metadata in database for faster querying
|
||||
type WorkflowMeta struct {
|
||||
bun.BaseModel `bun:"table:workflow_meta,alias:wm"`
|
||||
|
||||
ID int64 `bun:"id,pk,autoincrement" json:"id"`
|
||||
Name string `bun:"name,unique,notnull" json:"name"`
|
||||
Kind string `bun:"kind,notnull" json:"kind"` // "module" or "flow"
|
||||
Description string `bun:"description" json:"description"`
|
||||
FilePath string `bun:"file_path,notnull" json:"file_path"`
|
||||
Checksum string `bun:"checksum" json:"checksum"` // SHA256 for change detection
|
||||
Tags []string `bun:"tags,type:json" json:"tags"`
|
||||
|
||||
// Metadata
|
||||
StepCount int `bun:"step_count" json:"step_count"`
|
||||
ModuleCount int `bun:"module_count" json:"module_count"`
|
||||
ParamsJSON string `bun:"params_json" json:"params_json"` // Serialized params
|
||||
|
||||
// Timestamps
|
||||
IndexedAt time.Time `bun:"indexed_at,notnull" json:"indexed_at"`
|
||||
CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
|
||||
UpdatedAt time.Time `bun:"updated_at,notnull,default:current_timestamp" json:"updated_at"`
|
||||
}
|
||||
|
||||
// Vulnerability represents a security vulnerability discovered during scanning
|
||||
type Vulnerability struct {
|
||||
bun.BaseModel `bun:"table:vulnerabilities,alias:vl"`
|
||||
|
||||
ID int64 `bun:"id,pk,autoincrement" json:"id"`
|
||||
Workspace string `bun:"workspace,notnull" json:"workspace"`
|
||||
VulnInfo string `bun:"vuln_info" json:"vuln_info"`
|
||||
VulnTitle string `bun:"vuln_title" json:"vuln_title"`
|
||||
VulnDesc string `bun:"vuln_desc" json:"vuln_desc"`
|
||||
VulnPOC string `bun:"vuln_poc" json:"vuln_poc"`
|
||||
Severity string `bun:"severity" json:"severity"`
|
||||
Confidence string `bun:"confidence" json:"confidence"` // Certain, Firm, Tentative, Manual Review Required
|
||||
AssetType string `bun:"asset_type" json:"asset_type"`
|
||||
AssetValue string `bun:"asset_value" json:"asset_value"`
|
||||
Tags []string `bun:"tags,type:json" json:"tags,omitempty"`
|
||||
DetailHTTPRequest string `bun:"detail_http_request" json:"detail_http_request"`
|
||||
DetailHTTPResponse string `bun:"detail_http_response" json:"detail_http_response"`
|
||||
RawVulnJSON string `bun:"raw_vuln_json" json:"raw_vuln_json"`
|
||||
|
||||
// Timestamps
|
||||
CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
|
||||
UpdatedAt time.Time `bun:"updated_at,notnull,default:current_timestamp" json:"updated_at"`
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/database"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
// AssetRepository handles asset database operations
|
||||
type AssetRepository struct {
|
||||
db *bun.DB
|
||||
}
|
||||
|
||||
// NewAssetRepository creates a new asset repository
|
||||
func NewAssetRepository(db *bun.DB) *AssetRepository {
|
||||
return &AssetRepository{db: db}
|
||||
}
|
||||
|
||||
// AssetQuery represents query parameters for asset search
|
||||
type AssetQuery struct {
|
||||
Workspace string
|
||||
AssetValue string
|
||||
HostIP string
|
||||
StatusCode int
|
||||
ContentType string
|
||||
Tech string
|
||||
Source string
|
||||
Page int
|
||||
PerPage int
|
||||
}
|
||||
|
||||
// Create creates a new asset
|
||||
func (r *AssetRepository) Create(ctx context.Context, asset *database.Asset) error {
|
||||
_, err := r.db.NewInsert().Model(asset).Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetByID retrieves an asset by ID
|
||||
func (r *AssetRepository) GetByID(ctx context.Context, id int64) (*database.Asset, error) {
|
||||
asset := new(database.Asset)
|
||||
err := r.db.NewSelect().
|
||||
Model(asset).
|
||||
Where("id = ?", id).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return asset, nil
|
||||
}
|
||||
|
||||
// Update updates an existing asset
|
||||
func (r *AssetRepository) Update(ctx context.Context, asset *database.Asset) error {
|
||||
_, err := r.db.NewUpdate().
|
||||
Model(asset).
|
||||
WherePK().
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete deletes an asset by ID
|
||||
func (r *AssetRepository) Delete(ctx context.Context, id int64) error {
|
||||
_, err := r.db.NewDelete().
|
||||
Model((*database.Asset)(nil)).
|
||||
Where("id = ?", id).
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteByWorkspace deletes all assets in a workspace
|
||||
func (r *AssetRepository) DeleteByWorkspace(ctx context.Context, workspace string) (int64, error) {
|
||||
res, err := r.db.NewDelete().
|
||||
Model((*database.Asset)(nil)).
|
||||
Where("workspace = ?", workspace).
|
||||
Exec(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
// ListByWorkspace lists assets for a workspace with pagination
|
||||
func (r *AssetRepository) ListByWorkspace(ctx context.Context, workspace string, page, perPage int) ([]*database.Asset, int, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 {
|
||||
perPage = 50
|
||||
}
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
var assets []*database.Asset
|
||||
count, err := r.db.NewSelect().
|
||||
Model(&assets).
|
||||
Where("workspace = ?", workspace).
|
||||
Order("created_at DESC").
|
||||
Limit(perPage).
|
||||
Offset(offset).
|
||||
ScanAndCount(ctx)
|
||||
|
||||
return assets, count, err
|
||||
}
|
||||
|
||||
// ListByAssetValue lists assets for a specific asset value
|
||||
func (r *AssetRepository) ListByAssetValue(ctx context.Context, assetValue string) ([]*database.Asset, error) {
|
||||
var assets []*database.Asset
|
||||
err := r.db.NewSelect().
|
||||
Model(&assets).
|
||||
Where("asset_value = ?", assetValue).
|
||||
Order("created_at DESC").
|
||||
Scan(ctx)
|
||||
return assets, err
|
||||
}
|
||||
|
||||
// ListByHostIP lists assets for a specific IP
|
||||
func (r *AssetRepository) ListByHostIP(ctx context.Context, hostIP string) ([]*database.Asset, error) {
|
||||
var assets []*database.Asset
|
||||
err := r.db.NewSelect().
|
||||
Model(&assets).
|
||||
Where("host_ip = ?", hostIP).
|
||||
Order("created_at DESC").
|
||||
Scan(ctx)
|
||||
return assets, err
|
||||
}
|
||||
|
||||
// ListByStatus lists assets with a specific status code
|
||||
func (r *AssetRepository) ListByStatus(ctx context.Context, workspace string, statusCode int) ([]*database.Asset, error) {
|
||||
var assets []*database.Asset
|
||||
query := r.db.NewSelect().Model(&assets)
|
||||
|
||||
if workspace != "" {
|
||||
query = query.Where("workspace = ?", workspace)
|
||||
}
|
||||
|
||||
err := query.
|
||||
Where("status_code = ?", statusCode).
|
||||
Order("created_at DESC").
|
||||
Scan(ctx)
|
||||
return assets, err
|
||||
}
|
||||
|
||||
// Search searches assets with multiple criteria
|
||||
func (r *AssetRepository) Search(ctx context.Context, query AssetQuery) ([]*database.Asset, int, error) {
|
||||
if query.Page < 1 {
|
||||
query.Page = 1
|
||||
}
|
||||
if query.PerPage < 1 {
|
||||
query.PerPage = 50
|
||||
}
|
||||
offset := (query.Page - 1) * query.PerPage
|
||||
|
||||
var assets []*database.Asset
|
||||
q := r.db.NewSelect().Model(&assets)
|
||||
|
||||
if query.Workspace != "" {
|
||||
q = q.Where("workspace = ?", query.Workspace)
|
||||
}
|
||||
if query.AssetValue != "" {
|
||||
q = q.Where("asset_value LIKE ?", "%"+query.AssetValue+"%")
|
||||
}
|
||||
if query.HostIP != "" {
|
||||
q = q.Where("host_ip = ?", query.HostIP)
|
||||
}
|
||||
if query.StatusCode > 0 {
|
||||
q = q.Where("status_code = ?", query.StatusCode)
|
||||
}
|
||||
if query.ContentType != "" {
|
||||
q = q.Where("content_type LIKE ?", "%"+query.ContentType+"%")
|
||||
}
|
||||
if query.Source != "" {
|
||||
q = q.Where("source = ?", query.Source)
|
||||
}
|
||||
// Note: Tech search would need JSON-specific query depending on database
|
||||
|
||||
count, err := q.
|
||||
Order("created_at DESC").
|
||||
Limit(query.PerPage).
|
||||
Offset(offset).
|
||||
ScanAndCount(ctx)
|
||||
|
||||
return assets, count, err
|
||||
}
|
||||
|
||||
// ImportFromJSONL imports assets from a JSONL file
|
||||
func (r *AssetRepository) ImportFromJSONL(ctx context.Context, filePath, workspace, source string) (*database.ImportResult, error) {
|
||||
importer := database.NewJSONLImporter(r.db)
|
||||
return importer.ImportAssets(ctx, filePath, workspace, source)
|
||||
}
|
||||
|
||||
// ImportFromReader imports assets from an io.Reader
|
||||
func (r *AssetRepository) ImportFromReader(ctx context.Context, reader io.Reader, workspace, source string) (*database.ImportResult, error) {
|
||||
importer := database.NewJSONLImporter(r.db)
|
||||
return importer.ImportAssetsFromReader(ctx, reader, workspace, source)
|
||||
}
|
||||
|
||||
// CountByWorkspace returns the count of assets in a workspace
|
||||
func (r *AssetRepository) CountByWorkspace(ctx context.Context, workspace string) (int, error) {
|
||||
return r.db.NewSelect().
|
||||
Model((*database.Asset)(nil)).
|
||||
Where("workspace = ?", workspace).
|
||||
Count(ctx)
|
||||
}
|
||||
|
||||
// GetTechSummary returns a summary of technologies found in a workspace
|
||||
func (r *AssetRepository) GetTechSummary(ctx context.Context, workspace string) (map[string]int, error) {
|
||||
// This implementation varies by database
|
||||
// For SQLite/PostgreSQL with JSON support, we need to unnest the array
|
||||
var results []struct {
|
||||
Tech string `bun:"tech"`
|
||||
Count int `bun:"count"`
|
||||
}
|
||||
|
||||
// Simple approach: fetch all and count in Go
|
||||
// For production, use database-specific JSON functions
|
||||
var assets []*database.Asset
|
||||
err := r.db.NewSelect().
|
||||
Model(&assets).
|
||||
Column("tech").
|
||||
Where("workspace = ?", workspace).
|
||||
Where("tech IS NOT NULL").
|
||||
Scan(ctx)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
techCount := make(map[string]int)
|
||||
for _, asset := range assets {
|
||||
for _, tech := range asset.Technologies {
|
||||
techCount[tech]++
|
||||
}
|
||||
}
|
||||
|
||||
_ = results // unused in simple implementation
|
||||
return techCount, nil
|
||||
}
|
||||
|
||||
// GetStatusSummary returns a summary of status codes in a workspace
|
||||
func (r *AssetRepository) GetStatusSummary(ctx context.Context, workspace string) (map[int]int, error) {
|
||||
var results []struct {
|
||||
StatusCode int `bun:"status_code"`
|
||||
Count int `bun:"count"`
|
||||
}
|
||||
|
||||
err := r.db.NewSelect().
|
||||
Model((*database.Asset)(nil)).
|
||||
ColumnExpr("status_code, COUNT(*) AS count").
|
||||
Where("workspace = ?", workspace).
|
||||
Where("status_code > 0").
|
||||
Group("status_code").
|
||||
Order("count DESC").
|
||||
Scan(ctx, &results)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
summary := make(map[int]int)
|
||||
for _, r := range results {
|
||||
summary[r.StatusCode] = r.Count
|
||||
}
|
||||
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
// GetAssetValueSummary returns a summary of unique asset values in a workspace
|
||||
func (r *AssetRepository) GetAssetValueSummary(ctx context.Context, workspace string) (int, error) {
|
||||
var count int
|
||||
err := r.db.NewSelect().
|
||||
Model((*database.Asset)(nil)).
|
||||
ColumnExpr("COUNT(DISTINCT asset_value)").
|
||||
Where("workspace = ?", workspace).
|
||||
Scan(ctx, &count)
|
||||
|
||||
return count, err
|
||||
}
|
||||
|
||||
// Upsert creates or updates an asset based on workspace, asset_value, url
|
||||
func (r *AssetRepository) Upsert(ctx context.Context, asset *database.Asset) error {
|
||||
_, err := r.db.NewInsert().
|
||||
Model(asset).
|
||||
On("CONFLICT (workspace, asset_value, url) DO UPDATE").
|
||||
Set("status_code = EXCLUDED.status_code").
|
||||
Set("title = EXCLUDED.title").
|
||||
Set("tech = EXCLUDED.tech").
|
||||
Set("content_type = EXCLUDED.content_type").
|
||||
Set("content_length = EXCLUDED.content_length").
|
||||
Set("host_ip = EXCLUDED.host_ip").
|
||||
Set("a_records = EXCLUDED.a_records").
|
||||
Set("tls = EXCLUDED.tls").
|
||||
Set("response_time = EXCLUDED.response_time").
|
||||
Set("words = EXCLUDED.words").
|
||||
Set("lines = EXCLUDED.lines").
|
||||
Set("remarks = EXCLUDED.remarks").
|
||||
Set("raw_data = EXCLUDED.raw_data").
|
||||
Set("updated_at = EXCLUDED.updated_at").
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// BulkUpsert performs bulk upsert of assets
|
||||
func (r *AssetRepository) BulkUpsert(ctx context.Context, assets []*database.Asset) error {
|
||||
if len(assets) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := r.db.NewInsert().
|
||||
Model(&assets).
|
||||
On("CONFLICT (workspace, asset_value, url) DO UPDATE").
|
||||
Set("status_code = EXCLUDED.status_code").
|
||||
Set("title = EXCLUDED.title").
|
||||
Set("tech = EXCLUDED.tech").
|
||||
Set("content_type = EXCLUDED.content_type").
|
||||
Set("content_length = EXCLUDED.content_length").
|
||||
Set("host_ip = EXCLUDED.host_ip").
|
||||
Set("a_records = EXCLUDED.a_records").
|
||||
Set("tls = EXCLUDED.tls").
|
||||
Set("response_time = EXCLUDED.response_time").
|
||||
Set("words = EXCLUDED.words").
|
||||
Set("lines = EXCLUDED.lines").
|
||||
Set("remarks = EXCLUDED.remarks").
|
||||
Set("raw_data = EXCLUDED.raw_data").
|
||||
Set("updated_at = EXCLUDED.updated_at").
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// ExportToJSONL exports assets to a JSONL writer
|
||||
func (r *AssetRepository) ExportToJSONL(ctx context.Context, workspace string, w io.Writer) (int, error) {
|
||||
var assets []*database.Asset
|
||||
err := r.db.NewSelect().
|
||||
Model(&assets).
|
||||
Where("workspace = ?", workspace).
|
||||
Order("created_at ASC").
|
||||
Scan(ctx)
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
count := 0
|
||||
for _, asset := range assets {
|
||||
if asset.RawJsonData != "" {
|
||||
_, err := fmt.Fprintf(w, "%s\n", asset.RawJsonData)
|
||||
if err != nil {
|
||||
return count, err
|
||||
}
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/database"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
// EventLogRepository handles event log database operations
|
||||
type EventLogRepository struct {
|
||||
db *bun.DB
|
||||
}
|
||||
|
||||
// NewEventLogRepository creates a new event log repository
|
||||
func NewEventLogRepository(db *bun.DB) *EventLogRepository {
|
||||
return &EventLogRepository{db: db}
|
||||
}
|
||||
|
||||
// EventLogQuery represents query parameters for event search
|
||||
type EventLogQuery struct {
|
||||
Topic string
|
||||
Name string
|
||||
Source string
|
||||
Workspace string
|
||||
ScanID string
|
||||
WorkflowName string
|
||||
Processed *bool
|
||||
StartTime *time.Time
|
||||
EndTime *time.Time
|
||||
Page int
|
||||
PerPage int
|
||||
}
|
||||
|
||||
// Create creates a new event log
|
||||
func (r *EventLogRepository) Create(ctx context.Context, event *database.EventLog) error {
|
||||
if event.CreatedAt.IsZero() {
|
||||
event.CreatedAt = time.Now()
|
||||
}
|
||||
_, err := r.db.NewInsert().Model(event).Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetByID retrieves an event log by ID
|
||||
func (r *EventLogRepository) GetByID(ctx context.Context, id int64) (*database.EventLog, error) {
|
||||
event := new(database.EventLog)
|
||||
err := r.db.NewSelect().
|
||||
Model(event).
|
||||
Where("id = ?", id).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return event, nil
|
||||
}
|
||||
|
||||
// GetByEventID retrieves an event log by event ID (UUID)
|
||||
func (r *EventLogRepository) GetByEventID(ctx context.Context, eventID string) (*database.EventLog, error) {
|
||||
event := new(database.EventLog)
|
||||
err := r.db.NewSelect().
|
||||
Model(event).
|
||||
Where("event_id = ?", eventID).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return event, nil
|
||||
}
|
||||
|
||||
// Update updates an existing event log
|
||||
func (r *EventLogRepository) Update(ctx context.Context, event *database.EventLog) error {
|
||||
_, err := r.db.NewUpdate().
|
||||
Model(event).
|
||||
WherePK().
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete deletes an event log by ID
|
||||
func (r *EventLogRepository) Delete(ctx context.Context, id int64) error {
|
||||
_, err := r.db.NewDelete().
|
||||
Model((*database.EventLog)(nil)).
|
||||
Where("id = ?", id).
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListByTopic lists events by topic with pagination
|
||||
func (r *EventLogRepository) ListByTopic(ctx context.Context, topic string, page, perPage int) ([]*database.EventLog, int, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 {
|
||||
perPage = 50
|
||||
}
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
var events []*database.EventLog
|
||||
count, err := r.db.NewSelect().
|
||||
Model(&events).
|
||||
Where("topic = ?", topic).
|
||||
Order("created_at DESC").
|
||||
Limit(perPage).
|
||||
Offset(offset).
|
||||
ScanAndCount(ctx)
|
||||
|
||||
return events, count, err
|
||||
}
|
||||
|
||||
// ListByScanID lists events for a specific scan (uses run_id column)
|
||||
func (r *EventLogRepository) ListByScanID(ctx context.Context, scanID string) ([]*database.EventLog, error) {
|
||||
var events []*database.EventLog
|
||||
err := r.db.NewSelect().
|
||||
Model(&events).
|
||||
Where("run_id = ?", scanID).
|
||||
Order("created_at ASC").
|
||||
Scan(ctx)
|
||||
return events, err
|
||||
}
|
||||
|
||||
// ListByWorkspace lists events for a workspace with pagination
|
||||
func (r *EventLogRepository) ListByWorkspace(ctx context.Context, workspace string, page, perPage int) ([]*database.EventLog, int, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 {
|
||||
perPage = 50
|
||||
}
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
var events []*database.EventLog
|
||||
count, err := r.db.NewSelect().
|
||||
Model(&events).
|
||||
Where("workspace = ?", workspace).
|
||||
Order("created_at DESC").
|
||||
Limit(perPage).
|
||||
Offset(offset).
|
||||
ScanAndCount(ctx)
|
||||
|
||||
return events, count, err
|
||||
}
|
||||
|
||||
// ListUnprocessed lists unprocessed events
|
||||
func (r *EventLogRepository) ListUnprocessed(ctx context.Context, limit int) ([]*database.EventLog, error) {
|
||||
if limit < 1 {
|
||||
limit = 100
|
||||
}
|
||||
|
||||
var events []*database.EventLog
|
||||
err := r.db.NewSelect().
|
||||
Model(&events).
|
||||
Where("processed = ?", false).
|
||||
Order("created_at ASC").
|
||||
Limit(limit).
|
||||
Scan(ctx)
|
||||
return events, err
|
||||
}
|
||||
|
||||
// MarkProcessed marks an event as processed
|
||||
func (r *EventLogRepository) MarkProcessed(ctx context.Context, id int64, errorMsg string) error {
|
||||
now := time.Now()
|
||||
_, err := r.db.NewUpdate().
|
||||
Model((*database.EventLog)(nil)).
|
||||
Set("processed = ?", true).
|
||||
Set("processed_at = ?", now).
|
||||
Set("error = ?", errorMsg).
|
||||
Where("id = ?", id).
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Search searches events with multiple criteria
|
||||
func (r *EventLogRepository) Search(ctx context.Context, query EventLogQuery) ([]*database.EventLog, int, error) {
|
||||
if query.Page < 1 {
|
||||
query.Page = 1
|
||||
}
|
||||
if query.PerPage < 1 {
|
||||
query.PerPage = 50
|
||||
}
|
||||
offset := (query.Page - 1) * query.PerPage
|
||||
|
||||
var events []*database.EventLog
|
||||
q := r.db.NewSelect().Model(&events)
|
||||
|
||||
if query.Topic != "" {
|
||||
q = q.Where("topic = ?", query.Topic)
|
||||
}
|
||||
if query.Name != "" {
|
||||
q = q.Where("name = ?", query.Name)
|
||||
}
|
||||
if query.Source != "" {
|
||||
q = q.Where("source = ?", query.Source)
|
||||
}
|
||||
if query.Workspace != "" {
|
||||
q = q.Where("workspace = ?", query.Workspace)
|
||||
}
|
||||
if query.ScanID != "" {
|
||||
q = q.Where("run_id = ?", query.ScanID)
|
||||
}
|
||||
if query.WorkflowName != "" {
|
||||
q = q.Where("workflow_name = ?", query.WorkflowName)
|
||||
}
|
||||
if query.Processed != nil {
|
||||
q = q.Where("processed = ?", *query.Processed)
|
||||
}
|
||||
if query.StartTime != nil {
|
||||
q = q.Where("created_at >= ?", *query.StartTime)
|
||||
}
|
||||
if query.EndTime != nil {
|
||||
q = q.Where("created_at <= ?", *query.EndTime)
|
||||
}
|
||||
|
||||
count, err := q.
|
||||
Order("created_at DESC").
|
||||
Limit(query.PerPage).
|
||||
Offset(offset).
|
||||
ScanAndCount(ctx)
|
||||
|
||||
return events, count, err
|
||||
}
|
||||
|
||||
// CountByTopic returns the count of events for a topic
|
||||
func (r *EventLogRepository) CountByTopic(ctx context.Context, topic string) (int, error) {
|
||||
return r.db.NewSelect().
|
||||
Model((*database.EventLog)(nil)).
|
||||
Where("topic = ?", topic).
|
||||
Count(ctx)
|
||||
}
|
||||
|
||||
// GetTopicSummary returns a summary of events by topic
|
||||
func (r *EventLogRepository) GetTopicSummary(ctx context.Context) (map[string]int, error) {
|
||||
var results []struct {
|
||||
Topic string `bun:"topic"`
|
||||
Count int `bun:"count"`
|
||||
}
|
||||
|
||||
err := r.db.NewSelect().
|
||||
Model((*database.EventLog)(nil)).
|
||||
ColumnExpr("topic, COUNT(*) AS count").
|
||||
Group("topic").
|
||||
Order("count DESC").
|
||||
Scan(ctx, &results)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
summary := make(map[string]int)
|
||||
for _, r := range results {
|
||||
summary[r.Topic] = r.Count
|
||||
}
|
||||
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
// DeleteOlderThan deletes events older than the specified time
|
||||
func (r *EventLogRepository) DeleteOlderThan(ctx context.Context, before time.Time) (int64, error) {
|
||||
res, err := r.db.NewDelete().
|
||||
Model((*database.EventLog)(nil)).
|
||||
Where("created_at < ?", before).
|
||||
Exec(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
// DeleteByWorkspace deletes all events for a workspace
|
||||
func (r *EventLogRepository) DeleteByWorkspace(ctx context.Context, workspace string) (int64, error) {
|
||||
res, err := r.db.NewDelete().
|
||||
Model((*database.EventLog)(nil)).
|
||||
Where("workspace = ?", workspace).
|
||||
Exec(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
// CreateBatch creates multiple event logs in a batch
|
||||
func (r *EventLogRepository) CreateBatch(ctx context.Context, events []*database.EventLog) error {
|
||||
if len(events) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for _, event := range events {
|
||||
if event.CreatedAt.IsZero() {
|
||||
event.CreatedAt = now
|
||||
}
|
||||
}
|
||||
|
||||
_, err := r.db.NewInsert().Model(&events).Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetRecentByWorkflow gets recent events for a workflow
|
||||
func (r *EventLogRepository) GetRecentByWorkflow(ctx context.Context, workflowName string, limit int) ([]*database.EventLog, error) {
|
||||
if limit < 1 {
|
||||
limit = 10
|
||||
}
|
||||
|
||||
var events []*database.EventLog
|
||||
err := r.db.NewSelect().
|
||||
Model(&events).
|
||||
Where("workflow_name = ?", workflowName).
|
||||
Order("created_at DESC").
|
||||
Limit(limit).
|
||||
Scan(ctx)
|
||||
return events, err
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/database"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
// RunRepository handles run database operations
|
||||
type RunRepository struct {
|
||||
db *bun.DB
|
||||
}
|
||||
|
||||
// NewRunRepository creates a new run repository
|
||||
func NewRunRepository(db *bun.DB) *RunRepository {
|
||||
return &RunRepository{db: db}
|
||||
}
|
||||
|
||||
// Create creates a new run
|
||||
func (r *RunRepository) Create(ctx context.Context, scan *database.Run) error {
|
||||
_, err := r.db.NewInsert().Model(scan).Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetByID gets a run by ID
|
||||
func (r *RunRepository) GetByID(ctx context.Context, id string) (*database.Run, error) {
|
||||
scan := new(database.Run)
|
||||
err := r.db.NewSelect().
|
||||
Model(scan).
|
||||
Where("id = ?", id).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scan, nil
|
||||
}
|
||||
|
||||
// GetByRunID gets a run by run ID
|
||||
func (r *RunRepository) GetByRunID(ctx context.Context, runID string) (*database.Run, error) {
|
||||
scan := new(database.Run)
|
||||
err := r.db.NewSelect().
|
||||
Model(scan).
|
||||
Where("run_id = ?", runID).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scan, nil
|
||||
}
|
||||
|
||||
// Update updates a run
|
||||
func (r *RunRepository) Update(ctx context.Context, scan *database.Run) error {
|
||||
_, err := r.db.NewUpdate().
|
||||
Model(scan).
|
||||
WherePK().
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete deletes a run by ID
|
||||
func (r *RunRepository) Delete(ctx context.Context, id string) error {
|
||||
_, err := r.db.NewDelete().
|
||||
Model((*database.Run)(nil)).
|
||||
Where("id = ?", id).
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// List lists runs with pagination
|
||||
func (r *RunRepository) List(ctx context.Context, page, perPage int) ([]*database.Run, int, error) {
|
||||
var scans []*database.Run
|
||||
|
||||
count, err := r.db.NewSelect().
|
||||
Model(&scans).
|
||||
Order("created_at DESC").
|
||||
Limit(perPage).
|
||||
Offset((page - 1) * perPage).
|
||||
ScanAndCount(ctx)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
return scans, count, nil
|
||||
}
|
||||
|
||||
// ListByStatus lists runs by status
|
||||
func (r *RunRepository) ListByStatus(ctx context.Context, status string) ([]*database.Run, error) {
|
||||
var scans []*database.Run
|
||||
err := r.db.NewSelect().
|
||||
Model(&scans).
|
||||
Where("status = ?", status).
|
||||
Order("created_at DESC").
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scans, nil
|
||||
}
|
||||
|
||||
// ListByWorkflow lists runs by workflow name
|
||||
func (r *RunRepository) ListByWorkflow(ctx context.Context, workflowName string) ([]*database.Run, error) {
|
||||
var scans []*database.Run
|
||||
err := r.db.NewSelect().
|
||||
Model(&scans).
|
||||
Where("workflow_name = ?", workflowName).
|
||||
Order("created_at DESC").
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scans, nil
|
||||
}
|
||||
|
||||
// ListByTarget lists runs by target
|
||||
func (r *RunRepository) ListByTarget(ctx context.Context, target string) ([]*database.Run, error) {
|
||||
var scans []*database.Run
|
||||
err := r.db.NewSelect().
|
||||
Model(&scans).
|
||||
Where("target = ?", target).
|
||||
Order("created_at DESC").
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scans, nil
|
||||
}
|
||||
|
||||
// GetWithSteps gets a run with its step results
|
||||
func (r *RunRepository) GetWithSteps(ctx context.Context, id string) (*database.Run, error) {
|
||||
scan := new(database.Run)
|
||||
err := r.db.NewSelect().
|
||||
Model(scan).
|
||||
Relation("Steps").
|
||||
Where("r.id = ?", id).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scan, nil
|
||||
}
|
||||
|
||||
// GetWithArtifacts gets a run with its artifacts
|
||||
func (r *RunRepository) GetWithArtifacts(ctx context.Context, id string) (*database.Run, error) {
|
||||
scan := new(database.Run)
|
||||
err := r.db.NewSelect().
|
||||
Model(scan).
|
||||
Relation("Artifacts").
|
||||
Where("r.id = ?", id).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scan, nil
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/database"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
// VulnerabilityRepository handles vulnerability database operations
|
||||
type VulnerabilityRepository struct {
|
||||
db *bun.DB
|
||||
}
|
||||
|
||||
// NewVulnerabilityRepository creates a new vulnerability repository
|
||||
func NewVulnerabilityRepository(db *bun.DB) *VulnerabilityRepository {
|
||||
return &VulnerabilityRepository{db: db}
|
||||
}
|
||||
|
||||
// VulnerabilityQuery represents query parameters for vulnerability search
|
||||
type VulnerabilityQuery struct {
|
||||
Workspace string
|
||||
Severity string
|
||||
Confidence string
|
||||
AssetType string
|
||||
AssetValue string
|
||||
VulnTitle string
|
||||
Page int
|
||||
PerPage int
|
||||
}
|
||||
|
||||
// Create creates a new vulnerability
|
||||
func (r *VulnerabilityRepository) Create(ctx context.Context, vuln *database.Vulnerability) error {
|
||||
_, err := r.db.NewInsert().Model(vuln).Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetByID retrieves a vulnerability by ID
|
||||
func (r *VulnerabilityRepository) GetByID(ctx context.Context, id int64) (*database.Vulnerability, error) {
|
||||
vuln := new(database.Vulnerability)
|
||||
err := r.db.NewSelect().
|
||||
Model(vuln).
|
||||
Where("id = ?", id).
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return vuln, nil
|
||||
}
|
||||
|
||||
// Update updates an existing vulnerability
|
||||
func (r *VulnerabilityRepository) Update(ctx context.Context, vuln *database.Vulnerability) error {
|
||||
_, err := r.db.NewUpdate().
|
||||
Model(vuln).
|
||||
WherePK().
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete deletes a vulnerability by ID
|
||||
func (r *VulnerabilityRepository) Delete(ctx context.Context, id int64) error {
|
||||
_, err := r.db.NewDelete().
|
||||
Model((*database.Vulnerability)(nil)).
|
||||
Where("id = ?", id).
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteByWorkspace deletes all vulnerabilities in a workspace
|
||||
func (r *VulnerabilityRepository) DeleteByWorkspace(ctx context.Context, workspace string) (int64, error) {
|
||||
res, err := r.db.NewDelete().
|
||||
Model((*database.Vulnerability)(nil)).
|
||||
Where("workspace = ?", workspace).
|
||||
Exec(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
// ListByWorkspace lists vulnerabilities for a workspace with pagination
|
||||
func (r *VulnerabilityRepository) ListByWorkspace(ctx context.Context, workspace string, page, perPage int) ([]*database.Vulnerability, int, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if perPage < 1 {
|
||||
perPage = 50
|
||||
}
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
var vulns []*database.Vulnerability
|
||||
count, err := r.db.NewSelect().
|
||||
Model(&vulns).
|
||||
Where("workspace = ?", workspace).
|
||||
Order("created_at DESC").
|
||||
Limit(perPage).
|
||||
Offset(offset).
|
||||
ScanAndCount(ctx)
|
||||
|
||||
return vulns, count, err
|
||||
}
|
||||
|
||||
// CountByWorkspace returns the count of vulnerabilities in a workspace
|
||||
func (r *VulnerabilityRepository) CountByWorkspace(ctx context.Context, workspace string) (int, error) {
|
||||
return r.db.NewSelect().
|
||||
Model((*database.Vulnerability)(nil)).
|
||||
Where("workspace = ?", workspace).
|
||||
Count(ctx)
|
||||
}
|
||||
|
||||
// Search searches vulnerabilities with multiple criteria
|
||||
func (r *VulnerabilityRepository) Search(ctx context.Context, query VulnerabilityQuery) ([]*database.Vulnerability, int, error) {
|
||||
if query.Page < 1 {
|
||||
query.Page = 1
|
||||
}
|
||||
if query.PerPage < 1 {
|
||||
query.PerPage = 50
|
||||
}
|
||||
offset := (query.Page - 1) * query.PerPage
|
||||
|
||||
var vulns []*database.Vulnerability
|
||||
q := r.db.NewSelect().Model(&vulns)
|
||||
|
||||
if query.Workspace != "" {
|
||||
q = q.Where("workspace = ?", query.Workspace)
|
||||
}
|
||||
if query.Severity != "" {
|
||||
q = q.Where("severity = ?", query.Severity)
|
||||
}
|
||||
if query.Confidence != "" {
|
||||
q = q.Where("confidence = ?", query.Confidence)
|
||||
}
|
||||
if query.AssetType != "" {
|
||||
q = q.Where("asset_type = ?", query.AssetType)
|
||||
}
|
||||
if query.AssetValue != "" {
|
||||
q = q.Where("asset_value LIKE ?", "%"+query.AssetValue+"%")
|
||||
}
|
||||
if query.VulnTitle != "" {
|
||||
q = q.Where("vuln_title LIKE ?", "%"+query.VulnTitle+"%")
|
||||
}
|
||||
|
||||
count, err := q.
|
||||
Order("created_at DESC").
|
||||
Limit(query.PerPage).
|
||||
Offset(offset).
|
||||
ScanAndCount(ctx)
|
||||
|
||||
return vulns, count, err
|
||||
}
|
||||
|
||||
// GetSeveritySummary returns a summary of vulnerabilities by severity for a workspace
|
||||
func (r *VulnerabilityRepository) GetSeveritySummary(ctx context.Context, workspace string) (map[string]int, error) {
|
||||
var results []struct {
|
||||
Severity string `bun:"severity"`
|
||||
Count int `bun:"count"`
|
||||
}
|
||||
|
||||
query := r.db.NewSelect().
|
||||
Model((*database.Vulnerability)(nil)).
|
||||
ColumnExpr("severity, COUNT(*) AS count").
|
||||
Group("severity")
|
||||
|
||||
if workspace != "" {
|
||||
query = query.Where("workspace = ?", workspace)
|
||||
}
|
||||
|
||||
err := query.Scan(ctx, &results)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
summary := make(map[string]int)
|
||||
for _, r := range results {
|
||||
summary[r.Severity] = r.Count
|
||||
}
|
||||
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
// ListBySeverity lists vulnerabilities with a specific severity
|
||||
func (r *VulnerabilityRepository) ListBySeverity(ctx context.Context, workspace string, severity string) ([]*database.Vulnerability, error) {
|
||||
var vulns []*database.Vulnerability
|
||||
query := r.db.NewSelect().Model(&vulns)
|
||||
|
||||
if workspace != "" {
|
||||
query = query.Where("workspace = ?", workspace)
|
||||
}
|
||||
|
||||
err := query.
|
||||
Where("severity = ?", severity).
|
||||
Order("created_at DESC").
|
||||
Scan(ctx)
|
||||
return vulns, err
|
||||
}
|
||||
|
||||
// ListByAssetValue lists vulnerabilities for a specific asset value
|
||||
func (r *VulnerabilityRepository) ListByAssetValue(ctx context.Context, assetValue string) ([]*database.Vulnerability, error) {
|
||||
var vulns []*database.Vulnerability
|
||||
err := r.db.NewSelect().
|
||||
Model(&vulns).
|
||||
Where("asset_value = ?", assetValue).
|
||||
Order("created_at DESC").
|
||||
Scan(ctx)
|
||||
return vulns, err
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,197 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/parser"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
// SystemStats contains aggregated system statistics
|
||||
type SystemStats struct {
|
||||
Workflows WorkflowStats `json:"workflows"`
|
||||
Runs RunStats `json:"runs"`
|
||||
Workspaces WorkspaceStats `json:"workspaces"`
|
||||
Assets AssetStats `json:"assets"`
|
||||
Vulnerabilities VulnerabilityStats `json:"vulnerabilities"`
|
||||
Schedules ScheduleStats `json:"schedules"`
|
||||
}
|
||||
|
||||
// WorkflowStats contains workflow counts
|
||||
type WorkflowStats struct {
|
||||
Total int `json:"total"`
|
||||
Flows int `json:"flows"`
|
||||
Modules int `json:"modules"`
|
||||
}
|
||||
|
||||
// RunStats contains run counts by status
|
||||
type RunStats struct {
|
||||
Total int `json:"total"`
|
||||
Completed int `json:"completed"`
|
||||
Running int `json:"running"`
|
||||
Failed int `json:"failed"`
|
||||
Pending int `json:"pending"`
|
||||
}
|
||||
|
||||
// WorkspaceStats contains workspace counts
|
||||
type WorkspaceStats struct {
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// AssetStats contains asset counts
|
||||
type AssetStats struct {
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// VulnerabilityStats contains vulnerability counts by severity
|
||||
type VulnerabilityStats struct {
|
||||
Total int `json:"total"`
|
||||
Critical int `json:"critical"`
|
||||
High int `json:"high"`
|
||||
Medium int `json:"medium"`
|
||||
Low int `json:"low"`
|
||||
}
|
||||
|
||||
// ScheduleStats contains schedule counts
|
||||
type ScheduleStats struct {
|
||||
Total int `json:"total"`
|
||||
Enabled int `json:"enabled"`
|
||||
}
|
||||
|
||||
// GetSystemStats retrieves aggregated system statistics from the database and workflows
|
||||
func GetSystemStats(ctx context.Context, workflowsPath string) (*SystemStats, error) {
|
||||
db := GetDB()
|
||||
stats := &SystemStats{}
|
||||
|
||||
// Get workflow stats from loader
|
||||
if workflowsPath != "" {
|
||||
loader := parser.NewLoader(workflowsPath)
|
||||
flows, modules, err := loader.ListAllWorkflows()
|
||||
if err == nil {
|
||||
stats.Workflows = WorkflowStats{
|
||||
Total: len(flows) + len(modules),
|
||||
Flows: len(flows),
|
||||
Modules: len(modules),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get run stats
|
||||
runStats, err := getRunStats(ctx, db)
|
||||
if err == nil {
|
||||
stats.Runs = runStats
|
||||
}
|
||||
|
||||
// Get workspace stats
|
||||
workspaceCount, err := db.NewSelect().Model((*Workspace)(nil)).Count(ctx)
|
||||
if err == nil {
|
||||
stats.Workspaces = WorkspaceStats{Total: workspaceCount}
|
||||
}
|
||||
|
||||
// Get asset stats
|
||||
assetCount, err := db.NewSelect().Model((*Asset)(nil)).Count(ctx)
|
||||
if err == nil {
|
||||
stats.Assets = AssetStats{Total: assetCount}
|
||||
}
|
||||
|
||||
// Get vulnerability stats (aggregated from workspaces)
|
||||
vulnStats, err := getVulnerabilityStats(ctx, db)
|
||||
if err == nil {
|
||||
stats.Vulnerabilities = vulnStats
|
||||
}
|
||||
|
||||
// Get schedule stats
|
||||
scheduleStats, err := getScheduleStats(ctx, db)
|
||||
if err == nil {
|
||||
stats.Schedules = scheduleStats
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// getRunStats retrieves run counts grouped by status in a single query
|
||||
func getRunStats(ctx context.Context, db *bun.DB) (RunStats, error) {
|
||||
var result struct {
|
||||
Total int `bun:"total"`
|
||||
Completed int `bun:"completed"`
|
||||
Running int `bun:"running"`
|
||||
Failed int `bun:"failed"`
|
||||
Pending int `bun:"pending"`
|
||||
}
|
||||
|
||||
err := db.NewSelect().
|
||||
Model((*Run)(nil)).
|
||||
ColumnExpr("COUNT(*) AS total").
|
||||
ColumnExpr("SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) AS completed").
|
||||
ColumnExpr("SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) AS running").
|
||||
ColumnExpr("SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed").
|
||||
ColumnExpr("SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending").
|
||||
Scan(ctx, &result)
|
||||
|
||||
if err != nil {
|
||||
return RunStats{}, err
|
||||
}
|
||||
|
||||
return RunStats{
|
||||
Total: result.Total,
|
||||
Completed: result.Completed,
|
||||
Running: result.Running,
|
||||
Failed: result.Failed,
|
||||
Pending: result.Pending,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getVulnerabilityStats retrieves aggregated vulnerability counts from workspaces
|
||||
func getVulnerabilityStats(ctx context.Context, db *bun.DB) (VulnerabilityStats, error) {
|
||||
stats := VulnerabilityStats{}
|
||||
|
||||
var result struct {
|
||||
Critical int `bun:"critical"`
|
||||
High int `bun:"high"`
|
||||
Medium int `bun:"medium"`
|
||||
Low int `bun:"low"`
|
||||
}
|
||||
|
||||
err := db.NewSelect().
|
||||
Model((*Workspace)(nil)).
|
||||
ColumnExpr("COALESCE(SUM(vuln_critical), 0) AS critical").
|
||||
ColumnExpr("COALESCE(SUM(vuln_high), 0) AS high").
|
||||
ColumnExpr("COALESCE(SUM(vuln_medium), 0) AS medium").
|
||||
ColumnExpr("COALESCE(SUM(vuln_low), 0) AS low").
|
||||
Scan(ctx, &result)
|
||||
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
|
||||
stats.Critical = result.Critical
|
||||
stats.High = result.High
|
||||
stats.Medium = result.Medium
|
||||
stats.Low = result.Low
|
||||
stats.Total = result.Critical + result.High + result.Medium + result.Low
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// getScheduleStats retrieves schedule counts in a single query
|
||||
func getScheduleStats(ctx context.Context, db *bun.DB) (ScheduleStats, error) {
|
||||
var result struct {
|
||||
Total int `bun:"total"`
|
||||
Enabled int `bun:"enabled"`
|
||||
}
|
||||
|
||||
err := db.NewSelect().
|
||||
Model((*Schedule)(nil)).
|
||||
ColumnExpr("COUNT(*) AS total").
|
||||
ColumnExpr("SUM(CASE WHEN is_enabled = true THEN 1 ELSE 0 END) AS enabled").
|
||||
Scan(ctx, &result)
|
||||
|
||||
if err != nil {
|
||||
return ScheduleStats{}, err
|
||||
}
|
||||
|
||||
return ScheduleStats{
|
||||
Total: result.Total,
|
||||
Enabled: result.Enabled,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/parser"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
// WorkflowQuery holds query parameters for listing workflows from DB
|
||||
type WorkflowQuery struct {
|
||||
Tags []string // Filter by tags (any match)
|
||||
Kind string // Filter by kind (module/flow)
|
||||
Search string // Search in name/description
|
||||
Offset int
|
||||
Limit int
|
||||
}
|
||||
|
||||
// WorkflowMetaResult holds paginated workflow metadata results
|
||||
type WorkflowMetaResult struct {
|
||||
Data []WorkflowMeta `json:"data"`
|
||||
TotalCount int `json:"total_count"`
|
||||
Offset int `json:"offset"`
|
||||
Limit int `json:"limit"`
|
||||
}
|
||||
|
||||
// IndexResult holds the result of a workflow indexing operation
|
||||
type IndexResult struct {
|
||||
Added int `json:"added"`
|
||||
Updated int `json:"updated"`
|
||||
Removed int `json:"removed"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// IndexWorkflowsFromFilesystem scans workflow directory and updates database
|
||||
func IndexWorkflowsFromFilesystem(ctx context.Context, workflowsPath string, force bool) (*IndexResult, error) {
|
||||
if db == nil {
|
||||
return nil, fmt.Errorf("database not connected")
|
||||
}
|
||||
|
||||
result := &IndexResult{}
|
||||
|
||||
// Load all workflows from filesystem
|
||||
loader := parser.NewLoader(workflowsPath)
|
||||
workflows, err := loader.LoadAllWorkflows()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load workflows: %w", err)
|
||||
}
|
||||
|
||||
// Track which workflows we've seen
|
||||
seenNames := make(map[string]bool)
|
||||
|
||||
// Process each workflow
|
||||
for _, w := range workflows {
|
||||
seenNames[w.Name] = true
|
||||
|
||||
// Check if workflow already exists
|
||||
var existing WorkflowMeta
|
||||
existsErr := db.NewSelect().Model(&existing).Where("name = ?", w.Name).Scan(ctx)
|
||||
existed := existsErr == nil
|
||||
|
||||
if err := upsertWorkflowMeta(ctx, w, force); err != nil {
|
||||
result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", w.Name, err))
|
||||
} else {
|
||||
if existed {
|
||||
result.Updated++
|
||||
} else {
|
||||
result.Added++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove workflows that no longer exist on filesystem
|
||||
var allMeta []WorkflowMeta
|
||||
if err := db.NewSelect().Model(&allMeta).Scan(ctx); err == nil {
|
||||
for _, meta := range allMeta {
|
||||
if !seenNames[meta.Name] {
|
||||
_, err := db.NewDelete().Model(&meta).Where("id = ?", meta.ID).Exec(ctx)
|
||||
if err == nil {
|
||||
result.Removed++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// upsertWorkflowMeta inserts or updates a workflow metadata record
|
||||
func upsertWorkflowMeta(ctx context.Context, w *core.Workflow, force bool) error {
|
||||
// Check if workflow already exists
|
||||
var existing WorkflowMeta
|
||||
err := db.NewSelect().Model(&existing).Where("name = ?", w.Name).Scan(ctx)
|
||||
|
||||
// If exists and checksum unchanged (unless force), skip
|
||||
if err == nil && !force && existing.Checksum == w.Checksum {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Serialize params to JSON
|
||||
paramsJSON := ""
|
||||
if w.Params != nil {
|
||||
if data, err := json.Marshal(w.Params); err == nil {
|
||||
paramsJSON = string(data)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
if err == nil {
|
||||
// Update existing
|
||||
existing.Kind = string(w.Kind)
|
||||
existing.Description = w.Description
|
||||
existing.FilePath = w.FilePath
|
||||
existing.Checksum = w.Checksum
|
||||
existing.Tags = w.Tags
|
||||
existing.StepCount = len(w.Steps)
|
||||
existing.ModuleCount = len(w.Modules)
|
||||
existing.ParamsJSON = paramsJSON
|
||||
existing.IndexedAt = now
|
||||
existing.UpdatedAt = now
|
||||
|
||||
_, err = db.NewUpdate().Model(&existing).WherePK().Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Insert new
|
||||
meta := &WorkflowMeta{
|
||||
Name: w.Name,
|
||||
Kind: string(w.Kind),
|
||||
Description: w.Description,
|
||||
FilePath: w.FilePath,
|
||||
Checksum: w.Checksum,
|
||||
Tags: w.Tags,
|
||||
StepCount: len(w.Steps),
|
||||
ModuleCount: len(w.Modules),
|
||||
ParamsJSON: paramsJSON,
|
||||
IndexedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
_, err = db.NewInsert().Model(meta).Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListWorkflowsFromDB returns paginated workflow metadata from database
|
||||
func ListWorkflowsFromDB(ctx context.Context, query WorkflowQuery) (*WorkflowMetaResult, error) {
|
||||
if db == nil {
|
||||
return nil, fmt.Errorf("database not connected")
|
||||
}
|
||||
|
||||
result := &WorkflowMetaResult{
|
||||
Offset: query.Offset,
|
||||
Limit: query.Limit,
|
||||
}
|
||||
|
||||
if result.Limit <= 0 {
|
||||
result.Limit = 20
|
||||
}
|
||||
if result.Limit > 10000 {
|
||||
result.Limit = 10000
|
||||
}
|
||||
|
||||
// Build base query
|
||||
baseQuery := db.NewSelect().Model((*WorkflowMeta)(nil))
|
||||
|
||||
// Apply filters
|
||||
if query.Kind != "" {
|
||||
baseQuery = baseQuery.Where("kind = ?", query.Kind)
|
||||
}
|
||||
|
||||
if query.Search != "" {
|
||||
searchPattern := "%" + query.Search + "%"
|
||||
baseQuery = baseQuery.Where("(name LIKE ? OR description LIKE ?)", searchPattern, searchPattern)
|
||||
}
|
||||
|
||||
// Tag filtering - check if any tag matches
|
||||
if len(query.Tags) > 0 {
|
||||
baseQuery = baseQuery.WhereGroup(" AND ", func(q *bun.SelectQuery) *bun.SelectQuery {
|
||||
for _, tag := range query.Tags {
|
||||
// For SQLite JSON, use json_each to search array
|
||||
if IsSQLite() {
|
||||
q = q.WhereOr("EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)", tag)
|
||||
} else {
|
||||
// For PostgreSQL, use @> operator
|
||||
q = q.WhereOr("tags @> ?", fmt.Sprintf(`["%s"]`, tag))
|
||||
}
|
||||
}
|
||||
return q
|
||||
})
|
||||
}
|
||||
|
||||
// Get total count with filters
|
||||
totalCount, err := baseQuery.Count(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to count workflows: %w", err)
|
||||
}
|
||||
result.TotalCount = totalCount
|
||||
|
||||
// Fetch records with pagination
|
||||
var workflows []WorkflowMeta
|
||||
err = db.NewSelect().
|
||||
Model(&workflows).
|
||||
Apply(func(q *bun.SelectQuery) *bun.SelectQuery {
|
||||
if query.Kind != "" {
|
||||
q = q.Where("kind = ?", query.Kind)
|
||||
}
|
||||
if query.Search != "" {
|
||||
searchPattern := "%" + query.Search + "%"
|
||||
q = q.Where("(name LIKE ? OR description LIKE ?)", searchPattern, searchPattern)
|
||||
}
|
||||
if len(query.Tags) > 0 {
|
||||
q = q.WhereGroup(" AND ", func(sq *bun.SelectQuery) *bun.SelectQuery {
|
||||
for _, tag := range query.Tags {
|
||||
if IsSQLite() {
|
||||
sq = sq.WhereOr("EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)", tag)
|
||||
} else {
|
||||
sq = sq.WhereOr("tags @> ?", fmt.Sprintf(`["%s"]`, tag))
|
||||
}
|
||||
}
|
||||
return sq
|
||||
})
|
||||
}
|
||||
return q
|
||||
}).
|
||||
Order("name ASC").
|
||||
Offset(result.Offset).
|
||||
Limit(result.Limit).
|
||||
Scan(ctx)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch workflows: %w", err)
|
||||
}
|
||||
|
||||
result.Data = workflows
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetWorkflowFromDB returns a single workflow metadata by name
|
||||
func GetWorkflowFromDB(ctx context.Context, name string) (*WorkflowMeta, error) {
|
||||
if db == nil {
|
||||
return nil, fmt.Errorf("database not connected")
|
||||
}
|
||||
|
||||
var meta WorkflowMeta
|
||||
err := db.NewSelect().Model(&meta).Where("name = ?", name).Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &meta, nil
|
||||
}
|
||||
|
||||
// GetAllTags returns all unique tags from workflows
|
||||
func GetAllTags(ctx context.Context) ([]string, error) {
|
||||
if db == nil {
|
||||
return nil, fmt.Errorf("database not connected")
|
||||
}
|
||||
|
||||
var workflows []WorkflowMeta
|
||||
if err := db.NewSelect().Model(&workflows).Column("tags").Scan(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Collect unique tags
|
||||
tagMap := make(map[string]bool)
|
||||
for _, w := range workflows {
|
||||
for _, tag := range w.Tags {
|
||||
tagMap[strings.TrimSpace(tag)] = true
|
||||
}
|
||||
}
|
||||
|
||||
tags := make([]string, 0, len(tagMap))
|
||||
for tag := range tagMap {
|
||||
if tag != "" {
|
||||
tags = append(tags, tag)
|
||||
}
|
||||
}
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// GetWorkflowCount returns the total number of indexed workflows
|
||||
func GetWorkflowCount(ctx context.Context) (int, error) {
|
||||
if db == nil {
|
||||
return 0, fmt.Errorf("database not connected")
|
||||
}
|
||||
|
||||
return db.NewSelect().Model((*WorkflowMeta)(nil)).Count(ctx)
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
func EnsureWorkspaceRuntime(ctx context.Context, name, localPath, runWorkflow, stateExecutionLog, stateCompletedFile, stateWorkflowFile, stateWorkflowFolder string) error {
|
||||
if db == nil {
|
||||
return fmt.Errorf("database not connected")
|
||||
}
|
||||
|
||||
if name == "" {
|
||||
return fmt.Errorf("workspace name cannot be empty")
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
ws := &Workspace{
|
||||
Name: name,
|
||||
LocalPath: localPath,
|
||||
DataSource: "local",
|
||||
LastRun: &now,
|
||||
RunWorkflow: runWorkflow,
|
||||
StateExecutionLog: stateExecutionLog,
|
||||
StateCompletedFile: stateCompletedFile,
|
||||
StateWorkflowFile: stateWorkflowFile,
|
||||
StateWorkflowFolder: stateWorkflowFolder,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
_, err := db.NewInsert().Model(ws).
|
||||
On("CONFLICT (name) DO UPDATE").
|
||||
Set("local_path = EXCLUDED.local_path").
|
||||
Set("data_source = EXCLUDED.data_source").
|
||||
Set("last_run = EXCLUDED.last_run").
|
||||
Set("run_workflow = EXCLUDED.run_workflow").
|
||||
Set("state_execution_log = EXCLUDED.state_execution_log").
|
||||
Set("state_completed_file = EXCLUDED.state_completed_file").
|
||||
Set("state_workflow_file = EXCLUDED.state_workflow_file").
|
||||
Set("state_workflow_folder = EXCLUDED.state_workflow_folder").
|
||||
Set("updated_at = EXCLUDED.updated_at").
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
package distributed
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/config"
|
||||
"github.com/redis/rueidis"
|
||||
)
|
||||
|
||||
// Redis key prefixes
|
||||
const (
|
||||
KeyPrefix = "osm:"
|
||||
KeyTasksPending = KeyPrefix + "tasks:pending"
|
||||
KeyTasksRunning = KeyPrefix + "tasks:running"
|
||||
KeyTasksCompleted = KeyPrefix + "tasks:completed"
|
||||
KeyWorkers = KeyPrefix + "workers"
|
||||
KeyWorkersHeartbeat = KeyPrefix + "workers:heartbeat"
|
||||
KeyMasterLock = KeyPrefix + "master:lock"
|
||||
)
|
||||
|
||||
// Timeouts and intervals
|
||||
const (
|
||||
HeartbeatInterval = 30 * time.Second
|
||||
HeartbeatTimeout = 90 * time.Second // 3 missed heartbeats
|
||||
TaskPollTimeout = 5 * time.Second
|
||||
DefaultConnectTimeout = 60 * time.Second
|
||||
)
|
||||
|
||||
// Client wraps a rueidis client with helper methods
|
||||
type Client struct {
|
||||
client rueidis.Client
|
||||
cfg *config.RedisConfig
|
||||
}
|
||||
|
||||
// NewClient creates a new Redis client from configuration
|
||||
func NewClient(cfg *config.RedisConfig) (*Client, error) {
|
||||
if cfg.Host == "" {
|
||||
return nil, fmt.Errorf("redis host not configured")
|
||||
}
|
||||
|
||||
port := cfg.Port
|
||||
if port == 0 {
|
||||
port = 6379
|
||||
}
|
||||
|
||||
opts := rueidis.ClientOption{
|
||||
InitAddress: []string{fmt.Sprintf("%s:%d", cfg.Host, port)},
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
SelectDB: cfg.DB,
|
||||
DisableCache: true, // Disable client-side caching for simpler behavior
|
||||
}
|
||||
|
||||
client, err := rueidis.NewClient(opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create redis client: %w", err)
|
||||
}
|
||||
|
||||
return &Client{
|
||||
client: client,
|
||||
cfg: cfg,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewClientFromConfig creates a client from the global config
|
||||
func NewClientFromConfig(cfg *config.Config) (*Client, error) {
|
||||
return NewClient(&cfg.Redis)
|
||||
}
|
||||
|
||||
// ParseRedisURL parses a Redis connection URL into RedisConfig
|
||||
// Format: redis://[username:password@]host:port[/db]
|
||||
func ParseRedisURL(redisURL string) (*config.RedisConfig, error) {
|
||||
if !strings.HasPrefix(redisURL, "redis://") {
|
||||
redisURL = "redis://" + redisURL
|
||||
}
|
||||
|
||||
u, err := url.Parse(redisURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid redis URL: %w", err)
|
||||
}
|
||||
|
||||
cfg := &config.RedisConfig{
|
||||
Host: u.Hostname(),
|
||||
Port: 6379,
|
||||
ConnectionTimeout: 60,
|
||||
}
|
||||
|
||||
if u.Port() != "" {
|
||||
port, err := strconv.Atoi(u.Port())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid redis port: %w", err)
|
||||
}
|
||||
cfg.Port = port
|
||||
}
|
||||
|
||||
if u.User != nil {
|
||||
cfg.Username = u.User.Username()
|
||||
cfg.Password, _ = u.User.Password()
|
||||
}
|
||||
|
||||
if u.Path != "" && u.Path != "/" {
|
||||
db, err := strconv.Atoi(strings.TrimPrefix(u.Path, "/"))
|
||||
if err == nil {
|
||||
cfg.DB = db
|
||||
}
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// Close closes the Redis client
|
||||
func (c *Client) Close() {
|
||||
c.client.Close()
|
||||
}
|
||||
|
||||
// Ping tests the Redis connection
|
||||
func (c *Client) Ping(ctx context.Context) error {
|
||||
cmd := c.client.B().Ping().Build()
|
||||
return c.client.Do(ctx, cmd).Error()
|
||||
}
|
||||
|
||||
// Raw returns the underlying rueidis client
|
||||
func (c *Client) Raw() rueidis.Client {
|
||||
return c.client
|
||||
}
|
||||
|
||||
// PushTask pushes a task to the pending queue
|
||||
func (c *Client) PushTask(ctx context.Context, task *Task) error {
|
||||
data, err := task.MarshalJSON()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal task: %w", err)
|
||||
}
|
||||
|
||||
cmd := c.client.B().Lpush().Key(KeyTasksPending).Element(string(data)).Build()
|
||||
return c.client.Do(ctx, cmd).Error()
|
||||
}
|
||||
|
||||
// PopTask pops a task from the pending queue (blocking)
|
||||
func (c *Client) PopTask(ctx context.Context, timeout time.Duration) (*Task, error) {
|
||||
cmd := c.client.B().Brpop().Key(KeyTasksPending).Timeout(timeout.Seconds()).Build()
|
||||
result, err := c.client.Do(ctx, cmd).AsStrSlice()
|
||||
if err != nil {
|
||||
if rueidis.IsRedisNil(err) {
|
||||
return nil, nil // Timeout, no task available
|
||||
}
|
||||
return nil, fmt.Errorf("failed to pop task: %w", err)
|
||||
}
|
||||
|
||||
if len(result) < 2 {
|
||||
return nil, nil // No task
|
||||
}
|
||||
|
||||
return UnmarshalTask([]byte(result[1]))
|
||||
}
|
||||
|
||||
// SetTaskRunning moves a task to the running hash
|
||||
func (c *Client) SetTaskRunning(ctx context.Context, task *Task) error {
|
||||
data, err := task.MarshalJSON()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal task: %w", err)
|
||||
}
|
||||
|
||||
cmd := c.client.B().Hset().Key(KeyTasksRunning).FieldValue().FieldValue(task.ID, string(data)).Build()
|
||||
return c.client.Do(ctx, cmd).Error()
|
||||
}
|
||||
|
||||
// RemoveTaskRunning removes a task from the running hash
|
||||
func (c *Client) RemoveTaskRunning(ctx context.Context, taskID string) error {
|
||||
cmd := c.client.B().Hdel().Key(KeyTasksRunning).Field(taskID).Build()
|
||||
return c.client.Do(ctx, cmd).Error()
|
||||
}
|
||||
|
||||
// SetTaskResult stores a task result in the completed hash
|
||||
func (c *Client) SetTaskResult(ctx context.Context, result *TaskResult) error {
|
||||
data, err := result.MarshalJSON()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal result: %w", err)
|
||||
}
|
||||
|
||||
cmd := c.client.B().Hset().Key(KeyTasksCompleted).FieldValue().FieldValue(result.TaskID, string(data)).Build()
|
||||
return c.client.Do(ctx, cmd).Error()
|
||||
}
|
||||
|
||||
// GetTaskResult retrieves a task result from the completed hash
|
||||
func (c *Client) GetTaskResult(ctx context.Context, taskID string) (*TaskResult, error) {
|
||||
cmd := c.client.B().Hget().Key(KeyTasksCompleted).Field(taskID).Build()
|
||||
data, err := c.client.Do(ctx, cmd).ToString()
|
||||
if err != nil {
|
||||
if rueidis.IsRedisNil(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("failed to get task result: %w", err)
|
||||
}
|
||||
|
||||
return UnmarshalTaskResult([]byte(data))
|
||||
}
|
||||
|
||||
// GetRunningTask retrieves a running task by ID
|
||||
func (c *Client) GetRunningTask(ctx context.Context, taskID string) (*Task, error) {
|
||||
cmd := c.client.B().Hget().Key(KeyTasksRunning).Field(taskID).Build()
|
||||
data, err := c.client.Do(ctx, cmd).ToString()
|
||||
if err != nil {
|
||||
if rueidis.IsRedisNil(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("failed to get running task: %w", err)
|
||||
}
|
||||
|
||||
return UnmarshalTask([]byte(data))
|
||||
}
|
||||
|
||||
// GetAllRunningTasks retrieves all running tasks
|
||||
func (c *Client) GetAllRunningTasks(ctx context.Context) ([]*Task, error) {
|
||||
cmd := c.client.B().Hgetall().Key(KeyTasksRunning).Build()
|
||||
result, err := c.client.Do(ctx, cmd).AsStrMap()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get running tasks: %w", err)
|
||||
}
|
||||
|
||||
var tasks []*Task
|
||||
for _, data := range result {
|
||||
task, err := UnmarshalTask([]byte(data))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
tasks = append(tasks, task)
|
||||
}
|
||||
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
// RegisterWorker registers a worker in the workers hash
|
||||
func (c *Client) RegisterWorker(ctx context.Context, worker *WorkerInfo) error {
|
||||
data, err := worker.MarshalJSON()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal worker: %w", err)
|
||||
}
|
||||
|
||||
cmd := c.client.B().Hset().Key(KeyWorkers).FieldValue().FieldValue(worker.ID, string(data)).Build()
|
||||
return c.client.Do(ctx, cmd).Error()
|
||||
}
|
||||
|
||||
// UpdateWorkerHeartbeat updates a worker's heartbeat timestamp
|
||||
func (c *Client) UpdateWorkerHeartbeat(ctx context.Context, workerID string) error {
|
||||
timestamp := strconv.FormatInt(time.Now().Unix(), 10)
|
||||
cmd := c.client.B().Hset().Key(KeyWorkersHeartbeat).FieldValue().FieldValue(workerID, timestamp).Build()
|
||||
return c.client.Do(ctx, cmd).Error()
|
||||
}
|
||||
|
||||
// GetWorkerHeartbeat gets a worker's last heartbeat timestamp
|
||||
func (c *Client) GetWorkerHeartbeat(ctx context.Context, workerID string) (time.Time, error) {
|
||||
cmd := c.client.B().Hget().Key(KeyWorkersHeartbeat).Field(workerID).Build()
|
||||
data, err := c.client.Do(ctx, cmd).ToString()
|
||||
if err != nil {
|
||||
if rueidis.IsRedisNil(err) {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
ts, err := strconv.ParseInt(data, 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
return time.Unix(ts, 0), nil
|
||||
}
|
||||
|
||||
// GetAllWorkers retrieves all registered workers
|
||||
func (c *Client) GetAllWorkers(ctx context.Context) ([]*WorkerInfo, error) {
|
||||
cmd := c.client.B().Hgetall().Key(KeyWorkers).Build()
|
||||
result, err := c.client.Do(ctx, cmd).AsStrMap()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get workers: %w", err)
|
||||
}
|
||||
|
||||
var workers []*WorkerInfo
|
||||
for _, data := range result {
|
||||
worker, err := UnmarshalWorkerInfo([]byte(data))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
workers = append(workers, worker)
|
||||
}
|
||||
|
||||
return workers, nil
|
||||
}
|
||||
|
||||
// RemoveWorker removes a worker from the registry
|
||||
func (c *Client) RemoveWorker(ctx context.Context, workerID string) error {
|
||||
// Remove from both workers and heartbeat hashes
|
||||
cmd1 := c.client.B().Hdel().Key(KeyWorkers).Field(workerID).Build()
|
||||
cmd2 := c.client.B().Hdel().Key(KeyWorkersHeartbeat).Field(workerID).Build()
|
||||
|
||||
if err := c.client.Do(ctx, cmd1).Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.client.Do(ctx, cmd2).Error()
|
||||
}
|
||||
|
||||
// AcquireMasterLock tries to acquire the master lock
|
||||
func (c *Client) AcquireMasterLock(ctx context.Context, masterID string, ttl time.Duration) (bool, error) {
|
||||
cmd := c.client.B().Set().Key(KeyMasterLock).Value(masterID).Nx().Ex(ttl).Build()
|
||||
result, err := c.client.Do(ctx, cmd).ToString()
|
||||
if err != nil {
|
||||
if rueidis.IsRedisNil(err) {
|
||||
return false, nil // Lock not acquired
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return result == "OK", nil
|
||||
}
|
||||
|
||||
// RefreshMasterLock refreshes the master lock TTL
|
||||
func (c *Client) RefreshMasterLock(ctx context.Context, masterID string, ttl time.Duration) error {
|
||||
// Only refresh if we still own the lock
|
||||
cmd := c.client.B().Get().Key(KeyMasterLock).Build()
|
||||
current, err := c.client.Do(ctx, cmd).ToString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current != masterID {
|
||||
return fmt.Errorf("master lock lost")
|
||||
}
|
||||
|
||||
expireCmd := c.client.B().Expire().Key(KeyMasterLock).Seconds(int64(ttl.Seconds())).Build()
|
||||
return c.client.Do(ctx, expireCmd).Error()
|
||||
}
|
||||
|
||||
// ReleaseMasterLock releases the master lock
|
||||
func (c *Client) ReleaseMasterLock(ctx context.Context, masterID string) error {
|
||||
// Only release if we own the lock
|
||||
cmd := c.client.B().Get().Key(KeyMasterLock).Build()
|
||||
current, err := c.client.Do(ctx, cmd).ToString()
|
||||
if err != nil {
|
||||
if rueidis.IsRedisNil(err) {
|
||||
return nil // Already released
|
||||
}
|
||||
return err
|
||||
}
|
||||
if current != masterID {
|
||||
return nil // Not our lock
|
||||
}
|
||||
|
||||
delCmd := c.client.B().Del().Key(KeyMasterLock).Build()
|
||||
return c.client.Do(ctx, delCmd).Error()
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package distributed
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/config"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/terminal"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
MasterLockTTL = 60 * time.Second
|
||||
MasterLockRefresh = 30 * time.Second
|
||||
WorkerCheckPeriod = 30 * time.Second
|
||||
)
|
||||
|
||||
// Master represents a master node that coordinates workers
|
||||
type Master struct {
|
||||
ID string
|
||||
client *Client
|
||||
config *config.Config
|
||||
logger *zap.Logger
|
||||
printer *terminal.Printer
|
||||
|
||||
// For tracking
|
||||
mu sync.RWMutex
|
||||
running bool
|
||||
}
|
||||
|
||||
// NewMaster creates a new master node
|
||||
func NewMaster(cfg *config.Config) (*Master, error) {
|
||||
client, err := NewClientFromConfig(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create redis client: %w", err)
|
||||
}
|
||||
|
||||
hostname, _ := os.Hostname()
|
||||
masterID := fmt.Sprintf("master-%s-%s", hostname, uuid.NewString()[:8])
|
||||
|
||||
logger, _ := zap.NewProduction()
|
||||
|
||||
return &Master{
|
||||
ID: masterID,
|
||||
client: client,
|
||||
config: cfg,
|
||||
logger: logger,
|
||||
printer: terminal.NewPrinter(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Start starts the master node
|
||||
func (m *Master) Start(ctx context.Context) error {
|
||||
// Test connection
|
||||
if err := m.client.Ping(ctx); err != nil {
|
||||
return fmt.Errorf("failed to connect to redis: %w", err)
|
||||
}
|
||||
|
||||
// Acquire master lock
|
||||
acquired, err := m.client.AcquireMasterLock(ctx, m.ID, MasterLockTTL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to acquire master lock: %w", err)
|
||||
}
|
||||
if !acquired {
|
||||
return fmt.Errorf("another master is already running")
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
m.running = true
|
||||
m.mu.Unlock()
|
||||
|
||||
m.printer.Success("Master %s started", m.ID)
|
||||
m.printer.Info("Waiting for workers and tasks...")
|
||||
|
||||
// Start lock refresh goroutine
|
||||
lockCtx, cancelLock := context.WithCancel(ctx)
|
||||
defer cancelLock()
|
||||
go m.lockRefreshLoop(lockCtx)
|
||||
|
||||
// Start worker monitor goroutine
|
||||
monitorCtx, cancelMonitor := context.WithCancel(ctx)
|
||||
defer cancelMonitor()
|
||||
go m.workerMonitorLoop(monitorCtx)
|
||||
|
||||
// Wait for shutdown
|
||||
<-ctx.Done()
|
||||
|
||||
m.logger.Info("master shutting down", zap.String("master_id", m.ID))
|
||||
m.cleanup(context.Background())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// lockRefreshLoop periodically refreshes the master lock
|
||||
func (m *Master) lockRefreshLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(MasterLockRefresh)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := m.client.RefreshMasterLock(ctx, m.ID, MasterLockTTL); err != nil {
|
||||
m.logger.Error("failed to refresh master lock", zap.Error(err))
|
||||
// If we lose the lock, we should stop
|
||||
m.mu.Lock()
|
||||
m.running = false
|
||||
m.mu.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// workerMonitorLoop monitors worker heartbeats and handles failures
|
||||
func (m *Master) workerMonitorLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(WorkerCheckPeriod)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.checkWorkerHealth(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkWorkerHealth checks for dead workers and reassigns their tasks
|
||||
func (m *Master) checkWorkerHealth(ctx context.Context) {
|
||||
workers, err := m.client.GetAllWorkers(ctx)
|
||||
if err != nil {
|
||||
m.logger.Warn("failed to get workers", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for _, worker := range workers {
|
||||
heartbeat, err := m.client.GetWorkerHeartbeat(ctx, worker.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if worker is dead (missed heartbeats)
|
||||
if heartbeat.IsZero() || now.Sub(heartbeat) > HeartbeatTimeout {
|
||||
m.logger.Warn("worker appears dead",
|
||||
zap.String("worker_id", worker.ID),
|
||||
zap.Duration("since_heartbeat", now.Sub(heartbeat)),
|
||||
)
|
||||
m.printer.Warning("Worker %s appears dead, reassigning tasks...", worker.ID)
|
||||
|
||||
// Reassign the worker's running tasks
|
||||
m.reassignWorkerTasks(ctx, worker.ID)
|
||||
|
||||
// Remove the dead worker
|
||||
if err := m.client.RemoveWorker(ctx, worker.ID); err != nil {
|
||||
m.logger.Error("failed to remove dead worker", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reassignWorkerTasks moves a dead worker's tasks back to pending
|
||||
func (m *Master) reassignWorkerTasks(ctx context.Context, workerID string) {
|
||||
tasks, err := m.client.GetAllRunningTasks(ctx)
|
||||
if err != nil {
|
||||
m.logger.Error("failed to get running tasks", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
for _, task := range tasks {
|
||||
if task.WorkerID == workerID {
|
||||
m.logger.Info("reassigning task",
|
||||
zap.String("task_id", task.ID),
|
||||
zap.String("worker_id", workerID),
|
||||
)
|
||||
|
||||
// Reset task status
|
||||
task.Status = TaskStatusPending
|
||||
task.WorkerID = ""
|
||||
task.StartedAt = nil
|
||||
|
||||
// Push back to pending queue
|
||||
if err := m.client.PushTask(ctx, task); err != nil {
|
||||
m.logger.Error("failed to reassign task", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
// Remove from running
|
||||
if err := m.client.RemoveTaskRunning(ctx, task.ID); err != nil {
|
||||
m.logger.Error("failed to remove task from running", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanup releases the master lock
|
||||
func (m *Master) cleanup(ctx context.Context) {
|
||||
m.printer.Info("Cleaning up master %s...", m.ID)
|
||||
|
||||
m.mu.Lock()
|
||||
m.running = false
|
||||
m.mu.Unlock()
|
||||
|
||||
if err := m.client.ReleaseMasterLock(ctx, m.ID); err != nil {
|
||||
m.logger.Warn("failed to release master lock", zap.Error(err))
|
||||
}
|
||||
|
||||
m.client.Close()
|
||||
}
|
||||
|
||||
// SubmitTask submits a new task to the pending queue
|
||||
func (m *Master) SubmitTask(ctx context.Context, task *Task) error {
|
||||
if task.ID == "" {
|
||||
task.ID = uuid.NewString()[:8]
|
||||
}
|
||||
if task.CreatedAt.IsZero() {
|
||||
task.CreatedAt = time.Now()
|
||||
}
|
||||
task.Status = TaskStatusPending
|
||||
|
||||
m.logger.Info("submitting task",
|
||||
zap.String("task_id", task.ID),
|
||||
zap.String("workflow", task.WorkflowName),
|
||||
zap.String("target", task.Target),
|
||||
)
|
||||
|
||||
return m.client.PushTask(ctx, task)
|
||||
}
|
||||
|
||||
// GetTaskStatus retrieves the status of a task
|
||||
func (m *Master) GetTaskStatus(ctx context.Context, taskID string) (*Task, *TaskResult, error) {
|
||||
// Check running tasks first
|
||||
task, err := m.client.GetRunningTask(ctx, taskID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if task != nil {
|
||||
return task, nil, nil
|
||||
}
|
||||
|
||||
// Check completed tasks
|
||||
result, err := m.client.GetTaskResult(ctx, taskID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if result != nil {
|
||||
return nil, result, nil
|
||||
}
|
||||
|
||||
return nil, nil, fmt.Errorf("task not found: %s", taskID)
|
||||
}
|
||||
|
||||
// ListWorkers returns all registered workers with their current status
|
||||
func (m *Master) ListWorkers(ctx context.Context) ([]*WorkerInfo, error) {
|
||||
workers, err := m.client.GetAllWorkers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Enrich with heartbeat info
|
||||
now := time.Now()
|
||||
for _, worker := range workers {
|
||||
heartbeat, err := m.client.GetWorkerHeartbeat(ctx, worker.ID)
|
||||
if err == nil {
|
||||
worker.LastHeartbeat = heartbeat
|
||||
|
||||
// Update status based on heartbeat
|
||||
if now.Sub(heartbeat) > HeartbeatTimeout {
|
||||
worker.Status = "offline"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return workers, nil
|
||||
}
|
||||
|
||||
// ListTasks returns all tasks (running and completed)
|
||||
func (m *Master) ListTasks(ctx context.Context) ([]*Task, []*TaskResult, error) {
|
||||
running, err := m.client.GetAllRunningTasks(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Get completed tasks (we'd need to iterate the hash)
|
||||
// For now, return running tasks
|
||||
return running, nil, nil
|
||||
}
|
||||
|
||||
// GetClient returns the Redis client for external use
|
||||
func (m *Master) GetClient() *Client {
|
||||
return m.client
|
||||
}
|
||||
|
||||
// IsRunning returns whether the master is currently running
|
||||
func (m *Master) IsRunning() bool {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.running
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package distributed
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TaskStatus represents the status of a distributed task
|
||||
type TaskStatus string
|
||||
|
||||
const (
|
||||
TaskStatusPending TaskStatus = "pending"
|
||||
TaskStatusRunning TaskStatus = "running"
|
||||
TaskStatusCompleted TaskStatus = "completed"
|
||||
TaskStatusFailed TaskStatus = "failed"
|
||||
)
|
||||
|
||||
// Task represents a distributed scan task
|
||||
type Task struct {
|
||||
ID string `json:"id"`
|
||||
ScanID string `json:"scan_id,omitempty"`
|
||||
WorkflowName string `json:"workflow_name"`
|
||||
WorkflowKind string `json:"workflow_kind"` // "module" or "flow"
|
||||
Target string `json:"target"`
|
||||
Params map[string]interface{} `json:"params,omitempty"`
|
||||
Status TaskStatus `json:"status"`
|
||||
WorkerID string `json:"worker_id,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// TaskResult represents the result of a completed task
|
||||
type TaskResult struct {
|
||||
TaskID string `json:"task_id"`
|
||||
Status TaskStatus `json:"status"`
|
||||
Output string `json:"output,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Exports map[string]interface{} `json:"exports,omitempty"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
}
|
||||
|
||||
// WorkerInfo represents information about a worker node
|
||||
type WorkerInfo struct {
|
||||
ID string `json:"id"`
|
||||
Hostname string `json:"hostname"`
|
||||
Status string `json:"status"` // "idle", "busy", "offline"
|
||||
CurrentTaskID string `json:"current_task_id,omitempty"`
|
||||
JoinedAt time.Time `json:"joined_at"`
|
||||
LastHeartbeat time.Time `json:"last_heartbeat"`
|
||||
TasksComplete int `json:"tasks_complete"`
|
||||
TasksFailed int `json:"tasks_failed"`
|
||||
}
|
||||
|
||||
// NewTask creates a new task with the given parameters
|
||||
func NewTask(id, workflowName, workflowKind, target string, params map[string]interface{}) *Task {
|
||||
return &Task{
|
||||
ID: id,
|
||||
WorkflowName: workflowName,
|
||||
WorkflowKind: workflowKind,
|
||||
Target: target,
|
||||
Params: params,
|
||||
Status: TaskStatusPending,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// MarshalJSON serializes a task to JSON
|
||||
func (t *Task) MarshalJSON() ([]byte, error) {
|
||||
type Alias Task
|
||||
return json.Marshal(&struct {
|
||||
*Alias
|
||||
}{
|
||||
Alias: (*Alias)(t),
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalTask deserializes a task from JSON
|
||||
func UnmarshalTask(data []byte) (*Task, error) {
|
||||
var task Task
|
||||
if err := json.Unmarshal(data, &task); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &task, nil
|
||||
}
|
||||
|
||||
// MarshalJSON serializes a task result to JSON
|
||||
func (r *TaskResult) MarshalJSON() ([]byte, error) {
|
||||
type Alias TaskResult
|
||||
return json.Marshal(&struct {
|
||||
*Alias
|
||||
}{
|
||||
Alias: (*Alias)(r),
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalTaskResult deserializes a task result from JSON
|
||||
func UnmarshalTaskResult(data []byte) (*TaskResult, error) {
|
||||
var result TaskResult
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// MarshalJSON serializes worker info to JSON
|
||||
func (w *WorkerInfo) MarshalJSON() ([]byte, error) {
|
||||
type Alias WorkerInfo
|
||||
return json.Marshal(&struct {
|
||||
*Alias
|
||||
}{
|
||||
Alias: (*Alias)(w),
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalWorkerInfo deserializes worker info from JSON
|
||||
func UnmarshalWorkerInfo(data []byte) (*WorkerInfo, error) {
|
||||
var info WorkerInfo
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &info, nil
|
||||
}
|
||||
|
||||
// MarkRunning marks the task as running with the given worker
|
||||
func (t *Task) MarkRunning(workerID string) {
|
||||
t.Status = TaskStatusRunning
|
||||
t.WorkerID = workerID
|
||||
now := time.Now()
|
||||
t.StartedAt = &now
|
||||
}
|
||||
|
||||
// MarkCompleted marks the task as completed
|
||||
func (t *Task) MarkCompleted() {
|
||||
t.Status = TaskStatusCompleted
|
||||
now := time.Now()
|
||||
t.CompletedAt = &now
|
||||
}
|
||||
|
||||
// MarkFailed marks the task as failed with an error message
|
||||
func (t *Task) MarkFailed(err string) {
|
||||
t.Status = TaskStatusFailed
|
||||
t.Error = err
|
||||
now := time.Now()
|
||||
t.CompletedAt = &now
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
package distributed
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/config"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/executor"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/parser"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/terminal"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Worker represents a worker node that processes tasks
|
||||
type Worker struct {
|
||||
ID string
|
||||
Hostname string
|
||||
client *Client
|
||||
config *config.Config
|
||||
executor *executor.Executor
|
||||
loader *parser.Loader
|
||||
logger *zap.Logger
|
||||
printer *terminal.Printer
|
||||
|
||||
// Stats
|
||||
tasksComplete int
|
||||
tasksFailed int
|
||||
}
|
||||
|
||||
// NewWorker creates a new worker node
|
||||
func NewWorker(cfg *config.Config) (*Worker, error) {
|
||||
client, err := NewClientFromConfig(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create redis client: %w", err)
|
||||
}
|
||||
|
||||
hostname, _ := os.Hostname()
|
||||
workerID := fmt.Sprintf("%s-%s", hostname, uuid.NewString()[:8])
|
||||
|
||||
logger, _ := zap.NewProduction()
|
||||
exec := executor.NewExecutor()
|
||||
|
||||
return &Worker{
|
||||
ID: workerID,
|
||||
Hostname: hostname,
|
||||
client: client,
|
||||
config: cfg,
|
||||
executor: exec,
|
||||
loader: parser.NewLoader(cfg.WorkflowsPath),
|
||||
logger: logger,
|
||||
printer: terminal.NewPrinter(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run starts the worker loop
|
||||
func (w *Worker) Run(ctx context.Context) error {
|
||||
// Test connection
|
||||
if err := w.client.Ping(ctx); err != nil {
|
||||
return fmt.Errorf("failed to connect to redis: %w", err)
|
||||
}
|
||||
|
||||
// Register worker
|
||||
if err := w.register(ctx); err != nil {
|
||||
return fmt.Errorf("failed to register worker: %w", err)
|
||||
}
|
||||
|
||||
w.printer.Success("Worker %s joined successfully", w.ID)
|
||||
w.printer.Info("Waiting for tasks...")
|
||||
|
||||
// Start heartbeat goroutine
|
||||
heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx)
|
||||
defer cancelHeartbeat()
|
||||
go w.heartbeatLoop(heartbeatCtx)
|
||||
|
||||
// Main task loop
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.logger.Info("worker shutting down", zap.String("worker_id", w.ID))
|
||||
w.cleanup(context.Background())
|
||||
return nil
|
||||
default:
|
||||
if err := w.processNextTask(ctx); err != nil {
|
||||
w.logger.Error("error processing task", zap.Error(err))
|
||||
time.Sleep(time.Second) // Brief pause before retrying
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// register registers the worker with the master
|
||||
func (w *Worker) register(ctx context.Context) error {
|
||||
info := &WorkerInfo{
|
||||
ID: w.ID,
|
||||
Hostname: w.Hostname,
|
||||
Status: "idle",
|
||||
JoinedAt: time.Now(),
|
||||
LastHeartbeat: time.Now(),
|
||||
}
|
||||
|
||||
if err := w.client.RegisterWorker(ctx, info); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return w.client.UpdateWorkerHeartbeat(ctx, w.ID)
|
||||
}
|
||||
|
||||
// heartbeatLoop sends periodic heartbeats
|
||||
func (w *Worker) heartbeatLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(HeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := w.client.UpdateWorkerHeartbeat(ctx, w.ID); err != nil {
|
||||
w.logger.Warn("failed to send heartbeat", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processNextTask waits for and processes the next task
|
||||
func (w *Worker) processNextTask(ctx context.Context) error {
|
||||
// Block waiting for a task
|
||||
task, err := w.client.PopTask(ctx, TaskPollTimeout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if task == nil {
|
||||
return nil // Timeout, no task available
|
||||
}
|
||||
|
||||
w.logger.Info("received task",
|
||||
zap.String("task_id", task.ID),
|
||||
zap.String("workflow", task.WorkflowName),
|
||||
zap.String("target", task.Target),
|
||||
)
|
||||
w.printer.Info("Received task %s: %s -> %s", task.ID, task.WorkflowName, task.Target)
|
||||
|
||||
// Mark task as running
|
||||
task.MarkRunning(w.ID)
|
||||
if err := w.client.SetTaskRunning(ctx, task); err != nil {
|
||||
w.logger.Error("failed to mark task running", zap.Error(err))
|
||||
}
|
||||
|
||||
// Update worker status
|
||||
w.updateStatus(ctx, "busy", task.ID)
|
||||
|
||||
// Execute the task
|
||||
result := w.executeTask(ctx, task)
|
||||
|
||||
// Report result
|
||||
if err := w.client.SetTaskResult(ctx, result); err != nil {
|
||||
w.logger.Error("failed to report task result", zap.Error(err))
|
||||
}
|
||||
|
||||
// Remove from running
|
||||
if err := w.client.RemoveTaskRunning(ctx, task.ID); err != nil {
|
||||
w.logger.Error("failed to remove task from running", zap.Error(err))
|
||||
}
|
||||
|
||||
// Update stats and status
|
||||
if result.Status == TaskStatusCompleted {
|
||||
w.tasksComplete++
|
||||
w.printer.Success("Task %s completed", task.ID)
|
||||
} else {
|
||||
w.tasksFailed++
|
||||
w.printer.Error("Task %s failed: %s", task.ID, result.Error)
|
||||
}
|
||||
w.updateStatus(ctx, "idle", "")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// executeTask executes a workflow task
|
||||
func (w *Worker) executeTask(ctx context.Context, task *Task) *TaskResult {
|
||||
result := &TaskResult{
|
||||
TaskID: task.ID,
|
||||
CompletedAt: time.Now(),
|
||||
}
|
||||
|
||||
// Load workflow
|
||||
workflow, err := w.loader.LoadWorkflow(task.WorkflowName)
|
||||
if err != nil {
|
||||
result.Status = TaskStatusFailed
|
||||
result.Error = fmt.Sprintf("failed to load workflow: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
// Convert params to string map
|
||||
params := make(map[string]string)
|
||||
params["target"] = task.Target
|
||||
for k, v := range task.Params {
|
||||
if s, ok := v.(string); ok {
|
||||
params[k] = s
|
||||
}
|
||||
}
|
||||
|
||||
// Execute based on workflow kind
|
||||
var wfResult *core.WorkflowResult
|
||||
if workflow.IsFlow() {
|
||||
wfResult, err = w.executor.ExecuteFlow(ctx, workflow, params, w.config)
|
||||
} else {
|
||||
wfResult, err = w.executor.ExecuteModule(ctx, workflow, params, w.config)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
result.Status = TaskStatusFailed
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
|
||||
// Check result status
|
||||
if wfResult.Status == core.RunStatusFailed {
|
||||
result.Status = TaskStatusFailed
|
||||
if wfResult.Error != nil {
|
||||
result.Error = wfResult.Error.Error()
|
||||
} else {
|
||||
result.Error = "workflow execution failed"
|
||||
}
|
||||
} else {
|
||||
result.Status = TaskStatusCompleted
|
||||
result.Exports = wfResult.Exports
|
||||
}
|
||||
|
||||
result.CompletedAt = time.Now()
|
||||
return result
|
||||
}
|
||||
|
||||
// updateStatus updates the worker's status in Redis
|
||||
func (w *Worker) updateStatus(ctx context.Context, status string, taskID string) {
|
||||
info := &WorkerInfo{
|
||||
ID: w.ID,
|
||||
Hostname: w.Hostname,
|
||||
Status: status,
|
||||
CurrentTaskID: taskID,
|
||||
JoinedAt: time.Now(), // This will be overwritten, but we need a value
|
||||
LastHeartbeat: time.Now(),
|
||||
TasksComplete: w.tasksComplete,
|
||||
TasksFailed: w.tasksFailed,
|
||||
}
|
||||
if err := w.client.RegisterWorker(ctx, info); err != nil {
|
||||
w.logger.Warn("failed to update worker status", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// cleanup removes the worker from the registry
|
||||
func (w *Worker) cleanup(ctx context.Context) {
|
||||
w.printer.Info("Cleaning up worker %s...", w.ID)
|
||||
if err := w.client.RemoveWorker(ctx, w.ID); err != nil {
|
||||
w.logger.Warn("failed to remove worker", zap.Error(err))
|
||||
}
|
||||
w.client.Close()
|
||||
}
|
||||
|
||||
// GetID returns the worker ID
|
||||
func (w *Worker) GetID() string {
|
||||
return w.ID
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/database"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/template"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// RegisterArtifacts registers workflow reports and state files as artifacts in the database
|
||||
func RegisterArtifacts(workflow *core.Workflow, execCtx *core.ExecutionContext, logger *zap.Logger) error {
|
||||
db := database.GetDB()
|
||||
if db == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
templateEngine := template.NewEngine()
|
||||
|
||||
// Get output path
|
||||
outputPath, ok := execCtx.GetVariable("Output")
|
||||
if !ok {
|
||||
logger.Debug("Output variable not set, skipping artifact registration")
|
||||
return nil
|
||||
}
|
||||
outputStr, _ := outputPath.(string)
|
||||
|
||||
// Get run ID - try dbRunID first (from server mode), then execCtx.RunID
|
||||
runID := execCtx.RunID
|
||||
|
||||
// Register workflow reports
|
||||
for _, report := range workflow.Reports {
|
||||
// Render the path template
|
||||
renderedPath, err := templateEngine.Render(report.Path, execCtx.Variables)
|
||||
if err != nil {
|
||||
logger.Warn("Failed to render report path",
|
||||
zap.String("name", report.Name),
|
||||
zap.String("path", report.Path),
|
||||
zap.Error(err),
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
// Determine content type from report type
|
||||
contentType := mapReportTypeToContentType(report.Type)
|
||||
|
||||
artifact := database.Artifact{
|
||||
ID: uuid.New().String(),
|
||||
RunID: runID,
|
||||
Workspace: execCtx.WorkspaceName,
|
||||
Name: report.Name,
|
||||
ArtifactPath: renderedPath,
|
||||
ArtifactType: database.ArtifactTypeReport,
|
||||
ContentType: contentType,
|
||||
Description: report.Description,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
// Get file stats if file exists
|
||||
if info, err := os.Stat(renderedPath); err == nil {
|
||||
artifact.SizeBytes = info.Size()
|
||||
if !info.IsDir() {
|
||||
artifact.LineCount = countLines(renderedPath)
|
||||
}
|
||||
}
|
||||
|
||||
// Insert or update artifact
|
||||
_, err = db.NewInsert().Model(&artifact).
|
||||
On("CONFLICT (id) DO UPDATE").
|
||||
Set("artifact_path = EXCLUDED.artifact_path").
|
||||
Set("artifact_type = EXCLUDED.artifact_type").
|
||||
Set("content_type = EXCLUDED.content_type").
|
||||
Set("size_bytes = EXCLUDED.size_bytes").
|
||||
Set("line_count = EXCLUDED.line_count").
|
||||
Set("description = EXCLUDED.description").
|
||||
Exec(ctx)
|
||||
|
||||
if err != nil {
|
||||
logger.Warn("Failed to register report artifact",
|
||||
zap.String("name", report.Name),
|
||||
zap.Error(err),
|
||||
)
|
||||
} else {
|
||||
logger.Debug("Registered report artifact",
|
||||
zap.String("name", report.Name),
|
||||
zap.String("path", renderedPath),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Register state files
|
||||
for _, stateFile := range database.DefaultStateFiles {
|
||||
statePath := filepath.Join(outputStr, stateFile.FileName)
|
||||
|
||||
// Check if artifact already exists for this workspace + name
|
||||
var existingArtifact database.Artifact
|
||||
err := db.NewSelect().
|
||||
Model(&existingArtifact).
|
||||
Where("workspace = ? AND name = ?", execCtx.WorkspaceName, stateFile.Name).
|
||||
Scan(ctx)
|
||||
|
||||
if err == nil {
|
||||
// Artifact exists - update size_bytes and line_count only
|
||||
if info, statErr := os.Stat(statePath); statErr == nil {
|
||||
lineCount := 0
|
||||
if !info.IsDir() {
|
||||
lineCount = countLines(statePath)
|
||||
}
|
||||
_, updateErr := db.NewUpdate().
|
||||
Model(&existingArtifact).
|
||||
Set("size_bytes = ?", info.Size()).
|
||||
Set("line_count = ?", lineCount).
|
||||
Set("run_id = ?", runID). // Update run_id to latest run
|
||||
Where("id = ?", existingArtifact.ID).
|
||||
Exec(ctx)
|
||||
if updateErr != nil {
|
||||
logger.Warn("Failed to update artifact size",
|
||||
zap.String("name", stateFile.Name),
|
||||
zap.Error(updateErr),
|
||||
)
|
||||
} else {
|
||||
logger.Debug("Updated existing state file artifact",
|
||||
zap.String("name", stateFile.Name),
|
||||
zap.String("path", statePath),
|
||||
)
|
||||
}
|
||||
}
|
||||
continue // Skip insert
|
||||
}
|
||||
|
||||
// Insert new artifact (only if doesn't exist)
|
||||
artifact := database.Artifact{
|
||||
ID: uuid.New().String(),
|
||||
RunID: runID,
|
||||
Workspace: execCtx.WorkspaceName,
|
||||
Name: stateFile.Name,
|
||||
ArtifactPath: statePath,
|
||||
ArtifactType: stateFile.ArtifactType,
|
||||
ContentType: stateFile.ContentType,
|
||||
Description: stateFile.Description,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
// Get file stats if file exists
|
||||
if info, err := os.Stat(statePath); err == nil {
|
||||
artifact.SizeBytes = info.Size()
|
||||
if !info.IsDir() {
|
||||
artifact.LineCount = countLines(statePath)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = db.NewInsert().Model(&artifact).Exec(ctx)
|
||||
|
||||
if err != nil {
|
||||
logger.Warn("Failed to register state file artifact",
|
||||
zap.String("name", stateFile.Name),
|
||||
zap.Error(err),
|
||||
)
|
||||
} else {
|
||||
logger.Debug("Registered state file artifact",
|
||||
zap.String("name", stateFile.Name),
|
||||
zap.String("path", statePath),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// mapReportTypeToContentType converts workflow report type to database content type
|
||||
func mapReportTypeToContentType(reportType string) string {
|
||||
switch strings.ToLower(reportType) {
|
||||
case "json":
|
||||
return database.ContentTypeJSON
|
||||
case "jsonl":
|
||||
return database.ContentTypeJSONL
|
||||
case "yaml", "yml":
|
||||
return database.ContentTypeYAML
|
||||
case "html":
|
||||
return database.ContentTypeHTML
|
||||
case "markdown", "md":
|
||||
return database.ContentTypeMarkdown
|
||||
case "log":
|
||||
return database.ContentTypeLog
|
||||
case "pdf":
|
||||
return database.ContentTypePDF
|
||||
case "png", "image":
|
||||
return database.ContentTypePNG
|
||||
case "text", "txt":
|
||||
return database.ContentTypeText
|
||||
case "zip":
|
||||
return database.ContentTypeZip
|
||||
case "folder", "directory":
|
||||
return database.ContentTypeFolder
|
||||
default:
|
||||
return database.ContentTypeUnknown
|
||||
}
|
||||
}
|
||||
|
||||
// countLines counts the number of lines in a file
|
||||
func countLines(filePath string) int {
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return 0
|
||||
}
|
||||
count := 1
|
||||
for _, b := range data {
|
||||
if b == '\n' {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/runner"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/template"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// BashExecutor executes bash steps
|
||||
type BashExecutor struct {
|
||||
templateEngine *template.Engine
|
||||
runner runner.Runner
|
||||
}
|
||||
|
||||
// NewBashExecutor creates a new bash executor
|
||||
func NewBashExecutor(engine *template.Engine) *BashExecutor {
|
||||
return &BashExecutor{
|
||||
templateEngine: engine,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the executor name for logging/debugging
|
||||
func (e *BashExecutor) Name() string {
|
||||
return "bash"
|
||||
}
|
||||
|
||||
// StepTypes returns the step types this executor handles
|
||||
func (e *BashExecutor) StepTypes() []core.StepType {
|
||||
return []core.StepType{core.StepTypeBash}
|
||||
}
|
||||
|
||||
// SetRunner sets the runner for command execution
|
||||
func (e *BashExecutor) SetRunner(r runner.Runner) {
|
||||
e.runner = r
|
||||
}
|
||||
|
||||
// assembleCommand joins the command with structured args in order:
|
||||
// command + speed_args + config_args + input_args + output_args
|
||||
func assembleCommand(command, speedArgs, configArgs, inputArgs, outputArgs string) string {
|
||||
parts := []string{command}
|
||||
if speedArgs != "" {
|
||||
parts = append(parts, speedArgs)
|
||||
}
|
||||
if configArgs != "" {
|
||||
parts = append(parts, configArgs)
|
||||
}
|
||||
if inputArgs != "" {
|
||||
parts = append(parts, inputArgs)
|
||||
}
|
||||
if outputArgs != "" {
|
||||
parts = append(parts, outputArgs)
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// writeStdFile writes command output to the specified file
|
||||
func writeStdFile(path, content string) error {
|
||||
// Ensure parent directory exists
|
||||
if dir := filepath.Dir(path); dir != "" && dir != "." {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
}
|
||||
return os.WriteFile(path, []byte(content), 0644)
|
||||
}
|
||||
|
||||
// Execute executes a bash step
|
||||
func (e *BashExecutor) Execute(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) {
|
||||
result := &core.StepResult{
|
||||
StepName: step.Name,
|
||||
Status: core.StepStatusRunning,
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
|
||||
timeout, err := step.Timeout.Duration()
|
||||
if err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
var output string
|
||||
|
||||
// Determine execution mode
|
||||
if len(step.ParallelCommands) > 0 {
|
||||
output, err = e.executeParallel(ctx, step.ParallelCommands, timeout)
|
||||
} else if len(step.Commands) > 0 {
|
||||
output, err = e.executeSequential(ctx, step.Commands, timeout)
|
||||
} else if step.Command != "" {
|
||||
// Assemble command with structured args if present
|
||||
finalCmd := assembleCommand(step.Command, step.SpeedArgs, step.ConfigArgs, step.InputArgs, step.OutputArgs)
|
||||
output, err = e.executeCommand(ctx, finalCmd, timeout)
|
||||
} else {
|
||||
err = fmt.Errorf("no command specified")
|
||||
}
|
||||
|
||||
result.Output = output
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
|
||||
// Write stdout/stderr to file if std_file is specified
|
||||
if step.StdFile != "" {
|
||||
if writeErr := writeStdFile(step.StdFile, output); writeErr != nil {
|
||||
// Log warning but don't fail the step
|
||||
execCtx.Logger.Warn("Failed to write std_file",
|
||||
zap.String("path", step.StdFile),
|
||||
zap.Error(writeErr))
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
return result, err
|
||||
}
|
||||
|
||||
result.Status = core.StepStatusSuccess
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// executeCommand executes a single command
|
||||
|
||||
func (e *BashExecutor) executeCommand(ctx context.Context, command string, timeout time.Duration) (string, error) {
|
||||
// Apply timeout if specified
|
||||
if timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
// Use runner if available, otherwise fall back to local execution
|
||||
if e.runner != nil {
|
||||
result, err := e.runner.Execute(ctx, command)
|
||||
if err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return result.Output, fmt.Errorf("command timed out after %s", timeout)
|
||||
}
|
||||
return result.Output, fmt.Errorf("command failed: %w", err)
|
||||
}
|
||||
if result.ExitCode != 0 {
|
||||
return result.Output, fmt.Errorf("command exited with code %d", result.ExitCode)
|
||||
}
|
||||
return strings.TrimSpace(result.Output), nil
|
||||
}
|
||||
|
||||
// Fallback to local execution
|
||||
// @NOTE: yes yes, I know this is a security risk. This is the intended behavior.
|
||||
cmd := exec.CommandContext(ctx, "sh", "-c", command)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
err := cmd.Run()
|
||||
|
||||
output := stdout.String()
|
||||
if stderr.Len() > 0 {
|
||||
output += "\n" + stderr.String()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return output, fmt.Errorf("command timed out after %s", timeout)
|
||||
}
|
||||
return output, fmt.Errorf("command failed: %w\nstderr: %s", err, stderr.String())
|
||||
}
|
||||
|
||||
return strings.TrimSpace(output), nil
|
||||
}
|
||||
|
||||
// executeSequential executes commands sequentially
|
||||
func (e *BashExecutor) executeSequential(ctx context.Context, commands []string, timeout time.Duration) (string, error) {
|
||||
var outputs []string
|
||||
|
||||
for _, cmd := range commands {
|
||||
output, err := e.executeCommand(ctx, cmd, timeout)
|
||||
outputs = append(outputs, output)
|
||||
if err != nil {
|
||||
return strings.Join(outputs, "\n"), err
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(outputs, "\n"), nil
|
||||
}
|
||||
|
||||
// executeParallel executes commands in parallel
|
||||
func (e *BashExecutor) executeParallel(ctx context.Context, commands []string, timeout time.Duration) (string, error) {
|
||||
type result struct {
|
||||
index int
|
||||
output string
|
||||
err error
|
||||
}
|
||||
|
||||
results := make(chan result, len(commands))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i, cmd := range commands {
|
||||
wg.Add(1)
|
||||
go func(idx int, command string) {
|
||||
defer wg.Done()
|
||||
output, err := e.executeCommand(ctx, command, timeout)
|
||||
results <- result{index: idx, output: output, err: err}
|
||||
}(i, cmd)
|
||||
}
|
||||
|
||||
// Wait for all commands to complete
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(results)
|
||||
}()
|
||||
|
||||
// Collect results
|
||||
outputs := make([]string, len(commands))
|
||||
var firstError error
|
||||
|
||||
for r := range results {
|
||||
outputs[r.index] = r.output
|
||||
if r.err != nil && firstError == nil {
|
||||
firstError = r.err
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(outputs, "\n"), firstError
|
||||
}
|
||||
|
||||
// CanHandle returns true if this executor can handle the given step type
|
||||
func (e *BashExecutor) CanHandle(stepType core.StepType) bool {
|
||||
return stepType == core.StepTypeBash
|
||||
}
|
||||
@@ -0,0 +1,533 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/config"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/functions"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/logger"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/runner"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/template"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// functionCallPattern matches function call syntax like functionName(...)
|
||||
var functionCallPattern = regexp.MustCompile(`\w+\s*\(`)
|
||||
|
||||
// StepDispatcher dispatches steps to appropriate executors
|
||||
type StepDispatcher struct {
|
||||
registry *PluginRegistry
|
||||
templateEngine *template.Engine
|
||||
functionRegistry *functions.Registry
|
||||
dryRun bool
|
||||
runner runner.Runner
|
||||
// Keep direct references to executors that need special configuration
|
||||
bashExecutor *BashExecutor
|
||||
llmExecutor *LLMExecutor
|
||||
}
|
||||
|
||||
// SetDryRun enables or disables dry-run mode for the dispatcher
|
||||
func (d *StepDispatcher) SetDryRun(dryRun bool) {
|
||||
d.dryRun = dryRun
|
||||
}
|
||||
|
||||
// SetSilent enables or disables silent mode for executors that support it
|
||||
func (d *StepDispatcher) SetSilent(silent bool) {
|
||||
d.llmExecutor.SetSilent(silent)
|
||||
}
|
||||
|
||||
// SetRunner sets the runner for command execution
|
||||
func (d *StepDispatcher) SetRunner(r runner.Runner) {
|
||||
d.runner = r
|
||||
d.bashExecutor.SetRunner(r)
|
||||
}
|
||||
|
||||
// NewStepDispatcher creates a new step dispatcher
|
||||
func NewStepDispatcher() *StepDispatcher {
|
||||
d := &StepDispatcher{
|
||||
registry: NewPluginRegistry(),
|
||||
templateEngine: template.NewEngine(),
|
||||
functionRegistry: functions.NewRegistry(),
|
||||
}
|
||||
|
||||
// Create executors
|
||||
d.bashExecutor = NewBashExecutor(d.templateEngine)
|
||||
d.llmExecutor = NewLLMExecutor(d.templateEngine)
|
||||
|
||||
// Register all built-in plugins
|
||||
d.registry.Register(d.bashExecutor)
|
||||
d.registry.Register(NewFunctionExecutor(d.templateEngine, d.functionRegistry))
|
||||
d.registry.Register(NewParallelExecutor(d))
|
||||
d.registry.Register(NewForeachExecutor(d, d.templateEngine))
|
||||
d.registry.Register(NewRemoteBashExecutor(d.templateEngine))
|
||||
d.registry.Register(NewHTTPExecutor(d.templateEngine))
|
||||
d.registry.Register(d.llmExecutor)
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
// RegisterPlugin allows external plugin registration
|
||||
func (d *StepDispatcher) RegisterPlugin(plugin StepExecutorPlugin) {
|
||||
d.registry.Register(plugin)
|
||||
}
|
||||
|
||||
// SetConfig passes config to executors that need it
|
||||
func (d *StepDispatcher) SetConfig(cfg *config.Config) {
|
||||
d.llmExecutor.SetConfig(cfg)
|
||||
}
|
||||
|
||||
// Dispatch dispatches a step to the appropriate executor
|
||||
func (d *StepDispatcher) Dispatch(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) {
|
||||
log := logger.Get()
|
||||
|
||||
log.Debug("Dispatching step",
|
||||
zap.String("step_name", step.Name),
|
||||
zap.String("step_type", string(step.Type)),
|
||||
zap.Bool("dry_run", d.dryRun),
|
||||
)
|
||||
|
||||
// Render templates in step fields
|
||||
log.Debug("Rendering step templates")
|
||||
renderedStep, err := d.renderStep(step, execCtx)
|
||||
if err != nil {
|
||||
log.Debug("Template rendering failed", zap.Error(err))
|
||||
return nil, fmt.Errorf("template rendering failed: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Step templates rendered",
|
||||
zap.String("command", renderedStep.Command),
|
||||
)
|
||||
|
||||
// Log step message if provided
|
||||
if renderedStep.Log != "" {
|
||||
log.Info(renderedStep.Log,
|
||||
zap.String("step", step.Name),
|
||||
)
|
||||
}
|
||||
|
||||
// Dispatch based on step type using plugin registry
|
||||
log.Debug("Dispatching to executor",
|
||||
zap.String("executor_type", string(step.Type)),
|
||||
)
|
||||
|
||||
plugin, ok := d.registry.Get(step.Type)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown step type: %s", step.Type)
|
||||
}
|
||||
|
||||
log.Debug("Using plugin", zap.String("plugin_name", plugin.Name()))
|
||||
result, err := plugin.Execute(ctx, renderedStep, execCtx)
|
||||
|
||||
if err != nil {
|
||||
log.Debug("Step execution failed",
|
||||
zap.String("step", step.Name),
|
||||
zap.Error(err),
|
||||
)
|
||||
return result, err
|
||||
}
|
||||
|
||||
log.Debug("Step execution completed",
|
||||
zap.String("step", step.Name),
|
||||
zap.String("status", string(result.Status)),
|
||||
)
|
||||
|
||||
// Process exports
|
||||
if step.HasExports() {
|
||||
log.Debug("Processing exports",
|
||||
zap.Int("export_count", len(step.Exports)),
|
||||
)
|
||||
|
||||
// Merge auto-exports (e.g., from HTTP steps) into vars before evaluating user exports
|
||||
vars := execCtx.GetVariables()
|
||||
if result.Exports != nil {
|
||||
for k, v := range result.Exports {
|
||||
vars[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
// Render template variables in export values first, then evaluate if needed
|
||||
exports := make(map[string]interface{}, len(step.Exports))
|
||||
for name, expr := range step.Exports {
|
||||
rendered, err := d.templateEngine.Render(expr, vars)
|
||||
if err != nil {
|
||||
log.Warn("Failed to render export value, using original",
|
||||
zap.String("export", name),
|
||||
zap.Error(err))
|
||||
rendered = expr
|
||||
}
|
||||
|
||||
// Only evaluate with JS if the rendered value contains a function call
|
||||
// Otherwise, use the rendered string directly
|
||||
if functionCallPattern.MatchString(rendered) {
|
||||
value, err := d.functionRegistry.Execute(rendered, vars)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("export evaluation failed for %s: %w", name, err)
|
||||
}
|
||||
exports[name] = value
|
||||
} else {
|
||||
// Use rendered value directly as a string
|
||||
exports[name] = rendered
|
||||
}
|
||||
}
|
||||
if result.Exports == nil {
|
||||
result.Exports = make(map[string]interface{})
|
||||
}
|
||||
for k, v := range exports {
|
||||
result.Exports[k] = v
|
||||
}
|
||||
log.Debug("Exports processed", zap.Int("total_exports", len(result.Exports)))
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// renderStep renders all template fields in a step
|
||||
func (d *StepDispatcher) renderStep(step *core.Step, execCtx *core.ExecutionContext) (*core.Step, error) {
|
||||
vars := execCtx.GetVariables()
|
||||
|
||||
// Create a copy of the step
|
||||
rendered := *step
|
||||
|
||||
// Render command fields
|
||||
if step.Command != "" {
|
||||
cmd, err := d.templateEngine.Render(step.Command, vars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rendered.Command = cmd
|
||||
}
|
||||
|
||||
if len(step.Commands) > 0 {
|
||||
cmds, err := d.templateEngine.RenderSlice(step.Commands, vars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rendered.Commands = cmds
|
||||
}
|
||||
|
||||
if len(step.ParallelCommands) > 0 {
|
||||
cmds, err := d.templateEngine.RenderSlice(step.ParallelCommands, vars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rendered.ParallelCommands = cmds
|
||||
}
|
||||
|
||||
// Render structured argument fields (for bash/remote-bash steps)
|
||||
if step.SpeedArgs != "" {
|
||||
args, err := d.templateEngine.Render(step.SpeedArgs, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering speed_args: %w", err)
|
||||
}
|
||||
rendered.SpeedArgs = args
|
||||
}
|
||||
if step.ConfigArgs != "" {
|
||||
args, err := d.templateEngine.Render(step.ConfigArgs, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering config_args: %w", err)
|
||||
}
|
||||
rendered.ConfigArgs = args
|
||||
}
|
||||
if step.InputArgs != "" {
|
||||
args, err := d.templateEngine.Render(step.InputArgs, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering input_args: %w", err)
|
||||
}
|
||||
rendered.InputArgs = args
|
||||
}
|
||||
if step.OutputArgs != "" {
|
||||
args, err := d.templateEngine.Render(step.OutputArgs, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering output_args: %w", err)
|
||||
}
|
||||
rendered.OutputArgs = args
|
||||
}
|
||||
|
||||
// Render std_file for stdout/stderr capture
|
||||
if step.StdFile != "" {
|
||||
stdFile, err := d.templateEngine.Render(step.StdFile, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering std_file: %w", err)
|
||||
}
|
||||
rendered.StdFile = stdFile
|
||||
}
|
||||
|
||||
// Render function fields
|
||||
if step.Function != "" {
|
||||
fn, err := d.templateEngine.Render(step.Function, vars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rendered.Function = fn
|
||||
}
|
||||
|
||||
if len(step.Functions) > 0 {
|
||||
fns, err := d.templateEngine.RenderSlice(step.Functions, vars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rendered.Functions = fns
|
||||
}
|
||||
|
||||
if len(step.ParallelFunctions) > 0 {
|
||||
fns, err := d.templateEngine.RenderSlice(step.ParallelFunctions, vars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rendered.ParallelFunctions = fns
|
||||
}
|
||||
|
||||
// Render foreach fields
|
||||
if step.Input != "" {
|
||||
input, err := d.templateEngine.Render(step.Input, vars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rendered.Input = input
|
||||
}
|
||||
|
||||
// Render log message
|
||||
if step.Log != "" {
|
||||
log, err := d.templateEngine.Render(step.Log, vars)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rendered.Log = log
|
||||
}
|
||||
|
||||
if step.Timeout != "" {
|
||||
to, err := d.templateEngine.Render(string(step.Timeout), vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering timeout: %w", err)
|
||||
}
|
||||
rendered.Timeout = core.StepTimeout(to)
|
||||
}
|
||||
|
||||
if step.Threads != "" {
|
||||
th, err := d.templateEngine.Render(string(step.Threads), vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering threads: %w", err)
|
||||
}
|
||||
rendered.Threads = core.StepThreads(th)
|
||||
}
|
||||
|
||||
// Render HTTP step fields
|
||||
if step.URL != "" {
|
||||
url, err := d.templateEngine.Render(step.URL, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering url: %w", err)
|
||||
}
|
||||
rendered.URL = url
|
||||
}
|
||||
if step.Method != "" {
|
||||
method, err := d.templateEngine.Render(step.Method, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering method: %w", err)
|
||||
}
|
||||
rendered.Method = method
|
||||
}
|
||||
if step.RequestBody != "" {
|
||||
body, err := d.templateEngine.Render(step.RequestBody, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering request_body: %w", err)
|
||||
}
|
||||
rendered.RequestBody = body
|
||||
}
|
||||
if len(step.Headers) > 0 {
|
||||
headers, err := d.templateEngine.RenderMap(step.Headers, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering headers: %w", err)
|
||||
}
|
||||
rendered.Headers = headers
|
||||
}
|
||||
|
||||
// Render step_runner if it contains template variables
|
||||
if step.StepRunner != "" {
|
||||
sr, err := d.templateEngine.Render(string(step.StepRunner), vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_runner: %w", err)
|
||||
}
|
||||
rendered.StepRunner = core.RunnerType(sr)
|
||||
}
|
||||
|
||||
// Render step_runner_config fields for remote-bash steps
|
||||
if step.StepRunnerConfig != nil {
|
||||
renderedConfig := &core.StepRunnerConfig{}
|
||||
|
||||
if step.StepRunnerConfig.RunnerConfig != nil {
|
||||
cfg := *step.StepRunnerConfig.RunnerConfig
|
||||
|
||||
// Render string fields that may contain templates
|
||||
if cfg.Image != "" {
|
||||
img, err := d.templateEngine.Render(cfg.Image, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_runner_config.image: %w", err)
|
||||
}
|
||||
cfg.Image = img
|
||||
}
|
||||
if cfg.Host != "" {
|
||||
host, err := d.templateEngine.Render(cfg.Host, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_runner_config.host: %w", err)
|
||||
}
|
||||
cfg.Host = host
|
||||
}
|
||||
if cfg.User != "" {
|
||||
user, err := d.templateEngine.Render(cfg.User, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_runner_config.user: %w", err)
|
||||
}
|
||||
cfg.User = user
|
||||
}
|
||||
if cfg.Password != "" {
|
||||
pass, err := d.templateEngine.Render(cfg.Password, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_runner_config.password: %w", err)
|
||||
}
|
||||
cfg.Password = pass
|
||||
}
|
||||
if cfg.KeyFile != "" {
|
||||
keyFile, err := d.templateEngine.Render(cfg.KeyFile, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_runner_config.key_file: %w", err)
|
||||
}
|
||||
cfg.KeyFile = keyFile
|
||||
}
|
||||
if cfg.WorkDir != "" {
|
||||
workDir, err := d.templateEngine.Render(cfg.WorkDir, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_runner_config.workdir: %w", err)
|
||||
}
|
||||
cfg.WorkDir = workDir
|
||||
}
|
||||
if cfg.Network != "" {
|
||||
network, err := d.templateEngine.Render(cfg.Network, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_runner_config.network: %w", err)
|
||||
}
|
||||
cfg.Network = network
|
||||
}
|
||||
|
||||
// Render env map values
|
||||
if len(cfg.Env) > 0 {
|
||||
renderedEnv, err := d.templateEngine.RenderMap(cfg.Env, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_runner_config.env: %w", err)
|
||||
}
|
||||
cfg.Env = renderedEnv
|
||||
}
|
||||
|
||||
// Render volumes slice
|
||||
if len(cfg.Volumes) > 0 {
|
||||
renderedVols, err := d.templateEngine.RenderSlice(cfg.Volumes, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_runner_config.volumes: %w", err)
|
||||
}
|
||||
cfg.Volumes = renderedVols
|
||||
}
|
||||
|
||||
renderedConfig.RunnerConfig = &cfg
|
||||
}
|
||||
|
||||
rendered.StepRunnerConfig = renderedConfig
|
||||
}
|
||||
|
||||
// Render remote-bash file copy fields
|
||||
if step.StepRemoteFile != "" {
|
||||
remoteFile, err := d.templateEngine.Render(step.StepRemoteFile, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering step_remote_file: %w", err)
|
||||
}
|
||||
rendered.StepRemoteFile = remoteFile
|
||||
}
|
||||
if step.HostOutputFile != "" {
|
||||
hostFile, err := d.templateEngine.Render(step.HostOutputFile, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering host_output_file: %w", err)
|
||||
}
|
||||
rendered.HostOutputFile = hostFile
|
||||
}
|
||||
|
||||
// Render LLM step fields
|
||||
if len(step.Messages) > 0 {
|
||||
renderedMessages := make([]core.LLMMessage, len(step.Messages))
|
||||
for i, msg := range step.Messages {
|
||||
renderedMsg := msg
|
||||
|
||||
// Render content (can be string or []interface{})
|
||||
switch content := msg.Content.(type) {
|
||||
case string:
|
||||
renderedContent, err := d.templateEngine.Render(content, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering message content: %w", err)
|
||||
}
|
||||
renderedMsg.Content = renderedContent
|
||||
case []interface{}:
|
||||
// Handle multimodal content parts
|
||||
renderedParts := make([]interface{}, len(content))
|
||||
for j, part := range content {
|
||||
if partMap, ok := part.(map[string]interface{}); ok {
|
||||
renderedPartMap := make(map[string]interface{})
|
||||
for k, v := range partMap {
|
||||
renderedPartMap[k] = v
|
||||
}
|
||||
// Render text field
|
||||
if text, ok := partMap["text"].(string); ok {
|
||||
renderedText, err := d.templateEngine.Render(text, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering content part text: %w", err)
|
||||
}
|
||||
renderedPartMap["text"] = renderedText
|
||||
}
|
||||
// Render image_url.url if present
|
||||
if imgURL, ok := partMap["image_url"].(map[string]interface{}); ok {
|
||||
renderedImgURL := make(map[string]interface{})
|
||||
for k, v := range imgURL {
|
||||
renderedImgURL[k] = v
|
||||
}
|
||||
if url, ok := imgURL["url"].(string); ok {
|
||||
renderedURL, err := d.templateEngine.Render(url, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering image URL: %w", err)
|
||||
}
|
||||
renderedImgURL["url"] = renderedURL
|
||||
}
|
||||
renderedPartMap["image_url"] = renderedImgURL
|
||||
}
|
||||
renderedParts[j] = renderedPartMap
|
||||
} else {
|
||||
renderedParts[j] = part
|
||||
}
|
||||
}
|
||||
renderedMsg.Content = renderedParts
|
||||
}
|
||||
|
||||
renderedMessages[i] = renderedMsg
|
||||
}
|
||||
rendered.Messages = renderedMessages
|
||||
}
|
||||
|
||||
// Render embedding input
|
||||
if len(step.EmbeddingInput) > 0 {
|
||||
embInputs, err := d.templateEngine.RenderSlice(step.EmbeddingInput, vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error rendering embedding_input: %w", err)
|
||||
}
|
||||
rendered.EmbeddingInput = embInputs
|
||||
}
|
||||
|
||||
return &rendered, nil
|
||||
}
|
||||
|
||||
// GetFunctionRegistry returns the function registry
|
||||
func (d *StepDispatcher) GetFunctionRegistry() *functions.Registry {
|
||||
return d.functionRegistry
|
||||
}
|
||||
|
||||
// GetTemplateEngine returns the template engine
|
||||
func (d *StepDispatcher) GetTemplateEngine() *template.Engine {
|
||||
return d.templateEngine
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,319 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/template"
|
||||
)
|
||||
|
||||
// ForeachExecutor executes foreach steps
|
||||
type ForeachExecutor struct {
|
||||
dispatcher *StepDispatcher
|
||||
templateEngine *template.Engine
|
||||
}
|
||||
|
||||
// NewForeachExecutor creates a new foreach executor
|
||||
func NewForeachExecutor(dispatcher *StepDispatcher, engine *template.Engine) *ForeachExecutor {
|
||||
return &ForeachExecutor{
|
||||
dispatcher: dispatcher,
|
||||
templateEngine: engine,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the executor name for logging/debugging
|
||||
func (e *ForeachExecutor) Name() string {
|
||||
return "foreach"
|
||||
}
|
||||
|
||||
// StepTypes returns the step types this executor handles
|
||||
func (e *ForeachExecutor) StepTypes() []core.StepType {
|
||||
return []core.StepType{core.StepTypeForeach}
|
||||
}
|
||||
|
||||
// Execute executes a foreach step
|
||||
func (e *ForeachExecutor) Execute(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) {
|
||||
result := &core.StepResult{
|
||||
StepName: step.Name,
|
||||
Status: core.StepStatusRunning,
|
||||
StartTime: time.Now(),
|
||||
Exports: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
if step.Step == nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = fmt.Errorf("foreach step has no inner step")
|
||||
result.EndTime = time.Now()
|
||||
return result, result.Error
|
||||
}
|
||||
|
||||
threads, err := step.Threads.Int()
|
||||
if err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
if threads <= 0 {
|
||||
threads = 1
|
||||
}
|
||||
|
||||
// Execute with streaming worker pool
|
||||
outputs, err := e.executeWithWorkerPool(ctx, step, step.Input, threads, execCtx)
|
||||
|
||||
result.Output = strings.Join(outputs, "\n")
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
|
||||
if err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
return result, err
|
||||
}
|
||||
|
||||
result.Status = core.StepStatusSuccess
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// LineIterator provides streaming access to lines in a file
|
||||
type LineIterator struct {
|
||||
file *os.File
|
||||
scanner *bufio.Scanner
|
||||
current string
|
||||
err error
|
||||
}
|
||||
|
||||
// NewLineIterator creates an iterator for reading lines from a file
|
||||
func NewLineIterator(path string) (*LineIterator, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
// Increase scanner buffer for long lines
|
||||
scanner.Buffer(make([]byte, 64*1024), 10*1024*1024)
|
||||
|
||||
return &LineIterator{
|
||||
file: file,
|
||||
scanner: scanner,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Next advances to the next non-empty line, returns false when done
|
||||
func (it *LineIterator) Next() bool {
|
||||
for it.scanner.Scan() {
|
||||
line := strings.TrimSpace(it.scanner.Text())
|
||||
if line != "" {
|
||||
it.current = line
|
||||
return true
|
||||
}
|
||||
}
|
||||
it.err = it.scanner.Err()
|
||||
return false
|
||||
}
|
||||
|
||||
// Value returns the current line
|
||||
func (it *LineIterator) Value() string {
|
||||
return it.current
|
||||
}
|
||||
|
||||
// Err returns any error encountered during iteration
|
||||
func (it *LineIterator) Err() error {
|
||||
return it.err
|
||||
}
|
||||
|
||||
// Close closes the underlying file
|
||||
func (it *LineIterator) Close() error {
|
||||
if it.file != nil {
|
||||
return it.file.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// countInputLines counts non-empty lines in an input file (for result slice allocation)
|
||||
func countInputLines(path string) (int, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
count := 0
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Buffer(make([]byte, 64*1024), 10*1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line != "" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count, scanner.Err()
|
||||
}
|
||||
|
||||
// renderSecondaryTemplates clones the step and renders [[ ]] templates with loop context
|
||||
func (e *ForeachExecutor) renderSecondaryTemplates(step *core.Step, execCtx *core.ExecutionContext) *core.Step {
|
||||
// Clone the step to avoid modifying the original
|
||||
cloned := step.Clone()
|
||||
ctx := execCtx.GetVariables()
|
||||
|
||||
// Render Command if it has secondary variables
|
||||
if e.templateEngine.HasSecondaryVariable(cloned.Command) {
|
||||
rendered, err := e.templateEngine.RenderSecondary(cloned.Command, ctx)
|
||||
if err == nil {
|
||||
cloned.Command = rendered
|
||||
}
|
||||
}
|
||||
|
||||
// Render Commands array
|
||||
for i, cmd := range cloned.Commands {
|
||||
if e.templateEngine.HasSecondaryVariable(cmd) {
|
||||
rendered, err := e.templateEngine.RenderSecondary(cmd, ctx)
|
||||
if err == nil {
|
||||
cloned.Commands[i] = rendered
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Render Input (for nested foreach)
|
||||
if e.templateEngine.HasSecondaryVariable(cloned.Input) {
|
||||
rendered, err := e.templateEngine.RenderSecondary(cloned.Input, ctx)
|
||||
if err == nil {
|
||||
cloned.Input = rendered
|
||||
}
|
||||
}
|
||||
|
||||
return cloned
|
||||
}
|
||||
|
||||
// workItem represents a single item to process in the worker pool
|
||||
type workItem struct {
|
||||
index int
|
||||
value string
|
||||
}
|
||||
|
||||
// workResult represents the result of processing a work item
|
||||
type workResult struct {
|
||||
index int
|
||||
output string
|
||||
err error
|
||||
}
|
||||
|
||||
// executeWithWorkerPool executes the inner step using a streaming worker pool pattern
|
||||
// This is memory-efficient: creates only 'threads' goroutines instead of N goroutines
|
||||
// and streams input lines on-demand instead of loading all into memory
|
||||
func (e *ForeachExecutor) executeWithWorkerPool(ctx context.Context, step *core.Step, inputPath string, threads int, execCtx *core.ExecutionContext) ([]string, error) {
|
||||
// Count lines first for result slice allocation (fast, O(n) with minimal memory)
|
||||
lineCount, err := countInputLines(inputPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to count input lines: %w", err)
|
||||
}
|
||||
|
||||
if lineCount == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Create bounded work queue - buffer 2x thread count for smooth flow
|
||||
workQueue := make(chan workItem, threads*2)
|
||||
results := make(chan workResult, threads*2)
|
||||
|
||||
// Track completion
|
||||
var workerWg sync.WaitGroup
|
||||
var producerErr error
|
||||
|
||||
// Start fixed worker pool (only 'threads' goroutines, not N)
|
||||
for i := 0; i < threads; i++ {
|
||||
workerWg.Add(1)
|
||||
go func() {
|
||||
defer workerWg.Done()
|
||||
for work := range workQueue {
|
||||
// Check context cancellation
|
||||
if ctx.Err() != nil {
|
||||
results <- workResult{index: work.index, err: ctx.Err()}
|
||||
continue
|
||||
}
|
||||
|
||||
// Create optimized child context with loop variables pre-set
|
||||
childCtx := execCtx.CloneForLoop(step.Variable, work.value, work.index+1)
|
||||
|
||||
// Clone inner step and render secondary templates [[ ]]
|
||||
innerStep := e.renderSecondaryTemplates(step.Step, childCtx)
|
||||
|
||||
// Execute inner step
|
||||
stepResult, err := e.dispatcher.Dispatch(ctx, innerStep, childCtx)
|
||||
|
||||
var output string
|
||||
if stepResult != nil {
|
||||
output = stepResult.Output
|
||||
}
|
||||
|
||||
results <- workResult{index: work.index, output: output, err: err}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Producer: stream lines into work queue (separate goroutine)
|
||||
go func() {
|
||||
defer close(workQueue)
|
||||
|
||||
iter, err := NewLineIterator(inputPath)
|
||||
if err != nil {
|
||||
producerErr = err
|
||||
return
|
||||
}
|
||||
defer func() { _ = iter.Close() }()
|
||||
|
||||
idx := 0
|
||||
for iter.Next() {
|
||||
select {
|
||||
case workQueue <- workItem{index: idx, value: iter.Value()}:
|
||||
idx++
|
||||
case <-ctx.Done():
|
||||
producerErr = ctx.Err()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if iter.Err() != nil {
|
||||
producerErr = iter.Err()
|
||||
}
|
||||
}()
|
||||
|
||||
// Collector: close results when all workers done
|
||||
go func() {
|
||||
workerWg.Wait()
|
||||
close(results)
|
||||
}()
|
||||
|
||||
// Collect results in order
|
||||
outputs := make([]string, lineCount)
|
||||
var firstError error
|
||||
|
||||
for r := range results {
|
||||
if r.index < len(outputs) {
|
||||
outputs[r.index] = r.output
|
||||
}
|
||||
if r.err != nil && firstError == nil {
|
||||
firstError = r.err
|
||||
}
|
||||
}
|
||||
|
||||
// Check for producer error
|
||||
if producerErr != nil && firstError == nil {
|
||||
firstError = producerErr
|
||||
}
|
||||
|
||||
return outputs, firstError
|
||||
}
|
||||
|
||||
// CanHandle returns true if this executor can handle the given step type
|
||||
func (e *ForeachExecutor) CanHandle(stepType core.StepType) bool {
|
||||
return stepType == core.StepTypeForeach
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/functions"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/template"
|
||||
)
|
||||
|
||||
// FunctionExecutor executes function steps
|
||||
type FunctionExecutor struct {
|
||||
templateEngine *template.Engine
|
||||
functionRegistry *functions.Registry
|
||||
}
|
||||
|
||||
// NewFunctionExecutor creates a new function executor
|
||||
func NewFunctionExecutor(engine *template.Engine, registry *functions.Registry) *FunctionExecutor {
|
||||
return &FunctionExecutor{
|
||||
templateEngine: engine,
|
||||
functionRegistry: registry,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the executor name for logging/debugging
|
||||
func (e *FunctionExecutor) Name() string {
|
||||
return "function"
|
||||
}
|
||||
|
||||
// StepTypes returns the step types this executor handles
|
||||
func (e *FunctionExecutor) StepTypes() []core.StepType {
|
||||
return []core.StepType{core.StepTypeFunction}
|
||||
}
|
||||
|
||||
// Execute executes a function step
|
||||
func (e *FunctionExecutor) Execute(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) {
|
||||
result := &core.StepResult{
|
||||
StepName: step.Name,
|
||||
Status: core.StepStatusRunning,
|
||||
StartTime: time.Now(),
|
||||
Exports: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
vars := execCtx.GetVariables()
|
||||
var outputs []interface{}
|
||||
var err error
|
||||
|
||||
// Determine execution mode
|
||||
if len(step.ParallelFunctions) > 0 {
|
||||
outputs, err = e.executeParallel(ctx, step.ParallelFunctions, vars)
|
||||
} else if len(step.Functions) > 0 {
|
||||
outputs, err = e.executeSequential(ctx, step.Functions, vars)
|
||||
} else if step.Function != "" {
|
||||
var output interface{}
|
||||
output, err = e.executeFunction(ctx, step.Function, vars)
|
||||
outputs = []interface{}{output}
|
||||
} else {
|
||||
err = fmt.Errorf("no function specified")
|
||||
}
|
||||
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
|
||||
if err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Convert outputs to string
|
||||
if len(outputs) > 0 {
|
||||
result.Output = fmt.Sprintf("%v", outputs[0])
|
||||
}
|
||||
|
||||
result.Status = core.StepStatusSuccess
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// executeFunction executes a single function
|
||||
func (e *FunctionExecutor) executeFunction(ctx context.Context, expr string, vars map[string]interface{}) (interface{}, error) {
|
||||
return e.functionRegistry.Execute(expr, vars)
|
||||
}
|
||||
|
||||
// executeSequential executes functions sequentially
|
||||
func (e *FunctionExecutor) executeSequential(ctx context.Context, funcs []string, vars map[string]interface{}) ([]interface{}, error) {
|
||||
var outputs []interface{}
|
||||
|
||||
for _, fn := range funcs {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return outputs, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
output, err := e.executeFunction(ctx, fn, vars)
|
||||
if err != nil {
|
||||
return outputs, err
|
||||
}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
// executeParallel executes functions in parallel
|
||||
func (e *FunctionExecutor) executeParallel(ctx context.Context, funcs []string, vars map[string]interface{}) ([]interface{}, error) {
|
||||
type result struct {
|
||||
index int
|
||||
output interface{}
|
||||
err error
|
||||
}
|
||||
|
||||
results := make(chan result, len(funcs))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i, fn := range funcs {
|
||||
wg.Add(1)
|
||||
go func(idx int, expr string) {
|
||||
defer wg.Done()
|
||||
output, err := e.executeFunction(ctx, expr, vars)
|
||||
results <- result{index: idx, output: output, err: err}
|
||||
}(i, fn)
|
||||
}
|
||||
|
||||
// Wait for all functions to complete
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(results)
|
||||
}()
|
||||
|
||||
// Collect results
|
||||
outputs := make([]interface{}, len(funcs))
|
||||
var firstError error
|
||||
|
||||
for r := range results {
|
||||
outputs[r.index] = r.output
|
||||
if r.err != nil && firstError == nil {
|
||||
firstError = r.err
|
||||
}
|
||||
}
|
||||
|
||||
return outputs, firstError
|
||||
}
|
||||
|
||||
// CanHandle returns true if this executor can handle the given step type
|
||||
func (e *FunctionExecutor) CanHandle(stepType core.StepType) bool {
|
||||
return stepType == core.StepTypeFunction
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/retry"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/template"
|
||||
)
|
||||
|
||||
// HTTPExecutor executes HTTP steps
|
||||
type HTTPExecutor struct {
|
||||
templateEngine *template.Engine
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewHTTPExecutor creates a new HTTP executor with pooled connections
|
||||
func NewHTTPExecutor(engine *template.Engine) *HTTPExecutor {
|
||||
transport := &http.Transport{
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
}
|
||||
return &HTTPExecutor{
|
||||
templateEngine: engine,
|
||||
client: &http.Client{
|
||||
Transport: transport,
|
||||
// No global timeout - we use per-request context timeout
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the executor name for logging/debugging
|
||||
func (e *HTTPExecutor) Name() string {
|
||||
return "http"
|
||||
}
|
||||
|
||||
// StepTypes returns the step types this executor handles
|
||||
func (e *HTTPExecutor) StepTypes() []core.StepType {
|
||||
return []core.StepType{core.StepTypeHTTP}
|
||||
}
|
||||
|
||||
// sanitizeStepName converts step name to a valid variable name
|
||||
// Converts hyphens to underscores
|
||||
func sanitizeStepName(name string) string {
|
||||
return strings.ReplaceAll(name, "-", "_")
|
||||
}
|
||||
|
||||
// Execute executes an HTTP step
|
||||
func (e *HTTPExecutor) Execute(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) {
|
||||
result := &core.StepResult{
|
||||
StepName: step.Name,
|
||||
Status: core.StepStatusRunning,
|
||||
StartTime: time.Now(),
|
||||
Exports: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Validate required fields
|
||||
if step.URL == "" {
|
||||
err := fmt.Errorf("HTTP step '%s' requires 'url' field", step.Name)
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
stepTimeout, err := step.Timeout.Duration()
|
||||
if err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Default method to GET
|
||||
method := step.Method
|
||||
if method == "" {
|
||||
method = "GET"
|
||||
}
|
||||
method = strings.ToUpper(method)
|
||||
|
||||
// Build HTTP response structure
|
||||
httpResp := map[string]interface{}{
|
||||
"status_code": 0,
|
||||
"error": nil,
|
||||
"message": "",
|
||||
"response_headers": map[string]string{},
|
||||
"response_body": "",
|
||||
"content_length": 0,
|
||||
"response_time_ms": 0,
|
||||
}
|
||||
|
||||
// Create request
|
||||
var reqBody io.Reader
|
||||
if step.RequestBody != "" {
|
||||
reqBody = strings.NewReader(step.RequestBody)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, step.URL, reqBody)
|
||||
if err != nil {
|
||||
httpResp["error"] = err.Error()
|
||||
httpResp["message"] = "failed to create request"
|
||||
e.exportHTTPResponse(result, step.Name, httpResp)
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Add headers
|
||||
for key, value := range step.Headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
// Set timeout via context (allows connection reuse via shared client)
|
||||
timeout := 30 * time.Second
|
||||
if stepTimeout > 0 {
|
||||
timeout = stepTimeout
|
||||
}
|
||||
|
||||
// Create request context with timeout
|
||||
reqCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
// Update request with timeout context
|
||||
req = req.WithContext(reqCtx)
|
||||
|
||||
// Execute request with retry for transient errors
|
||||
startTime := time.Now()
|
||||
var resp *http.Response
|
||||
err = retry.Do(reqCtx, retry.Config{
|
||||
MaxAttempts: 3,
|
||||
InitialDelay: 200 * time.Millisecond,
|
||||
MaxDelay: 2 * time.Second,
|
||||
Multiplier: 2.0,
|
||||
}, func() error {
|
||||
var reqErr error
|
||||
resp, reqErr = e.client.Do(req)
|
||||
if reqErr != nil {
|
||||
// Network errors are retryable
|
||||
return retry.Retryable(reqErr)
|
||||
}
|
||||
// Retry on 5xx server errors
|
||||
if resp.StatusCode >= 500 {
|
||||
_ = resp.Body.Close()
|
||||
return retry.Retryable(fmt.Errorf("server error: %d", resp.StatusCode))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
responseTimeMs := time.Since(startTime).Milliseconds()
|
||||
httpResp["response_time_ms"] = responseTimeMs
|
||||
|
||||
if err != nil {
|
||||
httpResp["error"] = err.Error()
|
||||
httpResp["message"] = "request failed"
|
||||
e.exportHTTPResponse(result, step.Name, httpResp)
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
// Read response body
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
httpResp["status_code"] = resp.StatusCode
|
||||
httpResp["error"] = err.Error()
|
||||
httpResp["message"] = "failed to read response body"
|
||||
e.exportHTTPResponse(result, step.Name, httpResp)
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Build response headers map
|
||||
respHeaders := make(map[string]string)
|
||||
for key, values := range resp.Header {
|
||||
respHeaders[strings.ToLower(key)] = strings.Join(values, ", ")
|
||||
}
|
||||
|
||||
// Populate successful response
|
||||
httpResp["status_code"] = resp.StatusCode
|
||||
httpResp["error"] = nil
|
||||
httpResp["message"] = "success"
|
||||
httpResp["response_headers"] = respHeaders
|
||||
httpResp["response_body"] = string(body)
|
||||
httpResp["content_length"] = len(body)
|
||||
|
||||
// Export the response
|
||||
e.exportHTTPResponse(result, step.Name, httpResp)
|
||||
|
||||
// Set output to response body for logging/display
|
||||
result.Output = string(body)
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
result.Status = core.StepStatusSuccess
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// exportHTTPResponse exports the HTTP response to the result's exports map
|
||||
// Export key is: <sanitized_step_name>_http_resp
|
||||
func (e *HTTPExecutor) exportHTTPResponse(result *core.StepResult, stepName string, httpResp map[string]interface{}) {
|
||||
exportKey := sanitizeStepName(stepName) + "_http_resp"
|
||||
result.Exports[exportKey] = httpResp
|
||||
}
|
||||
|
||||
// CanHandle returns true if this executor can handle the given step type
|
||||
func (e *HTTPExecutor) CanHandle(stepType core.StepType) bool {
|
||||
return stepType == core.StepTypeHTTP
|
||||
}
|
||||
@@ -0,0 +1,860 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/charmbracelet/glamour"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/config"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/logger"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/template"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// LLMExecutor executes LLM steps
|
||||
type LLMExecutor struct {
|
||||
templateEngine *template.Engine
|
||||
client *http.Client
|
||||
config *config.Config
|
||||
silent bool
|
||||
}
|
||||
|
||||
// NewLLMExecutor creates a new LLM executor
|
||||
func NewLLMExecutor(engine *template.Engine) *LLMExecutor {
|
||||
return &LLMExecutor{
|
||||
templateEngine: engine,
|
||||
client: &http.Client{
|
||||
Timeout: 120 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the executor name for logging/debugging
|
||||
func (e *LLMExecutor) Name() string {
|
||||
return "llm"
|
||||
}
|
||||
|
||||
// StepTypes returns the step types this executor handles
|
||||
func (e *LLMExecutor) StepTypes() []core.StepType {
|
||||
return []core.StepType{core.StepTypeLLM}
|
||||
}
|
||||
|
||||
// SetConfig sets the application config for LLM settings
|
||||
func (e *LLMExecutor) SetConfig(cfg *config.Config) {
|
||||
e.config = cfg
|
||||
}
|
||||
|
||||
// SetSilent enables or disables silent mode (suppresses output)
|
||||
func (e *LLMExecutor) SetSilent(s bool) {
|
||||
e.silent = s
|
||||
}
|
||||
|
||||
// CanHandle returns true if this executor can handle the given step type
|
||||
func (e *LLMExecutor) CanHandle(stepType core.StepType) bool {
|
||||
return stepType == core.StepTypeLLM
|
||||
}
|
||||
|
||||
// MergedLLMConfig holds the final merged configuration
|
||||
type MergedLLMConfig struct {
|
||||
Model string
|
||||
MaxTokens int
|
||||
Temperature float64
|
||||
TopK int
|
||||
TopP float64
|
||||
N int
|
||||
Timeout string
|
||||
MaxRetries int
|
||||
Stream bool
|
||||
ResponseFormat *core.LLMResponseFormat
|
||||
CustomHeaders map[string]string
|
||||
SystemPrompt string
|
||||
}
|
||||
|
||||
// ChatCompletionRequest is the OpenAI-compatible request format
|
||||
type ChatCompletionRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []ChatMessage `json:"messages"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
Temperature float64 `json:"temperature,omitempty"`
|
||||
TopP float64 `json:"top_p,omitempty"`
|
||||
TopK int `json:"top_k,omitempty"`
|
||||
N int `json:"n,omitempty"`
|
||||
Stream bool `json:"stream,omitempty"`
|
||||
Tools []core.LLMTool `json:"tools,omitempty"`
|
||||
ToolChoice interface{} `json:"tool_choice,omitempty"`
|
||||
ResponseFormat *core.LLMResponseFormat `json:"response_format,omitempty"`
|
||||
}
|
||||
|
||||
// ChatMessage is the wire format for messages
|
||||
type ChatMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content interface{} `json:"content"` // string or []ContentPart
|
||||
Name string `json:"name,omitempty"`
|
||||
ToolCallID string `json:"tool_call_id,omitempty"`
|
||||
ToolCalls []core.LLMToolCall `json:"tool_calls,omitempty"`
|
||||
}
|
||||
|
||||
// ChatCompletionResponse is the OpenAI-compatible response format
|
||||
type ChatCompletionResponse struct {
|
||||
ID string `json:"id"`
|
||||
Object string `json:"object"`
|
||||
Created int64 `json:"created"`
|
||||
Model string `json:"model"`
|
||||
Choices []ChatChoice `json:"choices"`
|
||||
Usage ChatUsage `json:"usage"`
|
||||
Error *ChatError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ChatChoice represents a single choice in the response
|
||||
type ChatChoice struct {
|
||||
Index int `json:"index"`
|
||||
Message ChatMessage `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
}
|
||||
|
||||
// ChatUsage represents token usage in the response
|
||||
type ChatUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
// ChatError represents an error in the API response
|
||||
type ChatError struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
Code string `json:"code"`
|
||||
}
|
||||
|
||||
// EmbeddingRequest represents a request for embeddings
|
||||
type EmbeddingRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input []string `json:"input"`
|
||||
EncodingFormat string `json:"encoding_format,omitempty"`
|
||||
}
|
||||
|
||||
// EmbeddingResponse represents the response from embeddings API
|
||||
type EmbeddingResponse struct {
|
||||
Object string `json:"object"`
|
||||
Data []EmbeddingData `json:"data"`
|
||||
Model string `json:"model"`
|
||||
Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
} `json:"usage"`
|
||||
Error *ChatError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// EmbeddingData represents a single embedding in the response
|
||||
type EmbeddingData struct {
|
||||
Object string `json:"object"`
|
||||
Embedding []float64 `json:"embedding"`
|
||||
Index int `json:"index"`
|
||||
}
|
||||
|
||||
// Execute executes an LLM step
|
||||
func (e *LLMExecutor) Execute(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) {
|
||||
log := logger.Get()
|
||||
result := &core.StepResult{
|
||||
StepName: step.Name,
|
||||
Status: core.StepStatusRunning,
|
||||
StartTime: time.Now(),
|
||||
Exports: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Validate config is set
|
||||
if e.config == nil {
|
||||
err := fmt.Errorf("LLM executor config not set")
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Get merged LLM configuration
|
||||
llmConfig := e.getMergedConfig(step)
|
||||
|
||||
// Validate required fields
|
||||
if len(step.Messages) == 0 && len(step.EmbeddingInput) == 0 {
|
||||
err := fmt.Errorf("LLM step '%s' requires 'messages' or 'embedding_input' field", step.Name)
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
log.Debug("Executing LLM step",
|
||||
zap.String("step", step.Name),
|
||||
zap.Bool("is_embedding", step.IsEmbedding),
|
||||
zap.Int("messages_count", len(step.Messages)),
|
||||
)
|
||||
|
||||
// Handle embedding vs chat completion
|
||||
if step.IsEmbedding || len(step.EmbeddingInput) > 0 {
|
||||
return e.executeEmbedding(ctx, step, execCtx, result, llmConfig)
|
||||
}
|
||||
|
||||
return e.executeChatCompletion(ctx, step, execCtx, result, llmConfig)
|
||||
}
|
||||
|
||||
// executeChatCompletion executes a chat completion request with provider rotation
|
||||
func (e *LLMExecutor) executeChatCompletion(
|
||||
ctx context.Context,
|
||||
step *core.Step,
|
||||
execCtx *core.ExecutionContext,
|
||||
result *core.StepResult,
|
||||
llmConfig *MergedLLMConfig,
|
||||
) (*core.StepResult, error) {
|
||||
log := logger.Get()
|
||||
|
||||
// Build request
|
||||
request, err := e.buildChatRequest(step, llmConfig)
|
||||
if err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Execute with retry and provider rotation
|
||||
var response *ChatCompletionResponse
|
||||
var lastErr error
|
||||
|
||||
maxRetries := llmConfig.MaxRetries
|
||||
if maxRetries <= 0 {
|
||||
maxRetries = 3
|
||||
}
|
||||
providerCount := e.config.LLM.GetProviderCount()
|
||||
if providerCount == 0 {
|
||||
err := fmt.Errorf("no LLM providers configured")
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
totalAttempts := maxRetries * providerCount
|
||||
|
||||
for attempt := 0; attempt < totalAttempts; attempt++ {
|
||||
provider := e.config.LLM.GetCurrentProvider()
|
||||
if provider == nil {
|
||||
lastErr = fmt.Errorf("no LLM providers available")
|
||||
break
|
||||
}
|
||||
|
||||
// Update model from provider if not overridden
|
||||
if llmConfig.Model == "" {
|
||||
request.Model = provider.Model
|
||||
}
|
||||
|
||||
log.Debug("Attempting LLM request",
|
||||
zap.String("provider", provider.Provider),
|
||||
zap.String("model", request.Model),
|
||||
zap.Int("attempt", attempt+1),
|
||||
zap.Int("max_attempts", totalAttempts),
|
||||
)
|
||||
|
||||
response, lastErr = e.sendChatRequest(ctx, provider, request, llmConfig)
|
||||
|
||||
if lastErr == nil && response.Error == nil {
|
||||
break // Success
|
||||
}
|
||||
|
||||
// Check if we should rotate provider
|
||||
if isProviderError(lastErr) || isRateLimitError(response) {
|
||||
log.Warn("Provider error, rotating",
|
||||
zap.String("provider", provider.Provider),
|
||||
zap.Error(lastErr),
|
||||
)
|
||||
e.config.LLM.RotateProvider()
|
||||
}
|
||||
|
||||
// Small backoff before retry
|
||||
if attempt < totalAttempts-1 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = ctx.Err()
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, ctx.Err()
|
||||
case <-time.After(time.Duration(attempt+1) * 500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = lastErr
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, lastErr
|
||||
}
|
||||
|
||||
if response != nil && response.Error != nil {
|
||||
err := fmt.Errorf("LLM API error: %s (%s)", response.Error.Message, response.Error.Type)
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Process response and exports
|
||||
e.processChatResponse(result, step.Name, response)
|
||||
|
||||
result.Status = core.StepStatusSuccess
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// executeEmbedding executes an embedding request
|
||||
func (e *LLMExecutor) executeEmbedding(
|
||||
ctx context.Context,
|
||||
step *core.Step,
|
||||
execCtx *core.ExecutionContext,
|
||||
result *core.StepResult,
|
||||
llmConfig *MergedLLMConfig,
|
||||
) (*core.StepResult, error) {
|
||||
log := logger.Get()
|
||||
|
||||
if len(step.EmbeddingInput) == 0 {
|
||||
err := fmt.Errorf("embedding step '%s' requires 'embedding_input' field", step.Name)
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Build embedding request
|
||||
request := &EmbeddingRequest{
|
||||
Model: llmConfig.Model,
|
||||
Input: step.EmbeddingInput,
|
||||
}
|
||||
|
||||
// Execute with retry and provider rotation
|
||||
var response *EmbeddingResponse
|
||||
var lastErr error
|
||||
|
||||
maxRetries := llmConfig.MaxRetries
|
||||
if maxRetries <= 0 {
|
||||
maxRetries = 3
|
||||
}
|
||||
providerCount := e.config.LLM.GetProviderCount()
|
||||
if providerCount == 0 {
|
||||
err := fmt.Errorf("no LLM providers configured")
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
totalAttempts := maxRetries * providerCount
|
||||
|
||||
for attempt := 0; attempt < totalAttempts; attempt++ {
|
||||
provider := e.config.LLM.GetCurrentProvider()
|
||||
if provider == nil {
|
||||
lastErr = fmt.Errorf("no LLM providers available")
|
||||
break
|
||||
}
|
||||
|
||||
// Update model from provider if not overridden
|
||||
if request.Model == "" {
|
||||
request.Model = provider.Model
|
||||
}
|
||||
|
||||
log.Debug("Attempting embedding request",
|
||||
zap.String("provider", provider.Provider),
|
||||
zap.String("model", request.Model),
|
||||
zap.Int("attempt", attempt+1),
|
||||
)
|
||||
|
||||
response, lastErr = e.sendEmbeddingRequest(ctx, provider, request, llmConfig)
|
||||
|
||||
if lastErr == nil && response.Error == nil {
|
||||
break // Success
|
||||
}
|
||||
|
||||
// Check if we should rotate provider
|
||||
if isProviderError(lastErr) || (response != nil && response.Error != nil) {
|
||||
log.Warn("Provider error, rotating",
|
||||
zap.String("provider", provider.Provider),
|
||||
zap.Error(lastErr),
|
||||
)
|
||||
e.config.LLM.RotateProvider()
|
||||
}
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = lastErr
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, lastErr
|
||||
}
|
||||
|
||||
if response != nil && response.Error != nil {
|
||||
err := fmt.Errorf("embedding API error: %s (%s)", response.Error.Message, response.Error.Type)
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Process embedding response
|
||||
e.processEmbeddingResponse(result, step.Name, response)
|
||||
|
||||
result.Status = core.StepStatusSuccess
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// buildChatRequest builds an OpenAI-compatible chat request
|
||||
func (e *LLMExecutor) buildChatRequest(step *core.Step, llmConfig *MergedLLMConfig) (*ChatCompletionRequest, error) {
|
||||
request := &ChatCompletionRequest{
|
||||
Model: llmConfig.Model,
|
||||
MaxTokens: llmConfig.MaxTokens,
|
||||
Temperature: llmConfig.Temperature,
|
||||
TopP: llmConfig.TopP,
|
||||
TopK: llmConfig.TopK,
|
||||
N: llmConfig.N,
|
||||
Stream: llmConfig.Stream,
|
||||
}
|
||||
|
||||
// Convert messages
|
||||
messages := make([]ChatMessage, 0, len(step.Messages)+1)
|
||||
|
||||
// Auto-prepend system prompt if global one exists and step doesn't have one
|
||||
if llmConfig.SystemPrompt != "" {
|
||||
hasSystemMessage := false
|
||||
for _, msg := range step.Messages {
|
||||
if msg.Role == core.LLMRoleSystem {
|
||||
hasSystemMessage = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasSystemMessage {
|
||||
messages = append(messages, ChatMessage{
|
||||
Role: string(core.LLMRoleSystem),
|
||||
Content: llmConfig.SystemPrompt,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Add step messages
|
||||
for _, msg := range step.Messages {
|
||||
chatMsg := ChatMessage{
|
||||
Role: string(msg.Role),
|
||||
Content: msg.Content,
|
||||
Name: msg.Name,
|
||||
ToolCallID: msg.ToolCallID,
|
||||
ToolCalls: msg.ToolCalls,
|
||||
}
|
||||
messages = append(messages, chatMsg)
|
||||
}
|
||||
|
||||
request.Messages = messages
|
||||
|
||||
// Add tools if specified
|
||||
if len(step.Tools) > 0 {
|
||||
request.Tools = step.Tools
|
||||
}
|
||||
|
||||
// Add tool choice if specified
|
||||
if step.ToolChoice != nil {
|
||||
request.ToolChoice = step.ToolChoice
|
||||
}
|
||||
|
||||
// Add response format if specified
|
||||
if llmConfig.ResponseFormat != nil {
|
||||
request.ResponseFormat = llmConfig.ResponseFormat
|
||||
}
|
||||
|
||||
return request, nil
|
||||
}
|
||||
|
||||
// sendChatRequest sends an HTTP request to the LLM provider
|
||||
func (e *LLMExecutor) sendChatRequest(
|
||||
ctx context.Context,
|
||||
provider *config.LLMProvider,
|
||||
request *ChatCompletionRequest,
|
||||
llmConfig *MergedLLMConfig,
|
||||
) (*ChatCompletionResponse, error) {
|
||||
// Marshal request to JSON
|
||||
body, err := json.Marshal(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
// Create HTTP request
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", provider.BaseURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
// Set headers
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
if provider.AuthToken != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+provider.AuthToken)
|
||||
}
|
||||
|
||||
// Add custom headers
|
||||
for key, value := range llmConfig.CustomHeaders {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
// Set timeout
|
||||
timeout, err := time.ParseDuration(llmConfig.Timeout)
|
||||
if err != nil {
|
||||
timeout = 120 * time.Second
|
||||
}
|
||||
client := &http.Client{Timeout: timeout}
|
||||
|
||||
// Execute request
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
// Read response
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
// Parse response
|
||||
var response ChatCompletionResponse
|
||||
if err := json.Unmarshal(respBody, &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse response: %w (body: %s)", err, string(respBody))
|
||||
}
|
||||
|
||||
// Check for HTTP errors
|
||||
if resp.StatusCode >= 400 {
|
||||
if response.Error != nil {
|
||||
return &response, fmt.Errorf("HTTP %d: %s", resp.StatusCode, response.Error.Message)
|
||||
}
|
||||
return &response, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
return &response, nil
|
||||
}
|
||||
|
||||
// sendEmbeddingRequest sends an embedding request to the LLM provider
|
||||
func (e *LLMExecutor) sendEmbeddingRequest(
|
||||
ctx context.Context,
|
||||
provider *config.LLMProvider,
|
||||
request *EmbeddingRequest,
|
||||
llmConfig *MergedLLMConfig,
|
||||
) (*EmbeddingResponse, error) {
|
||||
// Marshal request to JSON
|
||||
body, err := json.Marshal(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
// Determine embedding endpoint - typically /v1/embeddings
|
||||
embeddingURL := provider.BaseURL
|
||||
if strings.HasSuffix(embeddingURL, "/chat/completions") {
|
||||
embeddingURL = strings.Replace(embeddingURL, "/chat/completions", "/embeddings", 1)
|
||||
}
|
||||
|
||||
// Create HTTP request
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", embeddingURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
// Set headers
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
if provider.AuthToken != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+provider.AuthToken)
|
||||
}
|
||||
|
||||
// Add custom headers
|
||||
for key, value := range llmConfig.CustomHeaders {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
// Set timeout
|
||||
timeout, err := time.ParseDuration(llmConfig.Timeout)
|
||||
if err != nil {
|
||||
timeout = 120 * time.Second
|
||||
}
|
||||
client := &http.Client{Timeout: timeout}
|
||||
|
||||
// Execute request
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
// Read response
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
// Parse response
|
||||
var response EmbeddingResponse
|
||||
if err := json.Unmarshal(respBody, &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse response: %w (body: %s)", err, string(respBody))
|
||||
}
|
||||
|
||||
// Check for HTTP errors
|
||||
if resp.StatusCode >= 400 {
|
||||
if response.Error != nil {
|
||||
return &response, fmt.Errorf("HTTP %d: %s", resp.StatusCode, response.Error.Message)
|
||||
}
|
||||
return &response, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
return &response, nil
|
||||
}
|
||||
|
||||
// printLLMOutput prints LLM response with glamour markdown rendering
|
||||
func printLLMOutput(content string) {
|
||||
// Render with glamour for markdown highlighting
|
||||
renderer, err := glamour.NewTermRenderer(
|
||||
glamour.WithAutoStyle(),
|
||||
glamour.WithWordWrap(120),
|
||||
)
|
||||
|
||||
var rendered string
|
||||
if err == nil {
|
||||
if out, renderErr := renderer.Render(content); renderErr == nil {
|
||||
rendered = out
|
||||
} else {
|
||||
rendered = content + "\n"
|
||||
}
|
||||
} else {
|
||||
rendered = content + "\n"
|
||||
}
|
||||
|
||||
fmt.Print(rendered)
|
||||
}
|
||||
|
||||
// processChatResponse exports the LLM response to step result
|
||||
func (e *LLMExecutor) processChatResponse(result *core.StepResult, stepName string, response *ChatCompletionResponse) {
|
||||
exportKey := sanitizeStepName(stepName) + "_llm_resp"
|
||||
|
||||
// Build comprehensive export structure
|
||||
llmResp := map[string]interface{}{
|
||||
"id": response.ID,
|
||||
"model": response.Model,
|
||||
"created": response.Created,
|
||||
"usage": map[string]interface{}{
|
||||
"prompt_tokens": response.Usage.PromptTokens,
|
||||
"completion_tokens": response.Usage.CompletionTokens,
|
||||
"total_tokens": response.Usage.TotalTokens,
|
||||
},
|
||||
}
|
||||
|
||||
// Export choices
|
||||
if len(response.Choices) > 0 {
|
||||
choice := response.Choices[0]
|
||||
llmResp["content"] = choice.Message.Content
|
||||
llmResp["finish_reason"] = choice.FinishReason
|
||||
llmResp["role"] = choice.Message.Role
|
||||
|
||||
// Export tool calls if present
|
||||
if len(choice.Message.ToolCalls) > 0 {
|
||||
llmResp["tool_calls"] = choice.Message.ToolCalls
|
||||
}
|
||||
|
||||
// Set output to content for display
|
||||
if content, ok := choice.Message.Content.(string); ok {
|
||||
result.Output = content
|
||||
// Print LLM output with symbol prefix and markdown formatting (skip in silent mode)
|
||||
if !e.silent {
|
||||
printLLMOutput(content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// All choices for n > 1
|
||||
if len(response.Choices) > 1 {
|
||||
allContents := make([]interface{}, len(response.Choices))
|
||||
for i, c := range response.Choices {
|
||||
allContents[i] = c.Message.Content
|
||||
}
|
||||
llmResp["all_contents"] = allContents
|
||||
}
|
||||
|
||||
result.Exports[exportKey] = llmResp
|
||||
|
||||
// Also export content directly for easy access
|
||||
if len(response.Choices) > 0 {
|
||||
contentKey := sanitizeStepName(stepName) + "_content"
|
||||
result.Exports[contentKey] = response.Choices[0].Message.Content
|
||||
}
|
||||
}
|
||||
|
||||
// processEmbeddingResponse exports the embedding response to step result
|
||||
func (e *LLMExecutor) processEmbeddingResponse(result *core.StepResult, stepName string, response *EmbeddingResponse) {
|
||||
exportKey := sanitizeStepName(stepName) + "_llm_resp"
|
||||
|
||||
// Build export structure
|
||||
llmResp := map[string]interface{}{
|
||||
"model": response.Model,
|
||||
"usage": map[string]interface{}{
|
||||
"prompt_tokens": response.Usage.PromptTokens,
|
||||
"total_tokens": response.Usage.TotalTokens,
|
||||
},
|
||||
}
|
||||
|
||||
// Export embeddings
|
||||
if len(response.Data) > 0 {
|
||||
embeddings := make([][]float64, len(response.Data))
|
||||
for i, d := range response.Data {
|
||||
embeddings[i] = d.Embedding
|
||||
}
|
||||
llmResp["embeddings"] = embeddings
|
||||
|
||||
// Set output to summary
|
||||
result.Output = fmt.Sprintf("Generated %d embeddings", len(embeddings))
|
||||
}
|
||||
|
||||
result.Exports[exportKey] = llmResp
|
||||
}
|
||||
|
||||
// getMergedConfig merges global llm_config with step-level overrides
|
||||
func (e *LLMExecutor) getMergedConfig(step *core.Step) *MergedLLMConfig {
|
||||
globalLLM := &e.config.LLM
|
||||
|
||||
merged := &MergedLLMConfig{
|
||||
MaxTokens: globalLLM.MaxTokens,
|
||||
Temperature: globalLLM.Temperature,
|
||||
TopK: globalLLM.TopK,
|
||||
TopP: globalLLM.TopP,
|
||||
N: globalLLM.N,
|
||||
Timeout: globalLLM.Timeout,
|
||||
MaxRetries: globalLLM.MaxRetries,
|
||||
Stream: globalLLM.Stream,
|
||||
SystemPrompt: globalLLM.SystemPrompt,
|
||||
CustomHeaders: make(map[string]string),
|
||||
}
|
||||
|
||||
// Set default response format if structured JSON is enabled globally
|
||||
if globalLLM.StructuredJSONFormat {
|
||||
merged.ResponseFormat = &core.LLMResponseFormat{
|
||||
Type: "json_object",
|
||||
}
|
||||
}
|
||||
|
||||
// Parse global custom headers (format: "Key1: Value1, Key2: Value2")
|
||||
if globalLLM.CustomHeaders != "" {
|
||||
for _, h := range strings.Split(globalLLM.CustomHeaders, ",") {
|
||||
if parts := strings.SplitN(strings.TrimSpace(h), ":", 2); len(parts) == 2 {
|
||||
merged.CustomHeaders[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply step-level overrides
|
||||
if step.LLMConfig != nil {
|
||||
cfg := step.LLMConfig
|
||||
|
||||
if cfg.Model != "" {
|
||||
merged.Model = cfg.Model
|
||||
}
|
||||
if cfg.MaxTokens != nil {
|
||||
merged.MaxTokens = *cfg.MaxTokens
|
||||
}
|
||||
if cfg.Temperature != nil {
|
||||
merged.Temperature = *cfg.Temperature
|
||||
}
|
||||
if cfg.TopK != nil {
|
||||
merged.TopK = *cfg.TopK
|
||||
}
|
||||
if cfg.TopP != nil {
|
||||
merged.TopP = *cfg.TopP
|
||||
}
|
||||
if cfg.N != nil {
|
||||
merged.N = *cfg.N
|
||||
}
|
||||
if cfg.Timeout != "" {
|
||||
merged.Timeout = cfg.Timeout
|
||||
}
|
||||
if cfg.MaxRetries != nil {
|
||||
merged.MaxRetries = *cfg.MaxRetries
|
||||
}
|
||||
if cfg.Stream != nil {
|
||||
merged.Stream = *cfg.Stream
|
||||
}
|
||||
if cfg.ResponseFormat != nil {
|
||||
merged.ResponseFormat = cfg.ResponseFormat
|
||||
}
|
||||
|
||||
// Merge custom headers (step overrides global)
|
||||
for k, v := range cfg.CustomHeaders {
|
||||
merged.CustomHeaders[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
// Apply extra LLM parameters (these can override anything)
|
||||
if step.ExtraLLMParams != nil {
|
||||
if model, ok := step.ExtraLLMParams["model"].(string); ok {
|
||||
merged.Model = model
|
||||
}
|
||||
if maxTokens, ok := step.ExtraLLMParams["max_tokens"].(int); ok {
|
||||
merged.MaxTokens = maxTokens
|
||||
}
|
||||
if temp, ok := step.ExtraLLMParams["temperature"].(float64); ok {
|
||||
merged.Temperature = temp
|
||||
}
|
||||
if topK, ok := step.ExtraLLMParams["top_k"].(int); ok {
|
||||
merged.TopK = topK
|
||||
}
|
||||
if topP, ok := step.ExtraLLMParams["top_p"].(float64); ok {
|
||||
merged.TopP = topP
|
||||
}
|
||||
}
|
||||
|
||||
return merged
|
||||
}
|
||||
|
||||
// isProviderError checks if error indicates provider-level failure
|
||||
func isProviderError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
errStr := err.Error()
|
||||
return strings.Contains(errStr, "connection refused") ||
|
||||
strings.Contains(errStr, "no such host") ||
|
||||
strings.Contains(errStr, "timeout") ||
|
||||
strings.Contains(errStr, "EOF") ||
|
||||
strings.Contains(errStr, "i/o timeout")
|
||||
}
|
||||
|
||||
// isRateLimitError checks if response indicates rate limiting
|
||||
func isRateLimitError(resp *ChatCompletionResponse) bool {
|
||||
if resp == nil || resp.Error == nil {
|
||||
return false
|
||||
}
|
||||
return resp.Error.Type == "rate_limit_error" ||
|
||||
strings.Contains(resp.Error.Code, "rate_limit") ||
|
||||
strings.Contains(resp.Error.Message, "rate limit") ||
|
||||
strings.Contains(resp.Error.Message, "Rate limit")
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
)
|
||||
|
||||
// ParallelExecutor executes parallel steps
|
||||
type ParallelExecutor struct {
|
||||
dispatcher *StepDispatcher
|
||||
}
|
||||
|
||||
// NewParallelExecutor creates a new parallel executor
|
||||
func NewParallelExecutor(dispatcher *StepDispatcher) *ParallelExecutor {
|
||||
return &ParallelExecutor{
|
||||
dispatcher: dispatcher,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the executor name for logging/debugging
|
||||
func (e *ParallelExecutor) Name() string {
|
||||
return "parallel"
|
||||
}
|
||||
|
||||
// StepTypes returns the step types this executor handles
|
||||
func (e *ParallelExecutor) StepTypes() []core.StepType {
|
||||
return []core.StepType{core.StepTypeParallel}
|
||||
}
|
||||
|
||||
// Execute executes a parallel step
|
||||
func (e *ParallelExecutor) Execute(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) {
|
||||
result := &core.StepResult{
|
||||
StepName: step.Name,
|
||||
Status: core.StepStatusRunning,
|
||||
StartTime: time.Now(),
|
||||
Exports: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
if len(step.ParallelSteps) == 0 {
|
||||
result.Status = core.StepStatusSuccess
|
||||
result.EndTime = time.Now()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Check if context is already cancelled
|
||||
if ctx.Err() != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = ctx.Err()
|
||||
result.EndTime = time.Now()
|
||||
return result, ctx.Err()
|
||||
}
|
||||
|
||||
type stepResult struct {
|
||||
index int
|
||||
result *core.StepResult
|
||||
err error
|
||||
}
|
||||
|
||||
results := make(chan stepResult, len(step.ParallelSteps))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := range step.ParallelSteps {
|
||||
wg.Add(1)
|
||||
go func(idx int, s *core.Step) {
|
||||
defer wg.Done()
|
||||
|
||||
// Check if context is cancelled before starting
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
results <- stepResult{index: idx, err: ctx.Err()}
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
// Clone context for parallel execution
|
||||
childCtx := execCtx.Clone()
|
||||
r, err := e.dispatcher.Dispatch(ctx, s, childCtx)
|
||||
|
||||
// Send result (use select to handle cancelled context)
|
||||
select {
|
||||
case results <- stepResult{index: idx, result: r, err: err}:
|
||||
case <-ctx.Done():
|
||||
// Context cancelled, still need to send a result
|
||||
results <- stepResult{index: idx, result: r, err: ctx.Err()}
|
||||
}
|
||||
}(i, &step.ParallelSteps[i])
|
||||
}
|
||||
|
||||
// Wait for all steps to complete
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(results)
|
||||
}()
|
||||
|
||||
// Collect results with context awareness
|
||||
stepResults := make([]*core.StepResult, len(step.ParallelSteps))
|
||||
var outputs []string
|
||||
var firstError error
|
||||
collected := 0
|
||||
|
||||
for collected < len(step.ParallelSteps) {
|
||||
select {
|
||||
case r, ok := <-results:
|
||||
if !ok {
|
||||
// Channel closed
|
||||
goto done
|
||||
}
|
||||
collected++
|
||||
stepResults[r.index] = r.result
|
||||
if r.result != nil && r.result.Output != "" {
|
||||
outputs = append(outputs, r.result.Output)
|
||||
}
|
||||
if r.err != nil && firstError == nil {
|
||||
firstError = r.err
|
||||
}
|
||||
// Merge exports
|
||||
if r.result != nil && r.result.Exports != nil {
|
||||
for k, v := range r.result.Exports {
|
||||
result.Exports[k] = v
|
||||
}
|
||||
}
|
||||
case <-ctx.Done():
|
||||
// Context cancelled - set error and wait for remaining results
|
||||
if firstError == nil {
|
||||
firstError = ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
done:
|
||||
result.Output = strings.Join(outputs, "\n")
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
|
||||
if firstError != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = firstError
|
||||
return result, firstError
|
||||
}
|
||||
|
||||
// Check if any step failed
|
||||
for _, sr := range stepResults {
|
||||
if sr != nil && sr.Status == core.StepStatusFailed {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = fmt.Errorf("one or more parallel steps failed")
|
||||
return result, result.Error
|
||||
}
|
||||
}
|
||||
|
||||
result.Status = core.StepStatusSuccess
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CanHandle returns true if this executor can handle the given step type
|
||||
func (e *ParallelExecutor) CanHandle(stepType core.StepType) bool {
|
||||
return stepType == core.StepTypeParallel
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
)
|
||||
|
||||
// StepExecutorPlugin defines the interface for step type plugins.
|
||||
// Any executor that handles step types should implement this interface.
|
||||
type StepExecutorPlugin interface {
|
||||
// Name returns the plugin name for logging/debugging
|
||||
Name() string
|
||||
|
||||
// StepTypes returns the step types this plugin handles
|
||||
StepTypes() []core.StepType
|
||||
|
||||
// Execute runs the step and returns the result
|
||||
Execute(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error)
|
||||
}
|
||||
|
||||
// PluginRegistry manages registered step executor plugins.
|
||||
// It maps step types to their corresponding plugin implementations.
|
||||
type PluginRegistry struct {
|
||||
plugins map[core.StepType]StepExecutorPlugin
|
||||
}
|
||||
|
||||
// NewPluginRegistry creates a new plugin registry
|
||||
func NewPluginRegistry() *PluginRegistry {
|
||||
return &PluginRegistry{
|
||||
plugins: make(map[core.StepType]StepExecutorPlugin),
|
||||
}
|
||||
}
|
||||
|
||||
// Register adds a plugin to the registry.
|
||||
// The plugin will be registered for all step types it reports via StepTypes().
|
||||
func (r *PluginRegistry) Register(plugin StepExecutorPlugin) {
|
||||
for _, stepType := range plugin.StepTypes() {
|
||||
r.plugins[stepType] = plugin
|
||||
}
|
||||
}
|
||||
|
||||
// Get returns the plugin for a step type, or nil if not found
|
||||
func (r *PluginRegistry) Get(stepType core.StepType) (StepExecutorPlugin, bool) {
|
||||
plugin, ok := r.plugins[stepType]
|
||||
return plugin, ok
|
||||
}
|
||||
|
||||
// Has checks if a step type is registered
|
||||
func (r *PluginRegistry) Has(stepType core.StepType) bool {
|
||||
_, ok := r.plugins[stepType]
|
||||
return ok
|
||||
}
|
||||
|
||||
// ListStepTypes returns all registered step types
|
||||
func (r *PluginRegistry) ListStepTypes() []core.StepType {
|
||||
types := make([]core.StepType, 0, len(r.plugins))
|
||||
for t := range r.plugins {
|
||||
types = append(types, t)
|
||||
}
|
||||
return types
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/runner"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/template"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// RemoteBashExecutor executes remote-bash steps on Docker/SSH runners
|
||||
type RemoteBashExecutor struct {
|
||||
templateEngine *template.Engine
|
||||
}
|
||||
|
||||
// NewRemoteBashExecutor creates a new remote bash executor
|
||||
func NewRemoteBashExecutor(engine *template.Engine) *RemoteBashExecutor {
|
||||
return &RemoteBashExecutor{
|
||||
templateEngine: engine,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the executor name for logging/debugging
|
||||
func (e *RemoteBashExecutor) Name() string {
|
||||
return "remote-bash"
|
||||
}
|
||||
|
||||
// StepTypes returns the step types this executor handles
|
||||
func (e *RemoteBashExecutor) StepTypes() []core.StepType {
|
||||
return []core.StepType{core.StepTypeRemoteBash}
|
||||
}
|
||||
|
||||
// Execute executes a remote-bash step
|
||||
func (e *RemoteBashExecutor) Execute(ctx context.Context, step *core.Step, execCtx *core.ExecutionContext) (*core.StepResult, error) {
|
||||
result := &core.StepResult{
|
||||
StepName: step.Name,
|
||||
Status: core.StepStatusRunning,
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
|
||||
timeout, err := step.Timeout.Duration()
|
||||
if err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Validate step_runner is set for remote-bash
|
||||
if step.StepRunner == "" || step.StepRunner == core.RunnerTypeHost {
|
||||
err := fmt.Errorf("remote-bash step '%s' requires step_runner to be 'docker' or 'ssh'", step.Name)
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Create runner based on step_runner and step_runner_config
|
||||
r, err := e.createRunner(step.StepRunner, step.StepRunnerConfig)
|
||||
if err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Setup the runner (fresh connection for each step)
|
||||
if err := r.Setup(ctx); err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = fmt.Errorf("runner setup failed: %w", err)
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
return result, result.Error
|
||||
}
|
||||
|
||||
// Ensure cleanup happens
|
||||
defer func() {
|
||||
cleanupCtx := context.Background() // Use fresh context for cleanup
|
||||
_ = r.Cleanup(cleanupCtx)
|
||||
}()
|
||||
|
||||
// Execute command(s) using the runner
|
||||
var output string
|
||||
if len(step.ParallelCommands) > 0 {
|
||||
output, err = e.executeParallel(ctx, r, step.ParallelCommands, timeout)
|
||||
} else if len(step.Commands) > 0 {
|
||||
output, err = e.executeSequential(ctx, r, step.Commands, timeout)
|
||||
} else if step.Command != "" {
|
||||
// Assemble command with structured args if present
|
||||
finalCmd := assembleCommand(step.Command, step.SpeedArgs, step.ConfigArgs, step.InputArgs, step.OutputArgs)
|
||||
output, err = e.executeCommand(ctx, r, finalCmd, timeout)
|
||||
} else {
|
||||
err = fmt.Errorf("no command specified")
|
||||
}
|
||||
|
||||
result.Output = output
|
||||
result.EndTime = time.Now()
|
||||
result.Duration = result.EndTime.Sub(result.StartTime)
|
||||
|
||||
// Write stdout/stderr to file if std_file is specified
|
||||
if step.StdFile != "" {
|
||||
if writeErr := writeStdFile(step.StdFile, output); writeErr != nil {
|
||||
// Log warning but don't fail the step
|
||||
execCtx.Logger.Warn("Failed to write std_file",
|
||||
zap.String("path", step.StdFile),
|
||||
zap.Error(writeErr))
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
result.Status = core.StepStatusFailed
|
||||
result.Error = err
|
||||
return result, err
|
||||
}
|
||||
|
||||
result.Status = core.StepStatusSuccess
|
||||
|
||||
// Copy remote file to host if specified (before cleanup)
|
||||
if step.StepRemoteFile != "" && step.HostOutputFile != "" {
|
||||
if copyErr := r.CopyFromRemote(ctx, step.StepRemoteFile, step.HostOutputFile); copyErr != nil {
|
||||
// Log warning but don't fail the step
|
||||
execCtx.Logger.Warn("Failed to copy remote file",
|
||||
zap.String("remote", step.StepRemoteFile),
|
||||
zap.String("local", step.HostOutputFile),
|
||||
zap.Error(copyErr))
|
||||
} else {
|
||||
execCtx.Logger.Debug("Copied remote file to host",
|
||||
zap.String("remote", step.StepRemoteFile),
|
||||
zap.String("local", step.HostOutputFile))
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// createRunner creates a runner based on step_runner type and step_runner_config
|
||||
func (e *RemoteBashExecutor) createRunner(runnerType core.RunnerType, cfg *core.StepRunnerConfig) (runner.Runner, error) {
|
||||
// Get the embedded RunnerConfig (or create empty one)
|
||||
runnerCfg := &core.RunnerConfig{}
|
||||
if cfg != nil && cfg.RunnerConfig != nil {
|
||||
runnerCfg = cfg.RunnerConfig
|
||||
}
|
||||
|
||||
// Pass empty string for binaryPath since we only execute shell commands,
|
||||
// not the osmedeus binary itself
|
||||
switch runnerType {
|
||||
case core.RunnerTypeDocker:
|
||||
return runner.NewDockerRunner(runnerCfg, "")
|
||||
case core.RunnerTypeSSH:
|
||||
return runner.NewSSHRunner(runnerCfg, "")
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported step_runner for remote-bash: %s (must be 'docker' or 'ssh')", runnerType)
|
||||
}
|
||||
}
|
||||
|
||||
// executeCommand executes a single command on the remote runner
|
||||
func (e *RemoteBashExecutor) executeCommand(ctx context.Context, r runner.Runner, command string, timeout time.Duration) (string, error) {
|
||||
if timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
cmdResult, err := r.Execute(ctx, command)
|
||||
if err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
output := ""
|
||||
if cmdResult != nil {
|
||||
output = cmdResult.Output
|
||||
}
|
||||
return output, fmt.Errorf("command timed out after %s", timeout)
|
||||
}
|
||||
output := ""
|
||||
if cmdResult != nil {
|
||||
output = cmdResult.Output
|
||||
}
|
||||
return output, fmt.Errorf("command failed: %w", err)
|
||||
}
|
||||
|
||||
if cmdResult.ExitCode != 0 {
|
||||
return cmdResult.Output, fmt.Errorf("command exited with code %d", cmdResult.ExitCode)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(cmdResult.Output), nil
|
||||
}
|
||||
|
||||
// executeSequential executes commands sequentially
|
||||
func (e *RemoteBashExecutor) executeSequential(ctx context.Context, r runner.Runner, commands []string, timeout time.Duration) (string, error) {
|
||||
var outputs []string
|
||||
|
||||
for _, cmd := range commands {
|
||||
output, err := e.executeCommand(ctx, r, cmd, timeout)
|
||||
outputs = append(outputs, output)
|
||||
if err != nil {
|
||||
return strings.Join(outputs, "\n"), err
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(outputs, "\n"), nil
|
||||
}
|
||||
|
||||
// executeParallel executes commands in parallel
|
||||
func (e *RemoteBashExecutor) executeParallel(ctx context.Context, r runner.Runner, commands []string, timeout time.Duration) (string, error) {
|
||||
type cmdResult struct {
|
||||
index int
|
||||
output string
|
||||
err error
|
||||
}
|
||||
|
||||
results := make(chan cmdResult, len(commands))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i, cmd := range commands {
|
||||
wg.Add(1)
|
||||
go func(idx int, command string) {
|
||||
defer wg.Done()
|
||||
output, err := e.executeCommand(ctx, r, command, timeout)
|
||||
results <- cmdResult{index: idx, output: output, err: err}
|
||||
}(i, cmd)
|
||||
}
|
||||
|
||||
// Wait for all commands to complete
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(results)
|
||||
}()
|
||||
|
||||
// Collect results in order
|
||||
outputs := make([]string, len(commands))
|
||||
var firstError error
|
||||
|
||||
for res := range results {
|
||||
outputs[res.index] = res.output
|
||||
if res.err != nil && firstError == nil {
|
||||
firstError = res.err
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(outputs, "\n"), firstError
|
||||
}
|
||||
|
||||
// CanHandle returns true if this executor can handle the given step type
|
||||
func (e *RemoteBashExecutor) CanHandle(stepType core.StepType) bool {
|
||||
return stepType == core.StepTypeRemoteBash
|
||||
}
|
||||
@@ -0,0 +1,375 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExecutor_RemoteBashStep_MissingStepRunner(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cfg := testConfig(t)
|
||||
|
||||
module := &core.Workflow{
|
||||
Name: "test-remote-bash-no-config",
|
||||
Kind: core.KindModule,
|
||||
Steps: []core.Step{
|
||||
{
|
||||
Name: "missing-config",
|
||||
Type: core.StepTypeRemoteBash,
|
||||
Command: "echo hello",
|
||||
// No StepRunner - should fail
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
executor := NewExecutor()
|
||||
executor.SetDryRun(false)
|
||||
executor.SetSpinner(false)
|
||||
|
||||
_, err := executor.ExecuteModule(ctx, module, map[string]string{
|
||||
"target": "test",
|
||||
}, cfg)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "step_runner")
|
||||
}
|
||||
|
||||
func TestExecutor_RemoteBashStep_HostRunnerNotSupported(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cfg := testConfig(t)
|
||||
|
||||
module := &core.Workflow{
|
||||
Name: "test-remote-bash-host",
|
||||
Kind: core.KindModule,
|
||||
Steps: []core.Step{
|
||||
{
|
||||
Name: "host-runner",
|
||||
Type: core.StepTypeRemoteBash,
|
||||
StepRunner: core.RunnerTypeHost, // Should fail - host not supported
|
||||
Command: "echo hello",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
executor := NewExecutor()
|
||||
executor.SetDryRun(false)
|
||||
executor.SetSpinner(false)
|
||||
|
||||
_, err := executor.ExecuteModule(ctx, module, map[string]string{
|
||||
"target": "test",
|
||||
}, cfg)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "docker")
|
||||
}
|
||||
|
||||
func TestExecutor_RemoteBashStep_DockerMissingImage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cfg := testConfig(t)
|
||||
|
||||
module := &core.Workflow{
|
||||
Name: "test-remote-bash-docker-no-image",
|
||||
Kind: core.KindModule,
|
||||
Steps: []core.Step{
|
||||
{
|
||||
Name: "docker-no-image",
|
||||
Type: core.StepTypeRemoteBash,
|
||||
StepRunner: core.RunnerTypeDocker,
|
||||
Command: "echo hello",
|
||||
StepRunnerConfig: &core.StepRunnerConfig{
|
||||
RunnerConfig: &core.RunnerConfig{
|
||||
// No Image - should fail
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
executor := NewExecutor()
|
||||
executor.SetDryRun(false)
|
||||
executor.SetSpinner(false)
|
||||
|
||||
_, err := executor.ExecuteModule(ctx, module, map[string]string{
|
||||
"target": "test",
|
||||
}, cfg)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "image")
|
||||
}
|
||||
|
||||
func TestExecutor_RemoteBashStep_SSHMissingHost(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cfg := testConfig(t)
|
||||
|
||||
module := &core.Workflow{
|
||||
Name: "test-remote-bash-ssh-no-host",
|
||||
Kind: core.KindModule,
|
||||
Steps: []core.Step{
|
||||
{
|
||||
Name: "ssh-no-host",
|
||||
Type: core.StepTypeRemoteBash,
|
||||
StepRunner: core.RunnerTypeSSH,
|
||||
Command: "echo hello",
|
||||
StepRunnerConfig: &core.StepRunnerConfig{
|
||||
RunnerConfig: &core.RunnerConfig{
|
||||
User: "testuser",
|
||||
// No Host - should fail
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
executor := NewExecutor()
|
||||
executor.SetDryRun(false)
|
||||
executor.SetSpinner(false)
|
||||
|
||||
_, err := executor.ExecuteModule(ctx, module, map[string]string{
|
||||
"target": "test",
|
||||
}, cfg)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "host")
|
||||
}
|
||||
|
||||
// Integration test - requires Docker
|
||||
func TestExecutor_RemoteBashStep_Docker(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping Docker integration test in short mode")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := testConfig(t)
|
||||
|
||||
module := &core.Workflow{
|
||||
Name: "test-remote-bash-docker",
|
||||
Kind: core.KindModule,
|
||||
Steps: []core.Step{
|
||||
{
|
||||
Name: "docker-echo",
|
||||
Type: core.StepTypeRemoteBash,
|
||||
StepRunner: core.RunnerTypeDocker,
|
||||
Command: "echo 'Hello from Docker'",
|
||||
StepRunnerConfig: &core.StepRunnerConfig{
|
||||
RunnerConfig: &core.RunnerConfig{
|
||||
Image: "alpine:latest",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
executor := NewExecutor()
|
||||
executor.SetDryRun(false)
|
||||
executor.SetSpinner(false)
|
||||
|
||||
result, err := executor.ExecuteModule(ctx, module, map[string]string{
|
||||
"target": "test",
|
||||
}, cfg)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, core.RunStatusCompleted, result.Status)
|
||||
require.Len(t, result.Steps, 1)
|
||||
assert.Equal(t, core.StepStatusSuccess, result.Steps[0].Status)
|
||||
assert.Contains(t, result.Steps[0].Output, "Hello from Docker")
|
||||
}
|
||||
|
||||
// Integration test - requires Docker
|
||||
func TestExecutor_RemoteBashStep_DockerMultipleCommands(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping Docker integration test in short mode")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := testConfig(t)
|
||||
|
||||
module := &core.Workflow{
|
||||
Name: "test-remote-bash-docker-multi",
|
||||
Kind: core.KindModule,
|
||||
Steps: []core.Step{
|
||||
{
|
||||
Name: "docker-multi",
|
||||
Type: core.StepTypeRemoteBash,
|
||||
StepRunner: core.RunnerTypeDocker,
|
||||
Commands: []string{
|
||||
"echo 'First'",
|
||||
"echo 'Second'",
|
||||
"echo 'Third'",
|
||||
},
|
||||
StepRunnerConfig: &core.StepRunnerConfig{
|
||||
RunnerConfig: &core.RunnerConfig{
|
||||
Image: "alpine:latest",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
executor := NewExecutor()
|
||||
executor.SetDryRun(false)
|
||||
executor.SetSpinner(false)
|
||||
|
||||
result, err := executor.ExecuteModule(ctx, module, map[string]string{
|
||||
"target": "test",
|
||||
}, cfg)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, core.RunStatusCompleted, result.Status)
|
||||
require.Len(t, result.Steps, 1)
|
||||
assert.Equal(t, core.StepStatusSuccess, result.Steps[0].Status)
|
||||
assert.Contains(t, result.Steps[0].Output, "First")
|
||||
assert.Contains(t, result.Steps[0].Output, "Second")
|
||||
assert.Contains(t, result.Steps[0].Output, "Third")
|
||||
}
|
||||
|
||||
// Integration test - requires Docker
|
||||
func TestExecutor_RemoteBashStep_DockerParallelCommands(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping Docker integration test in short mode")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := testConfig(t)
|
||||
|
||||
module := &core.Workflow{
|
||||
Name: "test-remote-bash-docker-parallel",
|
||||
Kind: core.KindModule,
|
||||
Steps: []core.Step{
|
||||
{
|
||||
Name: "docker-parallel",
|
||||
Type: core.StepTypeRemoteBash,
|
||||
StepRunner: core.RunnerTypeDocker,
|
||||
ParallelCommands: []string{
|
||||
"echo 'Parallel 1'",
|
||||
"echo 'Parallel 2'",
|
||||
"echo 'Parallel 3'",
|
||||
},
|
||||
StepRunnerConfig: &core.StepRunnerConfig{
|
||||
RunnerConfig: &core.RunnerConfig{
|
||||
Image: "alpine:latest",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
executor := NewExecutor()
|
||||
executor.SetDryRun(false)
|
||||
executor.SetSpinner(false)
|
||||
|
||||
result, err := executor.ExecuteModule(ctx, module, map[string]string{
|
||||
"target": "test",
|
||||
}, cfg)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, core.RunStatusCompleted, result.Status)
|
||||
require.Len(t, result.Steps, 1)
|
||||
assert.Equal(t, core.StepStatusSuccess, result.Steps[0].Status)
|
||||
assert.Contains(t, result.Steps[0].Output, "Parallel 1")
|
||||
assert.Contains(t, result.Steps[0].Output, "Parallel 2")
|
||||
assert.Contains(t, result.Steps[0].Output, "Parallel 3")
|
||||
}
|
||||
|
||||
// Integration test - requires SSH server on localhost:2222
|
||||
func TestExecutor_RemoteBashStep_SSH(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping SSH integration test in short mode")
|
||||
}
|
||||
conn, err := net.DialTimeout("tcp", "localhost:2222", 500*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Skip("skipping SSH integration test: SSH server not available on localhost:2222")
|
||||
}
|
||||
_ = conn.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := testConfig(t)
|
||||
|
||||
module := &core.Workflow{
|
||||
Name: "test-remote-bash-ssh",
|
||||
Kind: core.KindModule,
|
||||
Steps: []core.Step{
|
||||
{
|
||||
Name: "ssh-echo",
|
||||
Type: core.StepTypeRemoteBash,
|
||||
StepRunner: core.RunnerTypeSSH,
|
||||
Command: "echo 'Hello from SSH'",
|
||||
StepRunnerConfig: &core.StepRunnerConfig{
|
||||
RunnerConfig: &core.RunnerConfig{
|
||||
Host: "localhost",
|
||||
Port: 2222,
|
||||
User: "testuser",
|
||||
Password: "testpass",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
executor := NewExecutor()
|
||||
executor.SetDryRun(false)
|
||||
executor.SetSpinner(false)
|
||||
|
||||
result, err := executor.ExecuteModule(ctx, module, map[string]string{
|
||||
"target": "test",
|
||||
}, cfg)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, core.RunStatusCompleted, result.Status)
|
||||
require.Len(t, result.Steps, 1)
|
||||
assert.Equal(t, core.StepStatusSuccess, result.Steps[0].Status)
|
||||
assert.Contains(t, result.Steps[0].Output, "Hello from SSH")
|
||||
}
|
||||
|
||||
func TestRemoteBashExecutor_CanHandle(t *testing.T) {
|
||||
executor := NewRemoteBashExecutor(nil)
|
||||
|
||||
assert.True(t, executor.CanHandle(core.StepTypeRemoteBash))
|
||||
assert.False(t, executor.CanHandle(core.StepTypeBash))
|
||||
assert.False(t, executor.CanHandle(core.StepTypeFunction))
|
||||
assert.False(t, executor.CanHandle(core.StepTypeParallel))
|
||||
assert.False(t, executor.CanHandle(core.StepTypeForeach))
|
||||
}
|
||||
|
||||
func TestStepRunnerConfig_Clone(t *testing.T) {
|
||||
step := &core.Step{
|
||||
Name: "test-step",
|
||||
Type: core.StepTypeRemoteBash,
|
||||
StepRunner: core.RunnerTypeDocker,
|
||||
StepRunnerConfig: &core.StepRunnerConfig{
|
||||
RunnerConfig: &core.RunnerConfig{
|
||||
Image: "alpine:latest",
|
||||
Volumes: []string{"/host:/container"},
|
||||
Env: map[string]string{"KEY": "VALUE"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
cloned := step.Clone()
|
||||
|
||||
// Verify deep copy
|
||||
assert.Equal(t, step.StepRunner, cloned.StepRunner)
|
||||
assert.Equal(t, step.StepRunnerConfig.Image, cloned.StepRunnerConfig.Image)
|
||||
assert.Equal(t, step.StepRunnerConfig.Volumes, cloned.StepRunnerConfig.Volumes)
|
||||
assert.Equal(t, step.StepRunnerConfig.Env, cloned.StepRunnerConfig.Env)
|
||||
|
||||
// Modify cloned and verify original is unchanged
|
||||
cloned.StepRunner = core.RunnerTypeSSH
|
||||
cloned.StepRunnerConfig.Image = "ubuntu:latest"
|
||||
cloned.StepRunnerConfig.Volumes[0] = "/other:/path"
|
||||
cloned.StepRunnerConfig.Env["KEY"] = "CHANGED"
|
||||
|
||||
assert.Equal(t, core.RunnerTypeDocker, step.StepRunner)
|
||||
assert.Equal(t, "alpine:latest", step.StepRunnerConfig.Image)
|
||||
assert.Equal(t, "/host:/container", step.StepRunnerConfig.Volumes[0])
|
||||
assert.Equal(t, "VALUE", step.StepRunnerConfig.Env["KEY"])
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
)
|
||||
|
||||
// ExportRunCompleted writes run completion state to a JSON file
|
||||
// Uses the same format as run-state.json (StateExport) for consistency
|
||||
func ExportRunCompleted(path string, result *core.WorkflowResult, execCtx *core.ExecutionContext) error {
|
||||
return ExportState(path, result, execCtx)
|
||||
}
|
||||
|
||||
// RemoveRunCompleted removes the run-completed.json file if it exists
|
||||
func RemoveRunCompleted(path string) {
|
||||
if path != "" {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/state"
|
||||
)
|
||||
|
||||
// ExportState exports the current run state to a JSON file
|
||||
// It uses database data if available, otherwise falls back to in-memory data from result and execCtx
|
||||
func ExportState(stateFile string, result *core.WorkflowResult, execCtx *core.ExecutionContext) error {
|
||||
ctx := buildExportContext(result, execCtx)
|
||||
return state.Export(stateFile, ctx)
|
||||
}
|
||||
|
||||
func buildExportContext(result *core.WorkflowResult, execCtx *core.ExecutionContext) *state.ExportContext {
|
||||
ctx := &state.ExportContext{}
|
||||
|
||||
// Populate from execCtx
|
||||
if execCtx != nil {
|
||||
ctx.RunID = execCtx.RunID
|
||||
ctx.WorkflowName = execCtx.WorkflowName
|
||||
ctx.WorkflowKind = string(execCtx.WorkflowKind)
|
||||
ctx.Target = execCtx.Target
|
||||
ctx.WorkspacePath = execCtx.WorkspacePath
|
||||
ctx.WorkspaceName = execCtx.WorkspaceName
|
||||
ctx.Params = execCtx.Params
|
||||
}
|
||||
|
||||
// Populate/override from result
|
||||
if result != nil {
|
||||
if ctx.RunID == "" {
|
||||
ctx.RunID = result.RunID
|
||||
}
|
||||
if ctx.WorkflowName == "" {
|
||||
ctx.WorkflowName = result.WorkflowName
|
||||
}
|
||||
ctx.WorkflowKind = string(result.WorkflowKind)
|
||||
ctx.Target = result.Target
|
||||
ctx.Status = string(result.Status)
|
||||
startTime := result.StartTime
|
||||
endTime := result.EndTime
|
||||
ctx.StartedAt = &startTime
|
||||
ctx.CompletedAt = &endTime
|
||||
ctx.TotalSteps = len(result.Steps)
|
||||
|
||||
completedSteps := 0
|
||||
for _, step := range result.Steps {
|
||||
if step.Status == core.StepStatusSuccess {
|
||||
completedSteps++
|
||||
}
|
||||
}
|
||||
ctx.CompletedSteps = completedSteps
|
||||
|
||||
if result.Error != nil {
|
||||
ctx.ErrorMessage = result.Error.Error()
|
||||
}
|
||||
ctx.Artifacts = result.Artifacts
|
||||
}
|
||||
|
||||
return ctx
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/j3ssie/osmedeus/v5/internal/core"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// ExportWorkflowState writes the workflow YAML to the state file
|
||||
func ExportWorkflowState(stateFile string, workflow *core.Workflow) error {
|
||||
if stateFile == "" {
|
||||
return fmt.Errorf("state file path is empty")
|
||||
}
|
||||
|
||||
// Ensure directory exists
|
||||
dir := filepath.Dir(stateFile)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Marshal workflow to YAML
|
||||
data, err := yaml.Marshal(workflow)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal workflow: %w", err)
|
||||
}
|
||||
|
||||
// Write to file
|
||||
if err := os.WriteFile(stateFile, data, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write workflow state file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExportModuleWorkflowState writes a module workflow YAML to the modules folder
|
||||
func ExportModuleWorkflowState(folder string, moduleName string, workflow *core.Workflow) error {
|
||||
if folder == "" || moduleName == "" {
|
||||
return fmt.Errorf("folder or module name is empty")
|
||||
}
|
||||
|
||||
// Ensure directory exists
|
||||
if err := os.MkdirAll(folder, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Build filename: run-{module-name}.yaml
|
||||
filename := fmt.Sprintf("run-%s.yaml", moduleName)
|
||||
filePath := filepath.Join(folder, filename)
|
||||
|
||||
// Marshal workflow to YAML
|
||||
data, err := yaml.Marshal(workflow)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal workflow: %w", err)
|
||||
}
|
||||
|
||||
// Write to file
|
||||
if err := os.WriteFile(filePath, data, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write module workflow state file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package functions
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/logger"
|
||||
"github.com/j3ssie/osmedeus/v5/internal/storage"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// cdnUpload uploads a file to cloud storage
|
||||
// Usage: cdnUpload(localPath, remotePath) -> bool
|
||||
func (vf *vmFunc) cdnUpload(call goja.FunctionCall) goja.Value {
|
||||
localPath := call.Argument(0).String()
|
||||
remotePath := call.Argument(1).String()
|
||||
logger.Get().Debug("Calling cdnUpload", zap.String("localPath", localPath), zap.String("remotePath", remotePath))
|
||||
|
||||
if localPath == "undefined" || localPath == "" {
|
||||
logger.Get().Warn("cdnUpload: empty local path provided")
|
||||
return vf.vm.ToValue(false)
|
||||
}
|
||||
if remotePath == "undefined" || remotePath == "" {
|
||||
logger.Get().Warn("cdnUpload: empty remote path provided")
|
||||
return vf.vm.ToValue(false)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := storage.UploadFile(ctx, localPath, remotePath)
|
||||
if err != nil {
|
||||
logger.Get().Warn("cdnUpload: upload failed", zap.String("localPath", localPath), zap.Error(err))
|
||||
} else {
|
||||
logger.Get().Debug("cdnUpload result", zap.String("localPath", localPath), zap.String("remotePath", remotePath), zap.Bool("success", true))
|
||||
}
|
||||
return vf.vm.ToValue(err == nil)
|
||||
}
|
||||
|
||||
// cdnDownload downloads a file from cloud storage
|
||||
// Usage: cdnDownload(remotePath, localPath) -> bool
|
||||
func (vf *vmFunc) cdnDownload(call goja.FunctionCall) goja.Value {
|
||||
remotePath := call.Argument(0).String()
|
||||
localPath := call.Argument(1).String()
|
||||
logger.Get().Debug("Calling cdnDownload", zap.String("remotePath", remotePath), zap.String("localPath", localPath))
|
||||
|
||||
if remotePath == "undefined" || remotePath == "" {
|
||||
logger.Get().Warn("cdnDownload: empty remote path provided")
|
||||
return vf.vm.ToValue(false)
|
||||
}
|
||||
if localPath == "undefined" || localPath == "" {
|
||||
logger.Get().Warn("cdnDownload: empty local path provided")
|
||||
return vf.vm.ToValue(false)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := storage.DownloadFile(ctx, remotePath, localPath)
|
||||
if err != nil {
|
||||
logger.Get().Warn("cdnDownload: download failed", zap.String("remotePath", remotePath), zap.Error(err))
|
||||
} else {
|
||||
logger.Get().Debug("cdnDownload result", zap.String("remotePath", remotePath), zap.String("localPath", localPath), zap.Bool("success", true))
|
||||
}
|
||||
return vf.vm.ToValue(err == nil)
|
||||
}
|
||||
|
||||
// cdnExists checks if a file exists in cloud storage
|
||||
// Usage: cdnExists(remotePath) -> bool
|
||||
func (vf *vmFunc) cdnExists(call goja.FunctionCall) goja.Value {
|
||||
remotePath := call.Argument(0).String()
|
||||
logger.Get().Debug("Calling cdnExists", zap.String("remotePath", remotePath))
|
||||
|
||||
if remotePath == "undefined" || remotePath == "" {
|
||||
logger.Get().Warn("cdnExists: empty remote path provided")
|
||||
return vf.vm.ToValue(false)
|
||||
}
|
||||
|
||||
client, err := storage.NewClientFromGlobal()
|
||||
if err != nil {
|
||||
logger.Get().Warn("cdnExists: failed to create storage client", zap.Error(err))
|
||||
return vf.vm.ToValue(false)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
exists, _ := client.Exists(ctx, remotePath)
|
||||
logger.Get().Debug("cdnExists result", zap.String("remotePath", remotePath), zap.Bool("exists", exists))
|
||||
return vf.vm.ToValue(exists)
|
||||
}
|
||||
|
||||
// cdnDelete deletes a file from cloud storage
|
||||
// Usage: cdnDelete(remotePath) -> bool
|
||||
func (vf *vmFunc) cdnDelete(call goja.FunctionCall) goja.Value {
|
||||
remotePath := call.Argument(0).String()
|
||||
logger.Get().Debug("Calling cdnDelete", zap.String("remotePath", remotePath))
|
||||
|
||||
if remotePath == "undefined" || remotePath == "" {
|
||||
logger.Get().Warn("cdnDelete: empty remote path provided")
|
||||
return vf.vm.ToValue(false)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := storage.DeleteFile(ctx, remotePath)
|
||||
if err != nil {
|
||||
logger.Get().Warn("cdnDelete: delete failed", zap.String("remotePath", remotePath), zap.Error(err))
|
||||
} else {
|
||||
logger.Get().Debug("cdnDelete result", zap.String("remotePath", remotePath), zap.Bool("success", true))
|
||||
}
|
||||
return vf.vm.ToValue(err == nil)
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package functions
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCdnUpload_EmptyLocalPath(t *testing.T) {
|
||||
registry := NewRegistry()
|
||||
result, err := registry.Execute(
|
||||
`cdnUpload("", "remote/path")`,
|
||||
map[string]interface{}{},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, false, result)
|
||||
}
|
||||
|
||||
func TestCdnUpload_EmptyRemotePath(t *testing.T) {
|
||||
registry := NewRegistry()
|
||||
result, err := registry.Execute(
|
||||
`cdnUpload("/local/path", "")`,
|
||||
map[string]interface{}{},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, false, result)
|
||||
}
|
||||
|
||||
func TestCdnUpload_UndefinedArguments(t *testing.T) {
|
||||
registry := NewRegistry()
|
||||
result, err := registry.Execute(
|
||||
`cdnUpload()`,
|
||||
map[string]interface{}{},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, false, result)
|
||||
}
|
||||
|
||||
func TestCdnDownload_EmptyRemotePath(t *testing.T) {
|
||||
registry := NewRegistry()
|
||||
result, err := registry.Execute(
|
||||
`cdnDownload("", "/local/path")`,
|
||||
map[string]interface{}{},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, false, result)
|
||||
}
|
||||
|
||||
func TestCdnDownload_EmptyLocalPath(t *testing.T) {
|
||||
registry := NewRegistry()
|
||||
result, err := registry.Execute(
|
||||
`cdnDownload("remote/path", "")`,
|
||||
map[string]interface{}{},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, false, result)
|
||||
}
|
||||
|
||||
func TestCdnExists_EmptyPath(t *testing.T) {
|
||||
registry := NewRegistry()
|
||||
result, err := registry.Execute(
|
||||
`cdnExists("")`,
|
||||
map[string]interface{}{},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, false, result)
|
||||
}
|
||||
|
||||
func TestCdnDelete_EmptyPath(t *testing.T) {
|
||||
registry := NewRegistry()
|
||||
result, err := registry.Execute(
|
||||
`cdnDelete("")`,
|
||||
map[string]interface{}{},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, false, result)
|
||||
}
|
||||
|
||||
// Note: Actual CDN upload/download/delete tests require a configured
|
||||
// S3-compatible storage and are not included here. The functions will
|
||||
// return false when storage is not configured, which is expected.
|
||||
@@ -0,0 +1,612 @@
|
||||
package functions
|
||||
|
||||
// Function name constants for easy reference and consistency
|
||||
// This file serves as a central reference for all available workflow functions
|
||||
|
||||
// File Functions - Operations on files and directories
|
||||
const (
|
||||
FnFileExists = "fileExists" // fileExists(path) -> bool
|
||||
FnFileLength = "fileLength" // fileLength(path) -> int (non-empty line count)
|
||||
FnDirLength = "dirLength" // dirLength(path) -> int (entry count)
|
||||
FnFileContains = "fileContains" // fileContains(path, pattern) -> bool
|
||||
FnRegexExtract = "regexExtract" // regexExtract(path, pattern) -> []string
|
||||
FnReadFile = "readFile" // readFile(path) -> string
|
||||
FnReadLines = "readLines" // readLines(path) -> []string
|
||||
FnRemoveFile = "removeFile" // removeFile(path) -> bool
|
||||
FnRemoveFolder = "removeFolder" // removeFolder(path) -> bool
|
||||
FnRmRF = "rm_rf"
|
||||
FnRemoveAllExcept = "remove_all_except"
|
||||
FnCreateFolder = "createFolder" // createFolder(path) -> bool
|
||||
FnAppendFile = "appendFile" // appendFile(dest, source) -> bool
|
||||
FnMoveFile = "moveFile" // moveFile(source, dest) -> bool
|
||||
FnGlob = "glob" // glob(pattern) -> []string
|
||||
|
||||
FnGrepStringToFile = "grep_string_to_file" // grep_string_to_file(dest, source, str) -> bool
|
||||
FnGrepRegexToFile = "grep_regex_to_file" // grep_regex_to_file(dest, source, pattern) -> bool
|
||||
FnGrepString = "grep_string" // grep_string(source, str) -> string
|
||||
FnGrepRegex = "grep_regex" // grep_regex(source, pattern) -> string
|
||||
FnRemoveBlankLines = "remove_blank_lines" // remove_blank_lines(path) -> bool (in-place)
|
||||
)
|
||||
|
||||
// String Functions - String manipulation operations
|
||||
const (
|
||||
FnTrim = "trim" // trim(str) -> string
|
||||
FnSplit = "split" // split(str, delim) -> []string
|
||||
FnJoin = "join" // join(arr, delim) -> string
|
||||
FnReplace = "replace" // replace(str, old, new) -> string
|
||||
FnContains = "contains" // contains(str, substr) -> bool
|
||||
FnStartsWith = "startsWith" // startsWith(str, prefix) -> bool
|
||||
FnEndsWith = "endsWith" // endsWith(str, suffix) -> bool
|
||||
FnToLowerCase = "toLowerCase" // toLowerCase(str) -> string
|
||||
FnToUpperCase = "toUpperCase" // toUpperCase(str) -> string
|
||||
FnMatch = "match" // match(str, pattern) -> bool
|
||||
FnRegexMatch = "regex_match" // regex_match(pattern, str) -> bool (pattern first)
|
||||
FnCutWithDelim = "cut_with_delim" // cut_with_delim(input, delim, field) -> string (1-indexed like cut)
|
||||
FnNormalizePath = "normalize_path" // normalize_path(input) -> string (replace / | : etc with _)
|
||||
FnCleanSub = "clean_sub" // clean_sub(path, target?) -> bool (clean and deduplicate subdomains in file)
|
||||
)
|
||||
|
||||
// Type Conversion Functions - Convert between types
|
||||
const (
|
||||
FnParseInt = "parseInt" // parseInt(str) -> int
|
||||
FnParseFloat = "parseFloat" // parseFloat(str) -> float
|
||||
FnToString = "toString" // toString(val) -> string
|
||||
FnToBoolean = "toBoolean" // toBoolean(val) -> bool
|
||||
)
|
||||
|
||||
// Utility Functions - General utility operations
|
||||
const (
|
||||
FnLen = "len" // len(val) -> int
|
||||
FnIsEmpty = "isEmpty" // isEmpty(val) -> bool
|
||||
FnIsNotEmpty = "isNotEmpty" // isNotEmpty(val) -> bool
|
||||
FnPrintf = "printf" // printf(message) -> void (print message to stdout)
|
||||
FnCatFile = "cat_file" // cat_file(path) -> void (print file content to stdout)
|
||||
FnExit = "exit" // exit(code) -> void (exit scan with code)
|
||||
FnExecCmd = "exec_cmd" // exec_cmd(command) -> string (execute bash command, return stdout)
|
||||
FnSleep = "sleep" // sleep(seconds) -> void (pause for n seconds)
|
||||
)
|
||||
|
||||
// Logging Functions - Log messages with level prefixes
|
||||
const (
|
||||
FnLogDebug = "log_debug" // log_debug(message) -> void (print [DEBUG] message)
|
||||
FnLogInfo = "log_info" // log_info(message) -> void (print [INFO] message)
|
||||
FnLogWarn = "log_warn"
|
||||
FnLogError = "log_error"
|
||||
)
|
||||
|
||||
// HTTP and Network Functions
|
||||
const (
|
||||
FnHttpRequest = "httpRequest" // httpRequest(url, method, headers, body) -> {statusCode, body, headers}
|
||||
FnHttpGet = "http_get" // http_get(url) -> structured JSON response
|
||||
FnHttpPost = "http_post" // http_post(url, body) -> structured JSON response
|
||||
)
|
||||
|
||||
// Generation Functions - Generate random values
|
||||
const (
|
||||
FnRandomString = "randomString" // randomString(length) -> string
|
||||
FnUUID = "uuid" // uuid() -> string (UUID v4)
|
||||
)
|
||||
|
||||
// Encoding Functions - Encode/decode data
|
||||
const (
|
||||
FnBase64Encode = "base64Encode" // base64Encode(str) -> string
|
||||
FnBase64Decode = "base64Decode" // base64Decode(str) -> string
|
||||
)
|
||||
|
||||
// Data Query Functions - Query structured data
|
||||
const (
|
||||
FnJQ = "jq" // jq(jsonData, query) -> any (extract data using jq syntax)
|
||||
FnJQFromFile = "jq_from_file"
|
||||
)
|
||||
|
||||
// Notification Functions - Send notifications via various channels
|
||||
const (
|
||||
FnNotifyTelegram = "notifyTelegram" // notifyTelegram(message) -> bool
|
||||
FnSendTelegramFile = "sendTelegramFile" // sendTelegramFile(path, caption?) -> bool
|
||||
FnNotifyWebhook = "notifyWebhook" // notifyWebhook(message) -> bool
|
||||
FnSendWebhookEvent = "sendWebhookEvent" // sendWebhookEvent(eventType, data) -> bool
|
||||
)
|
||||
|
||||
// CDN/Storage Functions - Cloud storage operations
|
||||
const (
|
||||
FnCdnUpload = "cdnUpload" // cdnUpload(localPath, remotePath) -> bool
|
||||
FnCdnDownload = "cdnDownload" // cdnDownload(remotePath, localPath) -> bool
|
||||
FnCdnExists = "cdnExists" // cdnExists(remotePath) -> bool
|
||||
FnCdnDelete = "cdnDelete" // cdnDelete(remotePath) -> bool
|
||||
)
|
||||
|
||||
// Unix Command Wrappers - Wrappers around common Unix commands
|
||||
const (
|
||||
FnSortUnix = "sortUnix" // sortUnix(inputFile, outputFile?) -> bool (LC_ALL=C sort -u)
|
||||
FnWgetUnix = "wgetUnix" // wgetUnix(url, outputPath?) -> bool
|
||||
FnGitClone = "gitClone" // gitClone(repo, dest?) -> bool
|
||||
FnZipUnix = "zipUnix" // zipUnix(source, dest) -> bool (zip -r dest source)
|
||||
FnUnzipUnix = "unzipUnix" // unzipUnix(source, dest?) -> bool (unzip source -d dest)
|
||||
FnTarUnix = "tarUnix" // tarUnix(source, dest) -> bool (tar -czf dest source)
|
||||
FnUntarUnix = "untarUnix" // untarUnix(source, dest?) -> bool (tar -xzf source -C dest)
|
||||
FnDiffUnix = "diffUnix" // diffUnix(file1, file2, output?) -> string
|
||||
FnSedStringReplace = "sed_string_replace" // sed_string_replace(sed_syntax, source, dest) -> bool
|
||||
FnSedRegexReplace = "sed_regex_replace" // sed_regex_replace(sed_syntax, source, dest) -> bool
|
||||
)
|
||||
|
||||
// Archive Functions - Go implementations for zip/unzip
|
||||
const (
|
||||
FnZipDir = "zip_dir" // zip_dir(source, dest) -> bool
|
||||
FnUnzipDir = "unzip_dir" // unzip_dir(source, dest) -> bool
|
||||
)
|
||||
|
||||
// Diff Functions - Compare files
|
||||
const (
|
||||
FnExtractDiff = "extractDiff" // extractDiff(file1, file2) -> string (lines only in file2)
|
||||
)
|
||||
|
||||
// Output Functions - Save content to files
|
||||
const (
|
||||
FnSaveContent = "save_content" // save_content(content, path) -> bool
|
||||
FnJSONLToCSV = "jsonl_to_csv"
|
||||
FnCSVToJSONL = "csv_to_jsonl"
|
||||
FnJSONLUnique = "jsonl_unique"
|
||||
FnJSONLFilter = "jsonl_filter"
|
||||
)
|
||||
|
||||
// URL Processing Functions - URL deduplication and filtering
|
||||
const (
|
||||
FnInterestingUrls = "interesting_urls" // interesting_urls(src, dest, json_field?) -> bool
|
||||
)
|
||||
|
||||
// Markdown Functions - Markdown rendering and conversion
|
||||
const (
|
||||
FnRenderMarkdownFromFile = "render_markdown_from_file" // render_markdown_from_file(path) -> string (rendered markdown)
|
||||
FnPrintMarkdownFromFile = "print_markdown_from_file" // print_markdown_from_file(path) -> void (print with syntax highlight)
|
||||
FnConvertJSONLToMarkdown = "convert_jsonl_to_markdown" // convert_jsonl_to_markdown(input_path, output_path) -> bool (writes markdown table to file)
|
||||
FnConvertCSVToMarkdown = "convert_csv_to_markdown" // convert_csv_to_markdown(path) -> string (markdown table)
|
||||
FnRenderMarkdownReport = "render_markdown_report" // render_markdown_report(template_path, output_path) -> bool
|
||||
FnGenerateSecurityReport = "generate_security_report" // generate_security_report(template_path) -> bool (output to {{Output}}/security-report.md)
|
||||
)
|
||||
|
||||
// Database Functions - Database update and import operations
|
||||
const (
|
||||
FnDBUpdate = "db_update" // db_update(table, key, field, value) -> bool
|
||||
FnDBImportAsset = "db_import_asset" // db_import_asset(workspace, json_data) -> bool
|
||||
FnDBRawInsertAsset = "db_raw_insert_asset" // db_raw_insert_asset(workspace, json_data) -> int (asset ID)
|
||||
FnDBTotalURLs = "db_total_urls" // db_total_urls(file_path) -> int (count lines, update workspace)
|
||||
FnDBTotalSubdomains = "db_total_subdomains" // db_total_subdomains(file_path) -> int
|
||||
FnDBTotalAssets = "db_total_assets" // db_total_assets(file_path) -> int
|
||||
FnDBTotalVulns = "db_total_vulns" // db_total_vulns(file_path) -> int
|
||||
FnDBVulnCritical = "db_vuln_critical" // db_vuln_critical(file_path) -> int
|
||||
FnDBVulnHigh = "db_vuln_high" // db_vuln_high(file_path) -> int
|
||||
FnDBVulnMedium = "db_vuln_medium" // db_vuln_medium(file_path) -> int
|
||||
FnDBVulnLow = "db_vuln_low" // db_vuln_low(file_path) -> int
|
||||
FnDBTotalIPs = "db_total_ips" // db_total_ips(file_path) -> int
|
||||
FnDBTotalLinks = "db_total_links" // db_total_links(file_path) -> int
|
||||
FnDBTotalContent = "db_total_content" // db_total_content(file_path) -> int
|
||||
FnDBTotalArchive = "db_total_archive" // db_total_archive(file_path) -> int
|
||||
FnRuntimeExport = "runtime_export" // runtime_export() -> bool (export scan+workspace to run-state.json)
|
||||
FnDBRegisterArtifact = "register_artifact" // register_artifact(path, type?) -> bool (register file as scan artifact)
|
||||
FnStoreArtifact = "store_artifact" // store_artifact(path, type?) -> bool (store file as scan artifact)
|
||||
FnDBSelectAssets = "db_select_assets" // db_select_assets(workspace, format) -> string
|
||||
FnDBSelectAssetsFiltered = "db_select_assets_filtered" // db_select_assets_filtered(workspace, status_code, asset_type, format) -> string
|
||||
FnDBSelectVulnerabilities = "db_select_vulnerabilities" // db_select_vulnerabilities(workspace, format) -> string
|
||||
FnDBSelectVulnerabilitiesFiltered = "db_select_vulnerabilities_filtered" // db_select_vulnerabilities_filtered(workspace, severity, asset_value, format) -> string
|
||||
FnDBSelect = "db_select" // db_select(sql_query, format) -> string
|
||||
FnDBSelectToFile = "db_select_to_file" // db_select_to_file(sql_query, dest) -> bool
|
||||
FnDBSelectToJSONL = "db_select_to_jsonl" // db_select_to_jsonl(sql_query, fields, dest) -> bool
|
||||
|
||||
// SELECT functions - read workspace stats without arguments (uses current workspace context)
|
||||
FnDBSelectTotalSubdomains = "db_select_total_subdomains" // db_select_total_subdomains() -> int
|
||||
FnDBSelectTotalURLs = "db_select_total_urls" // db_select_total_urls() -> int
|
||||
FnDBSelectTotalAssets = "db_select_total_assets" // db_select_total_assets() -> int
|
||||
FnDBSelectTotalVulns = "db_select_total_vulns" // db_select_total_vulns() -> int
|
||||
FnDBSelectVulnCritical = "db_select_vuln_critical" // db_select_vuln_critical() -> int
|
||||
FnDBSelectVulnHigh = "db_select_vuln_high" // db_select_vuln_high() -> int
|
||||
FnDBSelectVulnMedium = "db_select_vuln_medium" // db_select_vuln_medium() -> int
|
||||
FnDBSelectVulnLow = "db_select_vuln_low" // db_select_vuln_low() -> int
|
||||
|
||||
// JSONL import functions - import data from JSONL files
|
||||
FnDBImportAssetFromFile = "db_import_asset_from_file" // db_import_asset_from_file(workspace, file_path) -> int (count)
|
||||
FnDBImportVuln = "db_import_vuln" // db_import_vuln(workspace, json_data) -> bool
|
||||
FnDBImportVulnFromFile = "db_import_vuln_from_file" // db_import_vuln_from_file(workspace, file_path) -> int (count)
|
||||
)
|
||||
|
||||
// AllFunctions returns a list of all available function names
|
||||
func AllFunctions() []string {
|
||||
return []string{
|
||||
// File Functions
|
||||
FnFileExists,
|
||||
FnFileLength,
|
||||
FnDirLength,
|
||||
FnFileContains,
|
||||
FnRegexExtract,
|
||||
FnReadFile,
|
||||
FnReadLines,
|
||||
FnRemoveFile,
|
||||
FnRemoveFolder,
|
||||
FnRmRF,
|
||||
FnRemoveAllExcept,
|
||||
FnCreateFolder,
|
||||
FnAppendFile,
|
||||
FnMoveFile,
|
||||
FnGlob,
|
||||
FnGrepStringToFile,
|
||||
FnGrepRegexToFile,
|
||||
FnGrepString,
|
||||
FnGrepRegex,
|
||||
FnRemoveBlankLines,
|
||||
|
||||
// String Functions
|
||||
FnTrim,
|
||||
FnSplit,
|
||||
FnJoin,
|
||||
FnReplace,
|
||||
FnContains,
|
||||
FnStartsWith,
|
||||
FnEndsWith,
|
||||
FnToLowerCase,
|
||||
FnToUpperCase,
|
||||
FnMatch,
|
||||
FnRegexMatch,
|
||||
FnCutWithDelim,
|
||||
FnNormalizePath,
|
||||
FnCleanSub,
|
||||
|
||||
// Type Conversion Functions
|
||||
FnParseInt,
|
||||
FnParseFloat,
|
||||
FnToString,
|
||||
FnToBoolean,
|
||||
|
||||
// Utility Functions
|
||||
FnLen,
|
||||
FnIsEmpty,
|
||||
FnIsNotEmpty,
|
||||
FnPrintf,
|
||||
FnCatFile,
|
||||
FnExit,
|
||||
FnExecCmd,
|
||||
FnSleep,
|
||||
|
||||
// Logging Functions
|
||||
FnLogDebug,
|
||||
FnLogInfo,
|
||||
FnLogWarn,
|
||||
FnLogError,
|
||||
|
||||
// HTTP Functions
|
||||
FnHttpRequest,
|
||||
FnHttpGet,
|
||||
FnHttpPost,
|
||||
|
||||
// Generation Functions
|
||||
FnRandomString,
|
||||
FnUUID,
|
||||
|
||||
// Encoding Functions
|
||||
FnBase64Encode,
|
||||
FnBase64Decode,
|
||||
|
||||
// Data Query Functions
|
||||
FnJQ,
|
||||
FnJQFromFile,
|
||||
|
||||
// Notification Functions
|
||||
FnNotifyTelegram,
|
||||
FnSendTelegramFile,
|
||||
FnNotifyWebhook,
|
||||
FnSendWebhookEvent,
|
||||
|
||||
// CDN/Storage Functions
|
||||
FnCdnUpload,
|
||||
FnCdnDownload,
|
||||
FnCdnExists,
|
||||
FnCdnDelete,
|
||||
|
||||
// Unix Command Wrappers
|
||||
FnSortUnix,
|
||||
FnWgetUnix,
|
||||
FnGitClone,
|
||||
FnZipUnix,
|
||||
FnUnzipUnix,
|
||||
FnTarUnix,
|
||||
FnUntarUnix,
|
||||
FnDiffUnix,
|
||||
FnSedStringReplace,
|
||||
FnSedRegexReplace,
|
||||
|
||||
// Archive Functions (Go implementations)
|
||||
FnZipDir,
|
||||
FnUnzipDir,
|
||||
|
||||
// Diff Functions
|
||||
FnExtractDiff,
|
||||
|
||||
// Output Functions
|
||||
FnSaveContent,
|
||||
FnJSONLToCSV,
|
||||
FnCSVToJSONL,
|
||||
FnJSONLUnique,
|
||||
FnJSONLFilter,
|
||||
|
||||
// URL Processing Functions
|
||||
FnInterestingUrls,
|
||||
|
||||
// Markdown Functions
|
||||
FnRenderMarkdownFromFile,
|
||||
FnPrintMarkdownFromFile,
|
||||
FnConvertJSONLToMarkdown,
|
||||
FnConvertCSVToMarkdown,
|
||||
FnRenderMarkdownReport,
|
||||
FnGenerateSecurityReport,
|
||||
|
||||
// Database Functions
|
||||
FnDBUpdate,
|
||||
FnDBImportAsset,
|
||||
FnDBRawInsertAsset,
|
||||
FnDBTotalURLs,
|
||||
FnDBTotalSubdomains,
|
||||
FnDBTotalAssets,
|
||||
FnDBTotalVulns,
|
||||
FnDBVulnCritical,
|
||||
FnDBVulnHigh,
|
||||
FnDBVulnMedium,
|
||||
FnDBVulnLow,
|
||||
FnDBTotalIPs,
|
||||
FnDBTotalLinks,
|
||||
FnDBTotalContent,
|
||||
FnDBTotalArchive,
|
||||
FnRuntimeExport,
|
||||
FnDBRegisterArtifact,
|
||||
FnStoreArtifact,
|
||||
FnDBSelectAssets,
|
||||
FnDBSelectAssetsFiltered,
|
||||
FnDBSelectVulnerabilities,
|
||||
FnDBSelectVulnerabilitiesFiltered,
|
||||
FnDBSelect,
|
||||
FnDBSelectToFile,
|
||||
FnDBSelectToJSONL,
|
||||
FnDBSelectTotalSubdomains,
|
||||
FnDBSelectTotalURLs,
|
||||
FnDBSelectTotalAssets,
|
||||
FnDBSelectTotalVulns,
|
||||
FnDBSelectVulnCritical,
|
||||
FnDBSelectVulnHigh,
|
||||
FnDBSelectVulnMedium,
|
||||
FnDBSelectVulnLow,
|
||||
|
||||
// JSONL import functions
|
||||
FnDBImportAssetFromFile,
|
||||
FnDBImportVuln,
|
||||
FnDBImportVulnFromFile,
|
||||
}
|
||||
}
|
||||
|
||||
// FunctionInfo describes a utility function with its metadata
|
||||
type FunctionInfo struct {
|
||||
Name string // Function name (e.g., "fileExists")
|
||||
Signature string // Full signature (e.g., "fileExists(path)")
|
||||
Description string // Human-readable description
|
||||
ReturnType string // Return type (e.g., "bool", "string")
|
||||
Example string // Example usage
|
||||
}
|
||||
|
||||
// Category keys for function registry
|
||||
const (
|
||||
CategoryFile = "file"
|
||||
CategoryString = "string"
|
||||
CategoryTypeConversion = "type_conversion"
|
||||
CategoryUtility = "utility"
|
||||
CategoryLogging = "logging"
|
||||
CategoryHTTP = "http"
|
||||
CategoryGeneration = "generation"
|
||||
CategoryEncoding = "encoding"
|
||||
CategoryDataQuery = "data_query"
|
||||
CategoryNotification = "notification"
|
||||
CategoryCDNStorage = "cdn_storage"
|
||||
CategoryUnixCommands = "unix_commands"
|
||||
CategoryArchive = "archive"
|
||||
CategoryDiff = "diff"
|
||||
CategoryOutput = "output"
|
||||
CategoryURLProcessing = "url_processing"
|
||||
CategoryMarkdown = "markdown"
|
||||
CategoryDatabase = "database"
|
||||
)
|
||||
|
||||
// CategoryInfo provides display metadata for a function category
|
||||
type CategoryInfo struct {
|
||||
Key string
|
||||
Title string
|
||||
ShortTitle string // Short version for table display
|
||||
}
|
||||
|
||||
// CategoryOrder returns the ordered list of categories with display titles
|
||||
func CategoryOrder() []CategoryInfo {
|
||||
return []CategoryInfo{
|
||||
{CategoryFile, "File Functions", "File"},
|
||||
{CategoryString, "String Functions", "String"},
|
||||
{CategoryTypeConversion, "Type Conversion", "Type"},
|
||||
{CategoryUtility, "Utility Functions", "Utility"},
|
||||
{CategoryLogging, "Logging Functions", "Logging"},
|
||||
{CategoryHTTP, "HTTP Functions", "HTTP"},
|
||||
{CategoryGeneration, "Generation Functions", "Generation"},
|
||||
{CategoryEncoding, "Encoding Functions", "Encoding"},
|
||||
{CategoryDataQuery, "Data Query Functions", "Data Query"},
|
||||
{CategoryNotification, "Notification Functions", "Notification"},
|
||||
{CategoryCDNStorage, "CDN/Storage Functions", "CDN/Storage"},
|
||||
{CategoryUnixCommands, "Unix Command Wrappers", "Unix"},
|
||||
{CategoryArchive, "Archive Functions (Go)", "Archive"},
|
||||
{CategoryDiff, "Diff Functions", "Diff"},
|
||||
{CategoryOutput, "Output Functions", "Output"},
|
||||
{CategoryURLProcessing, "URL Processing Functions", "URL"},
|
||||
{CategoryMarkdown, "Markdown Functions", "Markdown"},
|
||||
{CategoryDatabase, "Database Functions", "Database"},
|
||||
}
|
||||
}
|
||||
|
||||
// FunctionRegistry returns all function metadata organized by category
|
||||
func FunctionRegistry() map[string][]FunctionInfo {
|
||||
return map[string][]FunctionInfo{
|
||||
CategoryFile: {
|
||||
{FnFileExists, "fileExists(path)", "Check if file exists", "bool", "fileExists('/tmp/test.txt')"},
|
||||
{FnFileLength, "fileLength(path)", "Count non-empty lines in file", "int", "fileLength('{{Output}}/subdomains.txt')"},
|
||||
{FnDirLength, "dirLength(path)", "Count entries in directory", "int", "dirLength('{{Output}}/screenshots')"},
|
||||
{FnFileContains, "fileContains(path, pattern)", "Check if file contains pattern", "bool", "fileContains('{{Output}}/urls.txt', 'admin')"},
|
||||
{FnRegexExtract, "regexExtract(path, pattern)", "Extract matching lines from file", "[]string", "regexExtract('{{Output}}/urls.txt', '.*api.*')"},
|
||||
{FnReadFile, "readFile(path)", "Read entire file contents", "string", "readFile('{{Output}}/config.json')"},
|
||||
{FnReadLines, "readLines(path)", "Read file as array of lines", "[]string", "readLines('{{Output}}/subdomains.txt')"},
|
||||
{FnRemoveFile, "removeFile(path)", "Delete a file", "bool", "removeFile('{{Output}}/temp.txt')"},
|
||||
{FnRemoveFolder, "removeFolder(path)", "Delete folder recursively", "bool", "removeFolder('{{Output}}/cache')"},
|
||||
{FnRmRF, "rm_rf(path)", "Delete file or folder recursively", "bool", "rm_rf('{{Output}}/tmp')"},
|
||||
{FnRemoveAllExcept, "remove_all_except(folder, keep_file)", "Remove everything under folder except keep_file", "bool", "remove_all_except('{{Output}}', '{{Output}}/keep.txt')"},
|
||||
{FnCreateFolder, "createFolder(path)", "Create folder recursively", "bool", "createFolder('{{Output}}/new-folder')"},
|
||||
{FnAppendFile, "appendFile(dest, source)", "Append source file content into destination file", "bool", "appendFile('{{Output}}/all.txt', '{{Output}}/part.txt')"},
|
||||
{FnMoveFile, "moveFile(source, dest)", "Move file from source to destination (rename or copy+delete)", "bool", "moveFile('{{Output}}/raw.txt', '{{Output}}/processed.txt')"},
|
||||
{FnGlob, "glob(pattern)", "List filenames matching glob pattern", "[]string", "glob('{{Output}}/*.txt')"},
|
||||
{FnGrepStringToFile, "grep_string_to_file(dest, source, str)", "Write lines containing string to destination file", "bool", "grep_string_to_file('{{Output}}/out.txt', '{{Output}}/in.txt', 'admin')"},
|
||||
{FnGrepRegexToFile, "grep_regex_to_file(dest, source, pattern)", "Write lines matching regex to destination file", "bool", "grep_regex_to_file('{{Output}}/out.txt', '{{Output}}/in.txt', '.*api.*')"},
|
||||
{FnGrepString, "grep_string(source, str)", "Return lines containing string", "string", "grep_string('{{Output}}/in.txt', 'admin')"},
|
||||
{FnGrepRegex, "grep_regex(source, pattern)", "Return lines matching regex", "string", "grep_regex('{{Output}}/in.txt', '.*api.*')"},
|
||||
{FnRemoveBlankLines, "remove_blank_lines(path)", "Remove blank lines from file in-place", "bool", "remove_blank_lines('{{Output}}/urls.txt')"},
|
||||
},
|
||||
CategoryString: {
|
||||
{FnTrim, "trim(str)", "Trim whitespace", "string", "trim(' hello ')"},
|
||||
{FnSplit, "split(str, delim)", "Split string by delimiter", "[]string", "split('a,b,c', ',')"},
|
||||
{FnJoin, "join(arr, delim)", "Join array with delimiter", "string", "join(['a','b','c'], ',')"},
|
||||
{FnReplace, "replace(str, old, new)", "Replace all occurrences", "string", "replace('hello', 'l', 'L')"},
|
||||
{FnContains, "contains(str, substr)", "Check if string contains substring", "bool", "contains('hello', 'ell')"},
|
||||
{FnStartsWith, "startsWith(str, prefix)", "Check if string starts with prefix", "bool", "startsWith('hello', 'he')"},
|
||||
{FnEndsWith, "endsWith(str, suffix)", "Check if string ends with suffix", "bool", "endsWith('hello.txt', '.txt')"},
|
||||
{FnToLowerCase, "toLowerCase(str)", "Convert to lowercase", "string", "toLowerCase('HELLO')"},
|
||||
{FnToUpperCase, "toUpperCase(str)", "Convert to uppercase", "string", "toUpperCase('hello')"},
|
||||
{FnMatch, "match(str, pattern)", "Check if string matches regex", "bool", "match('test123', '[0-9]+')"},
|
||||
{FnRegexMatch, "regex_match(pattern, str)", "Check if string matches regex (pattern first)", "bool", "regex_match('[0-9]+', 'test123')"},
|
||||
{FnCutWithDelim, "cut_with_delim(input, delim, field)", "Extract field by delimiter (1-indexed)", "string", "cut_with_delim('a:b:c', ':', 2)"},
|
||||
{FnNormalizePath, "normalize_path(input)", "Replace special chars with underscore", "string", "normalize_path('test/path:file')"},
|
||||
{FnCleanSub, "clean_sub(path, target?)", "Clean and deduplicate subdomains in file, optionally filter by target domain", "bool", "clean_sub('{{Output}}/subdomains.txt', 'example.com')"},
|
||||
},
|
||||
CategoryTypeConversion: {
|
||||
{FnParseInt, "parseInt(str)", "Parse string to integer", "int", "parseInt('42')"},
|
||||
{FnParseFloat, "parseFloat(str)", "Parse string to float", "float", "parseFloat('3.14')"},
|
||||
{FnToString, "toString(val)", "Convert value to string", "string", "toString(123)"},
|
||||
{FnToBoolean, "toBoolean(val)", "Convert value to boolean", "bool", "toBoolean('true')"},
|
||||
},
|
||||
CategoryUtility: {
|
||||
{FnLen, "len(val)", "Get length of string or array", "int", "len('hello')"},
|
||||
{FnIsEmpty, "isEmpty(val)", "Check if value is empty", "bool", "isEmpty('')"},
|
||||
{FnIsNotEmpty, "isNotEmpty(val)", "Check if value is not empty", "bool", "isNotEmpty('test')"},
|
||||
{FnPrintf, "printf(message)", "Print message to stdout", "void", "printf('Scan started')"},
|
||||
{FnCatFile, "cat_file(path)", "Print file content to stdout", "void", "cat_file('{{Output}}/results.txt')"},
|
||||
{FnExit, "exit(code)", "Exit scan with code", "void", "exit(1)"},
|
||||
{FnExecCmd, "exec_cmd(command)", "Execute bash command and return output", "string", "exec_cmd('whoami')"},
|
||||
{FnSleep, "sleep(seconds)", "Pause for n seconds", "void", "sleep(5)"},
|
||||
},
|
||||
CategoryLogging: {
|
||||
{FnLogDebug, "log_debug(message)", "Log debug message with [DEBUG] prefix", "void", "log_debug('Processing target')"},
|
||||
{FnLogInfo, "log_info(message)", "Log info message with [INFO] prefix", "void", "log_info('Scan completed')"},
|
||||
{FnLogWarn, "log_warn(message)", "Log warning message with [WARN] prefix", "void", "log_warn('Timeout hit')"},
|
||||
{FnLogError, "log_error(message)", "Log error message with [ERROR] prefix", "void", "log_error('Request failed')"},
|
||||
},
|
||||
CategoryHTTP: {
|
||||
{FnHttpRequest, "httpRequest(url, method, headers, body)", "Make HTTP request", "object", "httpRequest('https://api.example.com', 'GET', {}, '')"},
|
||||
{FnHttpGet, "http_get(url)", "HTTP GET request with structured response", "object", "http_get('https://api.example.com/data')"},
|
||||
{FnHttpPost, "http_post(url, body)", "HTTP POST request with structured response", "object", "http_post('https://api.example.com', '{\"key\":\"value\"}')"},
|
||||
},
|
||||
CategoryGeneration: {
|
||||
{FnRandomString, "randomString(length)", "Generate random alphanumeric string", "string", "randomString(16)"},
|
||||
{FnUUID, "uuid()", "Generate UUID v4", "string", "uuid()"},
|
||||
},
|
||||
CategoryEncoding: {
|
||||
{FnBase64Encode, "base64Encode(str)", "Encode string to base64", "string", "base64Encode('hello')"},
|
||||
{FnBase64Decode, "base64Decode(str)", "Decode base64 string", "string", "base64Decode('aGVsbG8=')"},
|
||||
},
|
||||
CategoryDataQuery: {
|
||||
{FnJQ, "jq(jsonData, query)", "Extract data using jq syntax", "any", "jq('{\"name\":\"test\"}', '.name')"},
|
||||
{FnJQFromFile, "jq_from_file(path, query)", "Extract data using jq from JSON file", "any", "jq_from_file('{{Output}}/data.json', '.name')"},
|
||||
},
|
||||
CategoryNotification: {
|
||||
{FnNotifyTelegram, "notifyTelegram(message)", "Send message to Telegram", "bool", "notifyTelegram('Scan finished for {{Target}}')"},
|
||||
{FnSendTelegramFile, "sendTelegramFile(path, caption?)", "Send file to Telegram", "bool", "sendTelegramFile('{{Output}}/report.pdf', 'Scan report')"},
|
||||
{FnNotifyWebhook, "notifyWebhook(message)", "Send message to all webhooks", "bool", "notifyWebhook('Scan finished for {{Target}}')"},
|
||||
{FnSendWebhookEvent, "sendWebhookEvent(eventType, data)", "Send event to all webhooks", "bool", "sendWebhookEvent('scan_complete', {target: '{{Target}}'})"},
|
||||
},
|
||||
CategoryCDNStorage: {
|
||||
{FnCdnUpload, "cdnUpload(localPath, remotePath)", "Upload file to cloud storage", "bool", "cdnUpload('{{Output}}/report.zip', 'scans/{{Target}}/report.zip')"},
|
||||
{FnCdnDownload, "cdnDownload(remotePath, localPath)", "Download file from cloud storage", "bool", "cdnDownload('wordlists/common.txt', '/tmp/common.txt')"},
|
||||
{FnCdnExists, "cdnExists(remotePath)", "Check if file exists in cloud storage", "bool", "cdnExists('scans/{{Target}}/report.zip')"},
|
||||
{FnCdnDelete, "cdnDelete(remotePath)", "Delete file from cloud storage", "bool", "cdnDelete('scans/{{Target}}/old-report.zip')"},
|
||||
},
|
||||
CategoryUnixCommands: {
|
||||
{FnSortUnix, "sortUnix(input, output?)", "Sort file with LC_ALL=C sort -u", "bool", "sortUnix('{{Output}}/urls.txt')"},
|
||||
{FnWgetUnix, "wgetUnix(url, output?)", "Download file with wget", "bool", "wgetUnix('https://example.com/file.txt', '/tmp/file.txt')"},
|
||||
{FnGitClone, "gitClone(repo, dest?)", "Clone git repository (shallow)", "bool", "gitClone('https://github.com/user/repo', '/tmp/repo')"},
|
||||
{FnZipUnix, "zipUnix(source, dest)", "Create zip archive (zip -r)", "bool", "zipUnix('{{Output}}', '{{Output}}/archive.zip')"},
|
||||
{FnUnzipUnix, "unzipUnix(source, dest?)", "Extract zip archive (unzip)", "bool", "unzipUnix('/tmp/archive.zip', '/tmp/extracted')"},
|
||||
{FnTarUnix, "tarUnix(source, dest)", "Create tar.gz archive (tar -czf)", "bool", "tarUnix('{{Output}}', '{{Output}}/archive.tar.gz')"},
|
||||
{FnUntarUnix, "untarUnix(source, dest?)", "Extract tar.gz archive (tar -xzf)", "bool", "untarUnix('/tmp/archive.tar.gz', '/tmp/extracted')"},
|
||||
{FnDiffUnix, "diffUnix(file1, file2, output?)", "Compare files with diff command", "string", "diffUnix('old.txt', 'new.txt', 'diff.txt')"},
|
||||
{FnSedStringReplace, "sed_string_replace(sed_syntax, source, dest)", "String replacement with sed s/old/new/g syntax", "bool", "sed_string_replace('s/http/https/g', '{{Output}}/urls.txt', '{{Output}}/urls-fixed.txt')"},
|
||||
{FnSedRegexReplace, "sed_regex_replace(sed_syntax, source, dest)", "Regex replacement with sed s/pattern/repl/g syntax", "bool", "sed_regex_replace('s/[0-9]+/NUM/g', '{{Output}}/data.txt', '{{Output}}/data-clean.txt')"},
|
||||
},
|
||||
CategoryArchive: {
|
||||
{FnZipDir, "zip_dir(source, dest)", "Zip directory using Go archive/zip", "bool", "zip_dir('{{Output}}', '{{Output}}/archive.zip')"},
|
||||
{FnUnzipDir, "unzip_dir(source, dest)", "Unzip archive using Go archive/zip", "bool", "unzip_dir('/tmp/archive.zip', '/tmp/extracted')"},
|
||||
},
|
||||
CategoryDiff: {
|
||||
{FnExtractDiff, "extractDiff(file1, file2)", "Lines only in file2 (new content)", "string", "extractDiff('{{Output}}/old-subs.txt', '{{Output}}/new-subs.txt')"},
|
||||
},
|
||||
CategoryOutput: {
|
||||
{FnSaveContent, "save_content(content, path)", "Save string content to file", "bool", "save_content('hello', '{{Output}}/greeting.txt')"},
|
||||
{FnJSONLToCSV, "jsonl_to_csv(source, dest)", "Convert JSONL file to CSV", "bool", "jsonl_to_csv('{{Output}}/assets.jsonl', '{{Output}}/assets.csv')"},
|
||||
{FnCSVToJSONL, "csv_to_jsonl(source, dest)", "Convert CSV file to JSONL", "bool", "csv_to_jsonl('{{Output}}/assets.csv', '{{Output}}/assets.jsonl')"},
|
||||
{FnJSONLUnique, "jsonl_unique(source, dest, fields)", "Deduplicate JSONL by hashing selected fields", "bool", "jsonl_unique('{{Output}}/httpx.jsonl', '{{Output}}/httpx.unique.jsonl', ['status','words','lines'])"},
|
||||
{FnJSONLFilter, "jsonl_filter(source, dest, fields)", "Filter JSONL to selected fields (comma or array)", "bool", "jsonl_filter('{{Output}}/httpx.jsonl', '{{Output}}/httpx.filtered.jsonl', 'host,status,hash.body_sha256')"},
|
||||
},
|
||||
CategoryURLProcessing: {
|
||||
{FnInterestingUrls, "interesting_urls(src, dest, json_field?)", "Deduplicate URLs by hostname+path+params, filter static files and noise patterns", "bool", "interesting_urls('{{Output}}/all-urls.txt', '{{Output}}/interesting-urls.txt', 'url')"},
|
||||
},
|
||||
CategoryMarkdown: {
|
||||
{FnRenderMarkdownFromFile, "render_markdown_from_file(path)", "Render markdown with terminal styling", "string", "render_markdown_from_file('{{Output}}/report.md')"},
|
||||
{FnPrintMarkdownFromFile, "print_markdown_from_file(path)", "Print markdown with syntax highlighting", "void", "print_markdown_from_file('{{Output}}/summary.md')"},
|
||||
{FnConvertJSONLToMarkdown, "convert_jsonl_to_markdown(input_path, output_path)", "Convert JSONL to markdown table and write to file", "bool", "convert_jsonl_to_markdown('{{Output}}/assets.jsonl', '{{Output}}/assets.md')"},
|
||||
{FnConvertCSVToMarkdown, "convert_csv_to_markdown(path)", "Convert CSV to markdown table", "string", "convert_csv_to_markdown('{{Output}}/data.csv')"},
|
||||
{FnRenderMarkdownReport, "render_markdown_report(template_path, output_path)", "Render markdown template with osm-func blocks", "bool", "render_markdown_report('{{Templates}}/report.md', '{{Output}}/report.md')"},
|
||||
{FnGenerateSecurityReport, "generate_security_report(template_path)", "Generate security report from template to {{Output}}/security-report.md and register as artifact", "bool", "generate_security_report('{{MarkdownTemplates}}/security-report-template.md')"},
|
||||
},
|
||||
CategoryDatabase: {
|
||||
{FnDBRegisterArtifact, "register_artifact(path, type?)", "Register file as scan artifact", "bool", "register_artifact('{{Output}}/nuclei.json', 'nuclei')"},
|
||||
{FnStoreArtifact, "store_artifact(path)", "Store file as run artifact for current workspace", "bool", "store_artifact('{{Output}}/report.md')"},
|
||||
{FnDBUpdate, "db_update(table, key, field, value)", "Update database field", "bool", "db_update('workspaces', '{{Workspace}}', 'status', 'completed')"},
|
||||
{FnDBImportAsset, "db_import_asset(workspace, json)", "Import asset from JSON (upsert)", "bool", "db_import_asset('{{Workspace}}', '{\"asset_value\":\"sub.example.com\"}')"},
|
||||
{FnDBRawInsertAsset, "db_raw_insert_asset(workspace, json)", "Insert asset from JSON (pure insert)", "int", "db_raw_insert_asset('{{Workspace}}', '{\"asset_value\":\"api.example.com\"}')"},
|
||||
{FnDBTotalURLs, "db_total_urls(path)", "Count lines, update workspace URLs", "int", "db_total_urls('{{Output}}/urls.txt')"},
|
||||
{FnDBTotalSubdomains, "db_total_subdomains(path)", "Count lines, update workspace subdomains", "int", "db_total_subdomains('{{Output}}/subdomains.txt')"},
|
||||
{FnDBTotalAssets, "db_total_assets(path)", "Count lines, update workspace assets", "int", "db_total_assets('{{Output}}/assets.txt')"},
|
||||
{FnDBTotalVulns, "db_total_vulns(path)", "Count lines, update workspace vulns", "int", "db_total_vulns('{{Output}}/vulns.txt')"},
|
||||
{FnDBVulnCritical, "db_vuln_critical(path)", "Count critical vulns", "int", "db_vuln_critical('{{Output}}/nuclei.json')"},
|
||||
{FnDBVulnHigh, "db_vuln_high(path)", "Count high vulns", "int", "db_vuln_high('{{Output}}/nuclei.json')"},
|
||||
{FnDBVulnMedium, "db_vuln_medium(path)", "Count medium vulns", "int", "db_vuln_medium('{{Output}}/nuclei.json')"},
|
||||
{FnDBVulnLow, "db_vuln_low(path)", "Count low vulns", "int", "db_vuln_low('{{Output}}/nuclei.json')"},
|
||||
{FnDBTotalIPs, "db_total_ips(path)", "Count lines, update workspace IPs (+=, 0 to reset)", "int", "db_total_ips('{{Output}}/ips.txt')"},
|
||||
{FnDBTotalLinks, "db_total_links(path)", "Count lines, update workspace links (+=, 0 to reset)", "int", "db_total_links('{{Output}}/links.txt')"},
|
||||
{FnDBTotalContent, "db_total_content(path)", "Count lines, update workspace content (+=, 0 to reset)", "int", "db_total_content('{{Output}}/content.txt')"},
|
||||
{FnDBTotalArchive, "db_total_archive(path)", "Count lines, update workspace archive (+=, 0 to reset)", "int", "db_total_archive('{{Output}}/archive.txt')"},
|
||||
{FnRuntimeExport, "runtime_export()", "Export scan+workspace to run-state.json", "bool", "runtime_export()"},
|
||||
{FnDBSelectAssets, "db_select_assets(workspace, format)", "Select assets (markdown/jsonl)", "string", "db_select_assets('{{Workspace}}', 'markdown')"},
|
||||
{FnDBSelectAssetsFiltered, "db_select_assets_filtered(workspace, status_code, asset_type, format)", "Select assets with filters", "string", "db_select_assets_filtered('{{Workspace}}', '200', 'subdomain', 'jsonl')"},
|
||||
{FnDBSelectVulnerabilities, "db_select_vulnerabilities(workspace, format)", "Select vulnerabilities (markdown/jsonl)", "string", "db_select_vulnerabilities('{{Workspace}}', 'markdown')"},
|
||||
{FnDBSelectVulnerabilitiesFiltered, "db_select_vulnerabilities_filtered(workspace, severity, asset_value, format)", "Select vulns with filters", "string", "db_select_vulnerabilities_filtered('{{Workspace}}', 'critical', '', 'jsonl')"},
|
||||
{FnDBSelect, "db_select(sql_query, format)", "Execute SELECT query (markdown/jsonl)", "string", "db_select('SELECT * FROM assets LIMIT 10', 'markdown')"},
|
||||
{FnDBSelectToFile, "db_select_to_file(sql_query, dest)", "Execute SELECT and write markdown to file", "bool", "db_select_to_file('SELECT * FROM assets', '{{Output}}/assets.md')"},
|
||||
{FnDBSelectToJSONL, "db_select_to_jsonl(sql_query, fields, dest)", "Execute SELECT and write JSONL with specified fields to file", "bool", "db_select_to_jsonl('SELECT * FROM assets', 'asset_value,status_code', '{{Output}}/assets.jsonl')"},
|
||||
{FnDBSelectTotalSubdomains, "db_select_total_subdomains()", "Get total subdomains from workspace", "int", "db_select_total_subdomains()"},
|
||||
{FnDBSelectTotalURLs, "db_select_total_urls()", "Get total URLs from workspace", "int", "db_select_total_urls()"},
|
||||
{FnDBSelectTotalAssets, "db_select_total_assets()", "Get total assets from workspace", "int", "db_select_total_assets()"},
|
||||
{FnDBSelectTotalVulns, "db_select_total_vulns()", "Get total vulns from workspace", "int", "db_select_total_vulns()"},
|
||||
{FnDBSelectVulnCritical, "db_select_vuln_critical()", "Get critical vuln count from workspace", "int", "db_select_vuln_critical()"},
|
||||
{FnDBSelectVulnHigh, "db_select_vuln_high()", "Get high vuln count from workspace", "int", "db_select_vuln_high()"},
|
||||
{FnDBSelectVulnMedium, "db_select_vuln_medium()", "Get medium vuln count from workspace", "int", "db_select_vuln_medium()"},
|
||||
{FnDBSelectVulnLow, "db_select_vuln_low()", "Get low vuln count from workspace", "int", "db_select_vuln_low()"},
|
||||
{FnDBImportAssetFromFile, "db_import_asset_from_file(workspace, file_path)", "Import assets from JSONL file (httpx format)", "int", "db_import_asset_from_file('{{Workspace}}', '{{Output}}/httpx.jsonl')"},
|
||||
{FnDBImportVuln, "db_import_vuln(workspace, json_data)", "Import single vulnerability from JSON (nuclei format)", "bool", "db_import_vuln('{{Workspace}}', '{\"template-id\":\"...\",\"info\":{\"name\":\"...\",\"severity\":\"high\"}}')"},
|
||||
{FnDBImportVulnFromFile, "db_import_vuln_from_file(workspace, file_path)", "Import vulnerabilities from JSONL file (nuclei format)", "int", "db_import_vuln_from_file('{{Workspace}}', '{{Output}}/nuclei.jsonl')"},
|
||||
},
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user