From d44c94ef4af589ba32526df2a013a77fa3c1d2ff Mon Sep 17 00:00:00 2001 From: j3ssie Date: Tue, 20 Jan 2026 21:24:52 +0800 Subject: [PATCH] feat: enhance documentation and server startup flow - Add comprehensive API and CLI documentation for event receiver, asset/vulnerability diffing, settings management, and function evaluation - Improve event receiver startup by separating StartEventReceiver() from StartListener() for better control over trigger registration timing - Update README with new command examples for db list, func eval, and event-driven triggers - Refactor PrintStartupInfo to display detailed trigger information with tree formatting and better visual hierarchy - Change trigger registration logging from Info to Debug level to reduce noise - Clarify --workspace flag documentation in run/scan commands - Add Type field to EventReceiverTriggerInfo for better trigger visibility in API responses --- .github/workflows/nightly-release.yaml | 60 ++++ HACKING.md | 122 +++++++- README.md | 13 +- docs/api-swagger/docs.go | 383 +++++++++++++++++++++++++ docs/api-swagger/swagger.json | 383 +++++++++++++++++++++++++ docs/api-swagger/swagger.yaml | 254 ++++++++++++++++ internal/scheduler/scheduler.go | 4 +- pkg/cli/run.go | 2 +- pkg/cli/scan.go | 2 +- pkg/cli/server.go | 10 +- pkg/server/eventreceiver.go | 1 + pkg/server/handlers/event_receiver.go | 1 + pkg/server/server.go | 75 +++-- 13 files changed, 1276 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/nightly-release.yaml diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml new file mode 100644 index 0000000..3f55747 --- /dev/null +++ b/.github/workflows/nightly-release.yaml @@ -0,0 +1,60 @@ +name: Nightly Release + +on: + schedule: + # Run every Sunday at 00:00 UTC + - cron: '0 0 * * 0' + workflow_dispatch: + # Allow manual trigger + +permissions: + contents: write + +jobs: + nightly-release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.25' + + - name: Delete existing nightly release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Delete release if exists (tag will be force-updated) + gh release delete nightly --yes || true + + - name: Create or update nightly tag + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + # Use -f to force update if tag exists + git tag -fa nightly -m "Nightly build $(date -u +%Y-%m-%d)" + git push origin nightly --force + + - name: Create release notes + run: | + cat << 'EOF' > /tmp/release-notes.md + ## Nightly Build + + > **Note**: This is a nightly build, not a tagged release. Even though you enjoy the latest features, things may change when the official release. + + Built from commit: ${{ github.sha }} + EOF + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: '~> v2' + args: release --clean --release-notes=/tmp/release-notes.md + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GORELEASER_CURRENT_TAG: nightly diff --git a/HACKING.md b/HACKING.md index c5b47ce..abe64f4 100644 --- a/HACKING.md +++ b/HACKING.md @@ -18,6 +18,7 @@ This document describes the technical architecture and development practices for - [Database Layer](#database-layer) - [Testing](#testing) - [Adding New Features](#adding-new-features) +- [CLI Shortcuts and Tips](#cli-shortcuts-and-tips) ## Project Structure @@ -63,10 +64,11 @@ osmedeus-ng/ │ │ └── context.go # Template context │ ├── functions/ # Utility functions │ │ ├── registry.go # Function registry -│ │ ├── otto_runtime.go # JavaScript runtime +│ │ ├── goja_runtime.go # JavaScript runtime (Goja VM) │ │ ├── file_functions.go # File operations │ │ ├── string_functions.go # String operations │ │ ├── util_functions.go # Utility functions +│ │ ├── event_functions.go # Event generation functions │ │ └── jq.go # JSON query functions │ ├── scheduler/ # Trigger scheduling │ │ └── scheduler.go # Cron, event, watch triggers @@ -110,9 +112,10 @@ osmedeus-ng/ ├── pkg/ # Public packages │ ├── cli/ # CLI commands │ │ ├── root.go # Root command -│ │ ├── scan.go # Scan command +│ │ ├── run.go # Run command (scan) │ │ ├── workflow.go # Workflow command -│ │ ├── function.go # Function command +│ │ ├── function.go # Function command (with bulk processing) +│ │ ├── db.go # Database CLI command │ │ └── server.go # Server command │ └── server/ # REST API │ ├── server.go # Server setup @@ -667,6 +670,42 @@ steps: function: cat_file("{{Output}}/results.txt") ``` +### Event Functions + +These functions enable event-driven workflows by generating and emitting events: + +```go +// internal/functions/event_functions.go + +// generate_event emits a single structured event +// Usage: generate_event(workspace, topic, source, data_type, data) +func (vf *vmFunc) generateEvent(call goja.FunctionCall) goja.Value + +// generate_event_from_file emits an event for each line in a file +// Usage: generate_event_from_file(workspace, topic, source, data_type, filePath) +func (vf *vmFunc) generateEventFromFile(call goja.FunctionCall) goja.Value +``` + +Usage in workflows: +```yaml +steps: + - name: emit-single-event + type: function + function: | + generate_event("{{Workspace}}", "assets.new", "scanner", "subdomain", "api.example.com") + + - name: emit-from-file + type: function + function: | + generate_event_from_file("{{Workspace}}", "assets.new", "recon", "subdomain", "{{Output}}/subdomains.txt") +``` + +Event delivery uses a fallback chain: +1. **Server API** - POST to `/osm/api/events/emit` if server configured +2. **Redis Pub/Sub** - Publish to `osm:events:{topic}` in distributed mode +3. **Database Queue** - Store in `event_logs` table with `processed=false` +4. **Webhooks** - Send to configured webhook endpoints + ### Function Execution ```go @@ -1265,6 +1304,83 @@ func init() { - `osmedeus func` - alias for `osmedeus function` - `osmedeus func e` - alias for `osmedeus function eval` +- `osmedeus db ls` - alias for `osmedeus db list` + +### Database CLI Commands + +Query and manage database tables directly from the CLI: + +```bash +# List all tables with row counts +osmedeus db list + +# Query specific table (default columns shown) +osmedeus db list --table event_logs + +# List available columns for a table +osmedeus db list --table event_logs --list-columns + +# Filter by specific columns +osmedeus db list --table event_logs --columns topic,source,data_type,data + +# Show all columns including hidden ones (id, timestamps) +osmedeus db list --table event_logs --all + +# Filter by field value +osmedeus db list --table event_logs --where topic=assets.new +osmedeus db list --table event_logs --where processed=false + +# Search across all columns +osmedeus db list --table event_logs --search "nuclei" + +# Output as JSON for scripting +osmedeus db list --table event_logs --json + +# Pagination +osmedeus db list --table event_logs --offset 50 --limit 100 +``` + +Default columns per table: +- `runs`: run_id, job_id, workflow_name, target, status, started_at +- `event_logs`: topic, source, processed, data_type, workspace, data +- `assets`: asset_value, host_ip, title, status_code, last_seen_at, technologies +- `schedules`: name, workflow_name, trigger_type, schedule, is_enabled, run_count + +### Function Evaluation CLI + +Evaluate utility functions from the command line with bulk processing support: + +```bash +# Single expression evaluation +osmedeus func eval 'log_info("hello")' +osmedeus func eval -e 'fileLength("/path/to/file.txt")' + +# With target variable +osmedeus func eval -e 'httpGet("https://" + target)' -t example.com + +# Bulk processing from file (target variable available in script) +osmedeus func eval -e 'log_info("Processing: " + target)' -T targets.txt + +# Bulk processing with concurrency +osmedeus func eval -e 'httpGet("https://" + target)' -T targets.txt -c 10 + +# Using function files for reusable logic +osmedeus func eval --function-file check-host.js -T targets.txt -c 5 + +# Additional parameters +osmedeus func eval -e 'log_info(target + " in " + ws)' -T targets.txt --params ws=production + +# Function name with arguments +osmedeus func eval log_info "hello world" +osmedeus func eval -f httpGet "https://example.com" + +# Read script from stdin +echo 'log_info("hello")' | osmedeus func eval --stdin + +# List available functions +osmedeus func list +osmedeus func list event # Filter by category +``` ### New Scan Flags diff --git a/README.md b/README.md index 7f4f647..400f064 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,10 @@ Built for both beginners and experts, it delivers powerful, composable automatio - **Two Workflow Types** - Modules for single execution units, Flows for multi-module orchestration - **Multiple Runners** - Execute on local host, Docker containers, or remote machines via SSH - **Distributed Execution** - Scale with Redis-based master-worker pattern for parallel scanning -- **Event-Driven Triggers** - Cron scheduling, file watching, and event-based workflow triggers +- **Event-Driven Triggers** - Cron scheduling, file watching, and event-based workflow triggers with deduplication - **Decision Routing** - Conditional workflow branching with switch/case syntax - **Template Engine** - Powerful variable interpolation with built-in and custom variables -- **Utility Functions** - Rich function library for file operations, string manipulation, and JSON processing +- **Utility Functions** - Rich function library with event generation, bulk processing, and JSON operations - **REST API Server** - Manage and trigger workflows programmatically - **Database Support** - SQLite (default) and PostgreSQL for asset tracking - **Notifications** - Telegram bot and webhook integrations @@ -67,6 +67,14 @@ osmedeus serve # List available workflows osmedeus workflow list +# Query database tables +osmedeus db list --table runs +osmedeus db list --table event_logs --search "nuclei" + +# Evaluate utility functions +osmedeus func eval 'log_info("hello")' +osmedeus func eval -e 'httpGet("https://example.com")' -T targets.txt -c 10 + # Show all usage examples osmedeus --usage-example ``` @@ -188,6 +196,7 @@ The high-level ambitious plan for the project, in order: | Getting Started | [docs.osmedeus.org/getting-started](https://docs.osmedeus.org/getting-started) | | CLI Usage & Examples | [docs.osmedeus.org/getting-started/cli](https://docs.osmedeus.org/getting-started/cli) | | Writing Workflows | [docs.osmedeus.org/workflows/overview](https://docs.osmedeus.org/workflows/overview) | +| Event-Driven Triggers| [docs.osmedeus.org/advanced/event-driven](https://docs.osmedeus.org/advanced/event-driven) | | Deployment | [docs.osmedeus.org/deployment](https://docs.osmedeus.org/deployment) | | Architecture | [docs.osmedeus.org/concepts/architecture](https://docs.osmedeus.org/concepts/architecture) | | Development | [docs.osmedeus.org/development](https://docs.osmedeus.org/development) and [HACKING.md](HACKING.md) | diff --git a/docs/api-swagger/docs.go b/docs/api-swagger/docs.go index 30177e7..1c422c6 100644 --- a/docs/api-swagger/docs.go +++ b/docs/api-swagger/docs.go @@ -294,6 +294,68 @@ const docTemplate = `{ } } }, + "/osm/api/assets/diff": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Compare assets between two time points to find added, removed, and changed assets", + "produces": [ + "application/json" + ], + "tags": [ + "Assets" + ], + "summary": "Get asset diff", + "parameters": [ + { + "type": "string", + "description": "Workspace name", + "name": "workspace", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Start time (RFC3339 format or Unix timestamp)", + "name": "from", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "End time (default: now)", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Asset diff result", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Invalid parameters", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "500": { + "description": "Failed to get asset diff", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/osm/api/event-logs": { "get": { "security": [ @@ -385,6 +447,93 @@ const docTemplate = `{ } } }, + "/osm/api/event-receiver/status": { + "get": { + "description": "Returns the status of the event receiver including enabled state and counts", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "event-receiver" + ], + "summary": "Get event receiver status", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handlers.EventReceiverStatusResponse" + } + } + } + } + }, + "/osm/api/event-receiver/workflows": { + "get": { + "description": "Returns all workflows registered with the event receiver", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "event-receiver" + ], + "summary": "List event receiver workflows", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handlers.EventReceiverWorkflowsResponse" + } + } + } + } + }, + "/osm/api/events/emit": { + "post": { + "description": "Emits an event that can trigger event-triggered workflows", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "event-receiver" + ], + "summary": "Emit an event", + "parameters": [ + { + "description": "Event to emit", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/handlers.EmitEventRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handlers.EmitEventResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/osm/api/functions/eval": { "post": { "security": [ @@ -1294,6 +1443,72 @@ const docTemplate = `{ } } }, + "/osm/api/settings/reload": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Forces an immediate reload of the configuration file. Hot reload must be enabled.", + "produces": [ + "application/json" + ], + "tags": [ + "Settings" + ], + "summary": "Force config reload", + "responses": { + "200": { + "description": "Configuration reloaded successfully", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Hot reload not enabled", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "500": { + "description": "Failed to reload configuration", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, + "/osm/api/settings/status": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Returns the current configuration version and hot reload status", + "produces": [ + "application/json" + ], + "tags": [ + "Settings" + ], + "summary": "Get config status", + "responses": { + "200": { + "description": "Configuration status", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/osm/api/settings/yaml": { "get": { "security": [ @@ -1921,6 +2136,68 @@ const docTemplate = `{ } } }, + "/osm/api/vulnerabilities/diff": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Compare vulnerabilities between two time points to find added, removed, and changed vulnerabilities", + "produces": [ + "application/json" + ], + "tags": [ + "Vulnerabilities" + ], + "summary": "Get vulnerability diff", + "parameters": [ + { + "type": "string", + "description": "Workspace name", + "name": "workspace", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Start time (RFC3339 format or Unix timestamp)", + "name": "from", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "End time (default: now)", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Vulnerability diff result", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Invalid parameters", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "500": { + "description": "Failed to get vulnerability diff", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/osm/api/vulnerabilities/summary": { "get": { "security": [ @@ -2850,6 +3127,112 @@ const docTemplate = `{ } } }, + "handlers.EmitEventRequest": { + "type": "object", + "properties": { + "data": { + "type": "object", + "additionalProperties": true + }, + "data_type": { + "type": "string" + }, + "name": { + "type": "string" + }, + "source": { + "type": "string" + }, + "topic": { + "type": "string" + } + } + }, + "handlers.EmitEventResponse": { + "type": "object", + "properties": { + "event_id": { + "type": "string" + }, + "message": { + "type": "string" + }, + "success": { + "type": "boolean" + } + } + }, + "handlers.EventReceiverStatusResponse": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "running": { + "type": "boolean" + }, + "trigger_count": { + "type": "integer" + }, + "workflow_count": { + "type": "integer" + } + } + }, + "handlers.EventReceiverTriggerResponse": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "topic": { + "type": "string" + }, + "trigger_name": { + "type": "string" + }, + "workflow_name": { + "type": "string" + } + } + }, + "handlers.EventReceiverWorkflowResponse": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "name": { + "type": "string" + }, + "triggers": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "handlers.EventReceiverWorkflowsResponse": { + "type": "object", + "properties": { + "triggers": { + "type": "array", + "items": { + "$ref": "#/definitions/handlers.EventReceiverTriggerResponse" + } + }, + "workflows": { + "type": "array", + "items": { + "$ref": "#/definitions/handlers.EventReceiverWorkflowResponse" + } + } + } + }, "handlers.FunctionEvalRequest": { "type": "object", "properties": { diff --git a/docs/api-swagger/swagger.json b/docs/api-swagger/swagger.json index 69f38fa..c7114cd 100644 --- a/docs/api-swagger/swagger.json +++ b/docs/api-swagger/swagger.json @@ -283,6 +283,68 @@ } } }, + "/osm/api/assets/diff": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Compare assets between two time points to find added, removed, and changed assets", + "produces": [ + "application/json" + ], + "tags": [ + "Assets" + ], + "summary": "Get asset diff", + "parameters": [ + { + "type": "string", + "description": "Workspace name", + "name": "workspace", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Start time (RFC3339 format or Unix timestamp)", + "name": "from", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "End time (default: now)", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Asset diff result", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Invalid parameters", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "500": { + "description": "Failed to get asset diff", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/osm/api/event-logs": { "get": { "security": [ @@ -374,6 +436,93 @@ } } }, + "/osm/api/event-receiver/status": { + "get": { + "description": "Returns the status of the event receiver including enabled state and counts", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "event-receiver" + ], + "summary": "Get event receiver status", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handlers.EventReceiverStatusResponse" + } + } + } + } + }, + "/osm/api/event-receiver/workflows": { + "get": { + "description": "Returns all workflows registered with the event receiver", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "event-receiver" + ], + "summary": "List event receiver workflows", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handlers.EventReceiverWorkflowsResponse" + } + } + } + } + }, + "/osm/api/events/emit": { + "post": { + "description": "Emits an event that can trigger event-triggered workflows", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "event-receiver" + ], + "summary": "Emit an event", + "parameters": [ + { + "description": "Event to emit", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/handlers.EmitEventRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handlers.EmitEventResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/osm/api/functions/eval": { "post": { "security": [ @@ -1283,6 +1432,72 @@ } } }, + "/osm/api/settings/reload": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Forces an immediate reload of the configuration file. Hot reload must be enabled.", + "produces": [ + "application/json" + ], + "tags": [ + "Settings" + ], + "summary": "Force config reload", + "responses": { + "200": { + "description": "Configuration reloaded successfully", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Hot reload not enabled", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "500": { + "description": "Failed to reload configuration", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, + "/osm/api/settings/status": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Returns the current configuration version and hot reload status", + "produces": [ + "application/json" + ], + "tags": [ + "Settings" + ], + "summary": "Get config status", + "responses": { + "200": { + "description": "Configuration status", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/osm/api/settings/yaml": { "get": { "security": [ @@ -1910,6 +2125,68 @@ } } }, + "/osm/api/vulnerabilities/diff": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Compare vulnerabilities between two time points to find added, removed, and changed vulnerabilities", + "produces": [ + "application/json" + ], + "tags": [ + "Vulnerabilities" + ], + "summary": "Get vulnerability diff", + "parameters": [ + { + "type": "string", + "description": "Workspace name", + "name": "workspace", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Start time (RFC3339 format or Unix timestamp)", + "name": "from", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "End time (default: now)", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Vulnerability diff result", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Invalid parameters", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "500": { + "description": "Failed to get vulnerability diff", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/osm/api/vulnerabilities/summary": { "get": { "security": [ @@ -2839,6 +3116,112 @@ } } }, + "handlers.EmitEventRequest": { + "type": "object", + "properties": { + "data": { + "type": "object", + "additionalProperties": true + }, + "data_type": { + "type": "string" + }, + "name": { + "type": "string" + }, + "source": { + "type": "string" + }, + "topic": { + "type": "string" + } + } + }, + "handlers.EmitEventResponse": { + "type": "object", + "properties": { + "event_id": { + "type": "string" + }, + "message": { + "type": "string" + }, + "success": { + "type": "boolean" + } + } + }, + "handlers.EventReceiverStatusResponse": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "running": { + "type": "boolean" + }, + "trigger_count": { + "type": "integer" + }, + "workflow_count": { + "type": "integer" + } + } + }, + "handlers.EventReceiverTriggerResponse": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "topic": { + "type": "string" + }, + "trigger_name": { + "type": "string" + }, + "workflow_name": { + "type": "string" + } + } + }, + "handlers.EventReceiverWorkflowResponse": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "name": { + "type": "string" + }, + "triggers": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "handlers.EventReceiverWorkflowsResponse": { + "type": "object", + "properties": { + "triggers": { + "type": "array", + "items": { + "$ref": "#/definitions/handlers.EventReceiverTriggerResponse" + } + }, + "workflows": { + "type": "array", + "items": { + "$ref": "#/definitions/handlers.EventReceiverWorkflowResponse" + } + } + } + }, "handlers.FunctionEvalRequest": { "type": "object", "properties": { diff --git a/docs/api-swagger/swagger.yaml b/docs/api-swagger/swagger.yaml index 6396976..7784f4d 100644 --- a/docs/api-swagger/swagger.yaml +++ b/docs/api-swagger/swagger.yaml @@ -255,6 +255,75 @@ definitions: workspace: type: string type: object + handlers.EmitEventRequest: + properties: + data: + additionalProperties: true + type: object + data_type: + type: string + name: + type: string + source: + type: string + topic: + type: string + type: object + handlers.EmitEventResponse: + properties: + event_id: + type: string + message: + type: string + success: + type: boolean + type: object + handlers.EventReceiverStatusResponse: + properties: + enabled: + type: boolean + running: + type: boolean + trigger_count: + type: integer + workflow_count: + type: integer + type: object + handlers.EventReceiverTriggerResponse: + properties: + enabled: + type: boolean + topic: + type: string + trigger_name: + type: string + workflow_name: + type: string + type: object + handlers.EventReceiverWorkflowResponse: + properties: + description: + type: string + kind: + type: string + name: + type: string + triggers: + items: + type: string + type: array + type: object + handlers.EventReceiverWorkflowsResponse: + properties: + triggers: + items: + $ref: '#/definitions/handlers.EventReceiverTriggerResponse' + type: array + workflows: + items: + $ref: '#/definitions/handlers.EventReceiverWorkflowResponse' + type: array + type: object handlers.FunctionEvalRequest: properties: params: @@ -588,6 +657,48 @@ paths: summary: List assets tags: - Assets + /osm/api/assets/diff: + get: + description: Compare assets between two time points to find added, removed, + and changed assets + parameters: + - description: Workspace name + in: query + name: workspace + required: true + type: string + - description: Start time (RFC3339 format or Unix timestamp) + in: query + name: from + required: true + type: string + - description: 'End time (default: now)' + in: query + name: to + type: string + produces: + - application/json + responses: + "200": + description: Asset diff result + schema: + additionalProperties: true + type: object + "400": + description: Invalid parameters + schema: + additionalProperties: true + type: object + "500": + description: Failed to get asset diff + schema: + additionalProperties: true + type: object + security: + - BearerAuth: [] + summary: Get asset diff + tags: + - Assets /osm/api/event-logs: get: description: Get a paginated list of event logs with optional filtering @@ -648,6 +759,64 @@ paths: summary: List event logs tags: - EventLogs + /osm/api/event-receiver/status: + get: + consumes: + - application/json + description: Returns the status of the event receiver including enabled state + and counts + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/handlers.EventReceiverStatusResponse' + summary: Get event receiver status + tags: + - event-receiver + /osm/api/event-receiver/workflows: + get: + consumes: + - application/json + description: Returns all workflows registered with the event receiver + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/handlers.EventReceiverWorkflowsResponse' + summary: List event receiver workflows + tags: + - event-receiver + /osm/api/events/emit: + post: + consumes: + - application/json + description: Emits an event that can trigger event-triggered workflows + parameters: + - description: Event to emit + in: body + name: request + required: true + schema: + $ref: '#/definitions/handlers.EmitEventRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/handlers.EmitEventResponse' + "400": + description: Bad Request + schema: + additionalProperties: true + type: object + summary: Emit an event + tags: + - event-receiver /osm/api/functions/eval: post: consumes: @@ -1240,6 +1409,49 @@ paths: summary: Trigger a schedule tags: - Schedules + /osm/api/settings/reload: + post: + description: Forces an immediate reload of the configuration file. Hot reload + must be enabled. + produces: + - application/json + responses: + "200": + description: Configuration reloaded successfully + schema: + additionalProperties: true + type: object + "400": + description: Hot reload not enabled + schema: + additionalProperties: true + type: object + "500": + description: Failed to reload configuration + schema: + additionalProperties: true + type: object + security: + - BearerAuth: [] + summary: Force config reload + tags: + - Settings + /osm/api/settings/status: + get: + description: Returns the current configuration version and hot reload status + produces: + - application/json + responses: + "200": + description: Configuration status + schema: + additionalProperties: true + type: object + security: + - BearerAuth: [] + summary: Get config status + tags: + - Settings /osm/api/settings/yaml: get: description: Returns the entire configuration file with sensitive fields redacted @@ -1723,6 +1935,48 @@ paths: summary: Get vulnerability by ID tags: - Vulnerabilities + /osm/api/vulnerabilities/diff: + get: + description: Compare vulnerabilities between two time points to find added, + removed, and changed vulnerabilities + parameters: + - description: Workspace name + in: query + name: workspace + required: true + type: string + - description: Start time (RFC3339 format or Unix timestamp) + in: query + name: from + required: true + type: string + - description: 'End time (default: now)' + in: query + name: to + type: string + produces: + - application/json + responses: + "200": + description: Vulnerability diff result + schema: + additionalProperties: true + type: object + "400": + description: Invalid parameters + schema: + additionalProperties: true + type: object + "500": + description: Failed to get vulnerability diff + schema: + additionalProperties: true + type: object + security: + - BearerAuth: [] + summary: Get vulnerability diff + tags: + - Vulnerabilities /osm/api/vulnerabilities/summary: get: description: Get a summary of vulnerabilities grouped by severity diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 47230cd..2dc77cc 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -200,7 +200,7 @@ func (s *Scheduler) RegisterTrigger(workflow *core.Workflow, trigger *core.Trigg } s.triggers[triggerKey] = reg - s.logger.Info("Registered trigger", + s.logger.Debug("Registered trigger", zap.String("workflow", workflow.Name), zap.String("trigger", trigger.Name), zap.String("type", string(trigger.On)), @@ -429,7 +429,7 @@ func (s *Scheduler) Start() error { // Start dedupe cache cleanup goroutine go s.dedupeCleanupLoop() - s.logger.Info("Scheduler started") + s.logger.Debug("Scheduler started") return nil } diff --git a/pkg/cli/run.go b/pkg/cli/run.go index 3c05137..5a5b5f6 100644 --- a/pkg/cli/run.go +++ b/pkg/cli/run.go @@ -93,7 +93,7 @@ func init() { runCmd.Flags().StringVarP(&targetFile, "target-file", "T", "", "file containing targets (one per line)") runCmd.Flags().StringArrayVarP(¶mFlags, "params", "p", nil, "additional parameters (key=value format)") runCmd.Flags().StringVarP(¶msFile, "params-file", "P", "", "file containing parameters (JSON or YAML key:value pairs)") - runCmd.Flags().StringVarP(&workspacePath, "workspace", "w", "", "custom workspace path") + runCmd.Flags().StringVarP(&workspacePath, "workspace", "w", "", "custom workspace path (overrides {{TargetSpace}})") runCmd.Flags().BoolVar(&dryRun, "dry-run", false, "show what would be executed without running commands") runCmd.Flags().IntVar(&threadsHold, "threads-hold", 0, "override thread count (0 = use tactic default)") runCmd.Flags().IntVarP(&concurrency, "concurrency", "c", 1, "number of targets to run concurrently") diff --git a/pkg/cli/scan.go b/pkg/cli/scan.go index 866b85a..8dab950 100644 --- a/pkg/cli/scan.go +++ b/pkg/cli/scan.go @@ -34,7 +34,7 @@ func init() { scanCmd.Flags().StringVarP(&targetFile, "target-file", "T", "", "file containing targets (one per line)") scanCmd.Flags().StringArrayVarP(¶mFlags, "params", "p", nil, "additional parameters (key=value format)") scanCmd.Flags().StringVarP(¶msFile, "params-file", "P", "", "file containing parameters (JSON or YAML key:value pairs)") - scanCmd.Flags().StringVarP(&workspacePath, "workspace", "w", "", "custom workspace path") + scanCmd.Flags().StringVarP(&workspacePath, "workspace", "w", "", "custom workspace path (overrides {{TargetSpace}})") scanCmd.Flags().BoolVar(&dryRun, "dry-run", false, "show what would be executed without running commands") scanCmd.Flags().IntVar(&threadsHold, "threads-hold", 0, "override thread count (0 = use tactic default)") scanCmd.Flags().IntVarP(&concurrency, "concurrency", "c", 1, "number of targets to run concurrently") diff --git a/pkg/cli/server.go b/pkg/cli/server.go index 17606ef..0cf2cd1 100644 --- a/pkg/cli/server.go +++ b/pkg/cli/server.go @@ -136,14 +136,18 @@ func runServer(cmd *cobra.Command, args []string) error { log.Info("Debug mode enabled - request bodies and detailed errors will be logged") } - // Print startup info before starting the server addr := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port) + + // Start event receiver first (registers triggers synchronously) + srv.StartEventReceiver() + + // Print startup info (triggers are now registered) srv.PrintStartupInfo(addr) - // Start server in goroutine + // Start HTTP listener in goroutine serverErr := make(chan error, 1) go func() { - serverErr <- srv.Start(addr) + serverErr <- srv.StartListener(addr) }() // Wait for shutdown or server error diff --git a/pkg/server/eventreceiver.go b/pkg/server/eventreceiver.go index 5f9fd89..446ff97 100644 --- a/pkg/server/eventreceiver.go +++ b/pkg/server/eventreceiver.go @@ -278,6 +278,7 @@ func (er *EventReceiver) GetRegisteredTriggersInfo() []*handlers.EventReceiverTr result = append(result, &handlers.EventReceiverTriggerInfo{ WorkflowName: t.Workflow.Name, TriggerName: t.Trigger.Name, + Type: string(t.Trigger.On), Topic: topic, Enabled: t.Trigger.Enabled, }) diff --git a/pkg/server/handlers/event_receiver.go b/pkg/server/handlers/event_receiver.go index b99d10f..e1e5a28 100644 --- a/pkg/server/handlers/event_receiver.go +++ b/pkg/server/handlers/event_receiver.go @@ -21,6 +21,7 @@ type EventReceiverProvider interface { type EventReceiverTriggerInfo struct { WorkflowName string TriggerName string + Type string // "event", "cron", "watch", "manual" Topic string Enabled bool } diff --git a/pkg/server/server.go b/pkg/server/server.go index 4969a91..68c7b78 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -6,7 +6,6 @@ import ( "net/http" "os" "runtime" - "sort" "strings" "time" @@ -408,17 +407,28 @@ func (s *Server) serveEmbeddedUI() { })) } -// Start starts the server -func (s *Server) Start(addr string) error { - // Start event receiver if configured +// StartEventReceiver starts the event receiver and scheduler. +// This should be called before PrintStartupInfo to ensure triggers are registered. +func (s *Server) StartEventReceiver() { if s.eventReceiver != nil { if err := s.eventReceiver.Start(context.Background()); err != nil { oslogger.Get().Warn("Failed to start event receiver", zap.Error(err)) } } +} + +// StartListener starts only the HTTP listener (event receiver already started). +func (s *Server) StartListener(addr string) error { return s.app.Listen(addr) } +// Start starts the server (event receiver + HTTP listener). +// For finer control, use StartEventReceiver() + StartListener() instead. +func (s *Server) Start(addr string) error { + s.StartEventReceiver() + return s.StartListener(addr) +} + // Shutdown gracefully shuts down the server func (s *Server) Shutdown() error { // Stop event receiver if running @@ -500,6 +510,11 @@ func startMemoryMetricsCollector(ctx context.Context, interval time.Duration) { func (s *Server) PrintStartupInfo(addr string) { p := terminal.NewPrinter() + // Banner line + fmt.Printf("%s Initiating Osmedeus %s - Crafted with <3 by @j3ssie\n", + terminal.Yellow("ϟ"), + terminal.Cyan(core.VERSION)) + // Starting server p.Info("Starting Osmedeus server %s", terminal.Cyan("http://"+addr)) @@ -508,32 +523,48 @@ func (s *Server) PrintStartupInfo(addr string) { // Hot reload status if s.hotConfig != nil { - p.Info("Started config hot reload watcher %s", terminal.Cyan(s.hotConfig.GetConfigPath())) + p.Info("Started config hot reload watcher %s", terminal.Yellow(s.hotConfig.GetConfigPath())) } - // Event receiver info + // Workflows path (always show, in yellow) + p.Info("Workflows loaded from %s", terminal.Yellow(s.config.GetWorkflowsDir())) + + // Event receiver info with detailed triggers if s.eventReceiver != nil { triggers := s.eventReceiver.GetRegisteredTriggersInfo() - if len(triggers) > 0 { - // Collect unique topics - topicSet := make(map[string]struct{}) - for _, t := range triggers { - if t.Topic != "" { - topicSet[t.Topic] = struct{}{} - } - } - topics := make([]string, 0, len(topicSet)) - for topic := range topicSet { - topics = append(topics, topic) - } - sort.Strings(topics) + if len(triggers) > 0 { p.Info("Event receiver initialized") - p.Info("Loaded %s workflows with event triggers: %s", - terminal.Cyan(fmt.Sprintf("%d", len(s.eventReceiver.GetRegisteredWorkflows()))), - terminal.Cyan(strings.Join(topics, ", "))) + + // Show each registered trigger with tree formatting + for i, t := range triggers { + prefix := "├─" + if i == len(triggers)-1 { + prefix = "└─" + } + + // Format trigger details based on type + var detail string + if t.Topic != "" { + detail = fmt.Sprintf("%s: %s", terminal.Blue("event"), terminal.Blue(t.Topic)) + } else { + detail = terminal.Blue(t.Type) + } + + fmt.Printf(" %s Registered trigger: %s (%s)\n", + prefix, + terminal.Yellow(t.WorkflowName), + detail) + } + + // Scheduler started message + p.Info("Scheduler started with %s triggers", terminal.Cyan(fmt.Sprintf("%d", len(triggers)))) } else { p.Info("Event receiver initialized %s", terminal.Gray("(no event triggers)")) + p.Info("Scheduler started") } + } else { + // No event receiver, but still show scheduler status + p.Info("Scheduler started") } }