From 8ffbc91d432eaa64acfa6c2c3ec72c687dcda206 Mon Sep 17 00:00:00 2001 From: j3ssie Date: Sun, 25 Jan 2026 16:43:36 +0800 Subject: [PATCH] feat: add cdn tree listing and webhook config setters --- .gitignore | 1 + README.md | 13 +- internal/functions/cdn_functions.go | 190 ++++++++++++++++++++++ internal/functions/constants.go | 17 +- internal/functions/goja_runtime.go | 1 + internal/functions/installer_functions.go | 12 +- pkg/cli/config.go | 55 +++++++ pkg/cli/function.go | 34 +++- pkg/cli/usage.go | 8 +- 9 files changed, 309 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 4842786..4490b8a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ coverage.html .DS_Store .idea/ .vscode/ +.trae # cgo / compiler outputs *.o diff --git a/README.md b/README.md index fbacfe7..32e606a 100644 --- a/README.md +++ b/README.md @@ -23,20 +23,15 @@ Built for both beginners and experts, it delivers powerful, composable automatio ## Features - **Declarative YAML Workflows** - Define reconnaissance pipelines using simple, readable YAML syntax -- **Two Workflow Types** - Modules for single execution units, Flows for multi-module orchestration - **Multiple Runners** - Execute on local host, Docker containers, or remote machines via SSH -- **Distributed Execution** - Scale with Redis-based master-worker pattern for parallel scanning - **Event-Driven Triggers** - Cron scheduling, file watching, and event-based workflow triggers with deduplication and filter functions -- **Decision Routing** - Conditional workflow branching with switch/case syntax - **Template Engine** - Powerful variable interpolation with built-in and custom variables - **Utility Functions** - Rich function library with event generation, bulk processing, and JSON operations - **REST API Server** - Manage, trigger, and cancel workflows programmatically -- **Run Cancellation** - Cancel running workflows via API with process termination -- **Database Support** - SQLite (default) and PostgreSQL for asset tracking +- **Distributed Execution** - Scale with Redis-based master-worker pattern for parallel scanning - **Notifications** - Telegram bot and webhook integrations - **Cloud Storage** - S3-compatible storage for artifact management - **LLM Integration** - AI-powered workflow steps with chat completions and embeddings -- **Platform Detection** - Built-in variables for Docker, Kubernetes, and cloud provider detection See [Documentation Page](https://docs.osmedeus.org/) for more details. @@ -125,7 +120,7 @@ For more CLI usage and example commands, refer to the [CLI Reference](https://do └───────────────────────────────────────────────────────────────────────────┘ ``` -For writing your first workflow, refer to the [Workflow Overview](https://docs.osmedeus.org/workflows/overview). +For more information about the architecture, refer to the [Architecture Documentation](https://docs.osmedeus.org/architecture). ## Roadmap and Status @@ -137,8 +132,8 @@ The high-level ambitious plan for the project, in order: | 2 | Flexible workflows and step types | ✅ | | 3 | Event-driven architectural model and the different trigger event categories | ✅ | | 4 | Beautiful UI for visualize results and workflow diagram | ✅ | -| 5 | Rewriting the workflow to adapt to new architecture and syntax | ⚠️ | -| 6 | Testing more utility functions like notifications | ⚠️ | +| 5 | Rewriting the workflow to adapt to new architecture and syntax | ✅ | +| 6 | Testing more utility functions like notifications | ✅ | | 7 | Generate diff reports showing new/removed/unchanged assets between runs. | ❌ | | 8 | Adding step type from cloud provider that can be run via serverless | ❌ | | N | Fancy features (to be discussed later) | ❌ | diff --git a/internal/functions/cdn_functions.go b/internal/functions/cdn_functions.go index 26eb351..1552780 100644 --- a/internal/functions/cdn_functions.go +++ b/internal/functions/cdn_functions.go @@ -3,7 +3,9 @@ package functions import ( "context" "encoding/json" + "fmt" "path/filepath" + "sort" "strings" "time" @@ -11,6 +13,7 @@ import ( "github.com/j3ssie/osmedeus/v5/internal/config" "github.com/j3ssie/osmedeus/v5/internal/logger" "github.com/j3ssie/osmedeus/v5/internal/storage" + "github.com/j3ssie/osmedeus/v5/internal/terminal" "go.uber.org/zap" ) @@ -435,3 +438,190 @@ func (vf *vmFunc) cdnStat(call goja.FunctionCall) goja.Value { logger.Get().Debug("cdnStat result", zap.String("remotePath", remotePath), zap.Int64("size", info.Size)) return vf.vm.ToValue(string(jsonBytes)) } + +// cdnLsTree lists files from cloud storage in a tree format +// Usage: cdnLsTree(prefix?) -> string (tree format output) +func (vf *vmFunc) cdnLsTree(call goja.FunctionCall) goja.Value { + prefix := "" + if len(call.Arguments) > 0 && !goja.IsUndefined(call.Argument(0)) { + prefix = call.Argument(0).String() + if prefix == "undefined" { + prefix = "" + } + } + depth := int64(1) + if len(call.Arguments) > 1 && !goja.IsUndefined(call.Argument(1)) && !goja.IsNull(call.Argument(1)) { + depth = call.Argument(1).ToInteger() + if depth < 1 { + depth = 1 + } + } + logger.Get().Debug("Calling cdnLsTree", zap.String("prefix", prefix)) + + client, err := storage.GetClient() + if err != nil { + logger.Get().Warn("cdnLsTree: failed to get storage client", zap.Error(err)) + return vf.vm.ToValue("") + } + + ctx := context.Background() + files, err := client.ListWithInfo(ctx, prefix) + if err != nil { + logger.Get().Warn("cdnLsTree: list failed", zap.String("prefix", prefix), zap.Error(err)) + return vf.vm.ToValue("") + } + + // Build tree structure + tree := buildFileTree(files, prefix) + output := renderTree(tree, prefix, int(depth)) + + logger.Get().Debug("cdnLsTree result", zap.String("prefix", prefix), zap.Int("files", len(files))) + return vf.vm.ToValue(output) +} + +// treeNode represents a node in the file tree +type treeNode struct { + name string + isDir bool + size int64 + children map[string]*treeNode +} + +// buildFileTree builds a tree structure from flat file list +func buildFileTree(files []storage.FileInfo, prefix string) *treeNode { + root := &treeNode{ + name: prefix, + isDir: true, + children: make(map[string]*treeNode), + } + + for _, f := range files { + // Remove prefix from key for relative path + relPath := f.Key + if prefix != "" { + relPath = strings.TrimPrefix(f.Key, prefix) + } + relPath = strings.TrimPrefix(relPath, "/") + + if relPath == "" { + continue + } + + parts := strings.Split(relPath, "/") + current := root + + for i, part := range parts { + if part == "" { + continue + } + + if current.children[part] == nil { + isDir := i < len(parts)-1 + current.children[part] = &treeNode{ + name: part, + isDir: isDir, + children: make(map[string]*treeNode), + } + } + if i == len(parts)-1 { + current.children[part].size = f.Size + } + current = current.children[part] + } + } + + return root +} + +// renderTree renders the tree structure as a string +func renderTree(root *treeNode, prefix string, depth int) string { + var sb strings.Builder + + // Write root + rootName := prefix + if rootName == "" { + rootName = "." + } + sb.WriteString(terminal.Cyan(rootName)) + sb.WriteString("\n") + + // Get sorted children + if depth >= 1 { + children := getSortedChildren(root) + for i, child := range children { + isLast := i == len(children)-1 + renderTreeNode(&sb, child, "", isLast, depth, 1) + } + } + + return sb.String() +} + +// renderTreeNode recursively renders a tree node +func renderTreeNode(sb *strings.Builder, node *treeNode, indent string, isLast bool, depth, level int) { + // Choose connector + connector := "├── " + if isLast { + connector = "└── " + } + + sb.WriteString(indent) + sb.WriteString(connector) + if node.isDir { + sb.WriteString(terminal.Cyan(node.name)) + } else { + sb.WriteString(terminal.Green(node.name)) + } + if !node.isDir { + fmt.Fprintf(sb, " (%s)", formatSize(node.size)) + } + sb.WriteString("\n") + + // Update indent for children + childIndent := indent + if isLast { + childIndent += " " + } else { + childIndent += "│ " + } + + // Render children + if node.isDir && level < depth { + children := getSortedChildren(node) + for i, child := range children { + renderTreeNode(sb, child, childIndent, i == len(children)-1, depth, level+1) + } + } +} + +// getSortedChildren returns children sorted: directories first, then files, alphabetically +func getSortedChildren(node *treeNode) []*treeNode { + var dirs, files []*treeNode + for _, child := range node.children { + if child.isDir { + dirs = append(dirs, child) + } else { + files = append(files, child) + } + } + + // Sort each group alphabetically + sort.Slice(dirs, func(i, j int) bool { return dirs[i].name < dirs[j].name }) + sort.Slice(files, func(i, j int) bool { return files[i].name < files[j].name }) + + return append(dirs, files...) +} + +// formatSize formats bytes into human-readable size +func formatSize(bytes int64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} diff --git a/internal/functions/constants.go b/internal/functions/constants.go index 738a953..6cec938 100644 --- a/internal/functions/constants.go +++ b/internal/functions/constants.go @@ -128,14 +128,14 @@ const ( // Notification Functions - Send notifications via various channels const ( - FnNotifyTelegram = "notify_telegram" // notify_telegram(message) -> bool - FnSendTelegramFile = "send_telegram_file" // send_telegram_file(path, caption?) -> bool - FnNotifyTelegramChannel = "notify_telegram_channel" // notify_telegram_channel(channel, message) -> bool - FnSendTelegramFileChannel = "send_telegram_file_channel" // send_telegram_file_channel(channel, path, caption?) -> bool - FnNotifyMessageAsFileTelegram = "notify_message_as_file_telegram" // notify_message_as_file_telegram(path) -> bool + FnNotifyTelegram = "notify_telegram" // notify_telegram(message) -> bool + FnSendTelegramFile = "send_telegram_file" // send_telegram_file(path, caption?) -> bool + FnNotifyTelegramChannel = "notify_telegram_channel" // notify_telegram_channel(channel, message) -> bool + FnSendTelegramFileChannel = "send_telegram_file_channel" // send_telegram_file_channel(channel, path, caption?) -> bool + FnNotifyMessageAsFileTelegram = "notify_message_as_file_telegram" // notify_message_as_file_telegram(path) -> bool FnNotifyMessageAsFileTelegramChannel = "notify_message_as_file_telegram_channel" // notify_message_as_file_telegram_channel(channel, path) -> bool - FnNotifyWebhook = "notify_webhook" // notify_webhook(message) -> bool - FnSendWebhookEvent = "send_webhook_event" // send_webhook_event(eventType, data) -> bool + FnNotifyWebhook = "notify_webhook" // notify_webhook(message) -> bool + FnSendWebhookEvent = "send_webhook_event" // send_webhook_event(eventType, data) -> bool ) // Event Generation Functions - Generate structured events @@ -156,6 +156,7 @@ const ( FnCdnList = "cdn_list" // cdn_list(prefix?) -> []object FnCdnStat = "cdn_stat" // cdn_stat(remotePath) -> object|null FnCdnRead = "cdn_read" // cdn_read(remotePath) -> string + FnCdnLsTree = "cdn_ls_tree" // cdn_ls_tree(prefix?) -> string (tree format) ) // Unix Command Wrappers - Wrappers around common Unix commands @@ -417,6 +418,7 @@ func AllFunctions() []string { FnCdnList, FnCdnStat, FnCdnRead, + FnCdnLsTree, // Unix Command Wrappers FnSortUnix, @@ -723,6 +725,7 @@ func FunctionRegistry() map[string][]FunctionInfo { {FnCdnList, "cdn_list(pattern?)", "List files with metadata from cloud storage (supports glob patterns)", "JSON string", "cdn_list('scans/')"}, {FnCdnStat, "cdn_stat(remotePath)", "Get file metadata from cloud storage", "JSON string", "cdn_stat('scans/target/report.zip')"}, {FnCdnRead, "cdn_read(remotePath)", "Read file content from cloud storage", "string", "cdn_read('config/settings.yaml')"}, + {FnCdnLsTree, "cdn_ls_tree(prefix?, depth?)", "List files from cloud storage in tree format", "string", "cdn_ls_tree('scans/', 2)"}, }, CategoryUnixCommands: { {FnSortUnix, "sort_unix(input, output?)", "Sort file with LC_ALL=C sort -u", "bool", "sort_unix('{{Output}}/urls.txt')"}, diff --git a/internal/functions/goja_runtime.go b/internal/functions/goja_runtime.go index 5a70132..e4c3442 100644 --- a/internal/functions/goja_runtime.go +++ b/internal/functions/goja_runtime.go @@ -174,6 +174,7 @@ func (r *GojaRuntime) registerFunctionsOnVM(vm *goja.Runtime) { _ = vm.Set(FnCdnList, vf.cdnList) _ = vm.Set(FnCdnStat, vf.cdnStat) _ = vm.Set(FnCdnRead, vf.cdnRead) + _ = vm.Set(FnCdnLsTree, vf.cdnLsTree) // Unix command wrappers _ = vm.Set(FnSortUnix, vf.sortUnix) diff --git a/internal/functions/installer_functions.go b/internal/functions/installer_functions.go index 1498745..81a1b37 100644 --- a/internal/functions/installer_functions.go +++ b/internal/functions/installer_functions.go @@ -313,7 +313,9 @@ func extractAndInstallBinary(archivePath, toolName, destPath string, extractFn f if err != nil { return fmt.Errorf("failed to create temp directory: %w", err) } - defer os.RemoveAll(tmpDir) + defer func() { + _ = os.RemoveAll(tmpDir) + }() // Extract archive if err := extractFn(archivePath, tmpDir); err != nil { @@ -370,13 +372,17 @@ func copyBinaryFile(src, dst string) error { if err != nil { return err } - defer sourceFile.Close() + defer func() { + _ = sourceFile.Close() + }() destFile, err := os.Create(dst) if err != nil { return err } - defer destFile.Close() + defer func() { + _ = destFile.Close() + }() _, err = io.Copy(destFile, sourceFile) return err diff --git a/pkg/cli/config.go b/pkg/cli/config.go index 38aefe7..ff77e41 100644 --- a/pkg/cli/config.go +++ b/pkg/cli/config.go @@ -1023,6 +1023,11 @@ func setNotificationValue(cfg *config.Config, parts []string, value string) erro return fmt.Errorf("missing telegram field") } return setTelegramValue(cfg, parts[1:], value) + case "webhooks": + if len(parts) < 2 { + return fmt.Errorf("missing webhooks index and field (e.g., webhooks.0.url)") + } + return setWebhooksValue(cfg, parts[1:], value) default: return fmt.Errorf("unknown notification field: %s", parts[0]) } @@ -1055,6 +1060,56 @@ func setTelegramValue(cfg *config.Config, parts []string, value string) error { return nil } +// setWebhooksValue sets webhook config fields (notification.webhooks..) +func setWebhooksValue(cfg *config.Config, parts []string, value string) error { + if len(parts) < 2 { + return fmt.Errorf("missing webhook index and field (e.g., webhooks.0.url)") + } + + index, err := strconv.Atoi(parts[0]) + if err != nil { + return fmt.Errorf("webhook index must be a number: %s", parts[0]) + } + + // Ensure webhooks slice is large enough + for len(cfg.Notification.Webhooks) <= index { + cfg.Notification.Webhooks = append(cfg.Notification.Webhooks, config.WebhookConfig{}) + } + + field := parts[1] + switch field { + case "url": + cfg.Notification.Webhooks[index].URL = value + case "enabled": + enabled, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("enabled must be true or false") + } + cfg.Notification.Webhooks[index].Enabled = enabled + case "timeout": + timeout, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("timeout must be a number") + } + cfg.Notification.Webhooks[index].Timeout = timeout + case "retry_count": + retryCount, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("retry_count must be a number") + } + cfg.Notification.Webhooks[index].RetryCount = retryCount + case "skip_tls_verify": + skip, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("skip_tls_verify must be true or false") + } + cfg.Notification.Webhooks[index].SkipTLSVerify = skip + default: + return fmt.Errorf("unknown webhook field: %s (available: url, enabled, timeout, retry_count, skip_tls_verify)", field) + } + return nil +} + // setEnvironmentsValue sets an environments config field func setEnvironmentsValue(cfg *config.Config, parts []string, value string) error { if len(parts) == 0 { diff --git a/pkg/cli/function.go b/pkg/cli/function.go index 82cba81..52cb167 100644 --- a/pkg/cli/function.go +++ b/pkg/cli/function.go @@ -172,8 +172,8 @@ func runFunctionEval(cmd *cobra.Command, args []string) error { } script = fmt.Sprintf("%s(%s)", args[0], strings.Join(quotedArgs, ", ")) } else { - // Single arg: use as-is (could be full expression or function name with no args) - script = args[0] + // Single arg: check if it is a bare function name and add () if needed + script = normalizeScriptExpression(args[0]) } } else if evalScript != "" { // Script provided via -e flag @@ -188,6 +188,10 @@ func runFunctionEval(cmd *cobra.Command, args []string) error { script = strings.TrimSpace(string(data)) } + if script != "" { + script = normalizeScriptExpression(script) + } + if script == "" { return fmt.Errorf("no script provided: use positional argument, -e flag, --function-file, or --stdin") } @@ -443,3 +447,29 @@ func runFunctionList(cmd *cobra.Command, args []string) error { } return nil } + +// normalizeScriptExpression checks if the input is a bare function name (without parentheses) +// and adds () to make it a valid function call. This allows "osmedeus eval cdn_ls_tree" +// to work the same as "osmedeus eval cdn_ls_tree()" +func normalizeScriptExpression(expr string) string { + expr = strings.TrimSpace(expr) + + // If it already has parentheses, return as-is + if strings.Contains(expr, "(") { + return expr + } + + // Check if it's a known function name + allFuncs := functions.AllFunctions() + for _, fn := range allFuncs { + if expr == fn { + if fn == functions.FnCdnLsTree { + return expr + "(\"\")" + } + return expr + "()" + } + } + + // Not a known function, return as-is (could be a variable or expression) + return expr +} diff --git a/pkg/cli/usage.go b/pkg/cli/usage.go index 2a34a07..52e576e 100644 --- a/pkg/cli/usage.go +++ b/pkg/cli/usage.go @@ -420,7 +420,13 @@ func UsageConfigSet() string { ` + terminal.Yellow("redis.port") + ` Redis port ` + terminal.Yellow("global_vars.") + ` Set a global variable ` + terminal.Yellow("notification.enabled") + ` Enable notifications (true/false) - ` + terminal.Yellow("notification.telegram.bot_token") + ` Telegram bot token + ` + terminal.Yellow("notification.provider") + ` Notification provider (telegram, webhook) + ` + terminal.Yellow("notification.telegram.enabled") + ` Enable Telegram notifications (true/false) + ` + terminal.Yellow("notification.telegram.bot_token") + ` Telegram bot token from @BotFather + ` + terminal.Yellow("notification.telegram.chat_id") + ` Telegram chat ID to send messages to + ` + terminal.Yellow("notification.webhooks.0.url") + ` Webhook URL (use index 0, 1, 2... for multiple) + ` + terminal.Yellow("notification.webhooks.0.enabled") + ` Enable webhook (true/false) + ` + terminal.Yellow("notification.webhooks.0.timeout") + ` Webhook timeout in seconds ` + terminal.Yellow("environments.external_binaries_path") + ` Binaries directory ` + terminal.Yellow("storage.enabled") + ` Enable cloud storage (true/false)