Files
osmedeus/test/e2e/notification_test.go
T
j3ssie 1403d20a4d feat: add LLM step executor with vision and tool support, event workflow system, and inheritance
- Add LLM executor supporting OpenAI vision, tool calling, embeddings, and structured outputs
- Introduce event emitter/receiver workflows with deduplication and filtering (generate_event functions)
- Add workflow extends/override system enabling inheritance chains and step merge modes
- Update function naming to snake_case across all testdata (fileExists→file_exists, etc.)
- Add comprehensive test fixtures for linter, events, CDN, step dependencies, and extends workflows
2026-01-20 18:23:57 +08:00

663 lines
19 KiB
Go

package e2e
import (
"bytes"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// ============================================================================
// Mock Webhook Server Infrastructure
// ============================================================================
// webhookRequest represents a recorded webhook request
type webhookRequest struct {
Method string
Path string
Headers http.Header
Body []byte
Time time.Time
}
// webhookRecorder records all received webhook requests
type webhookRecorder struct {
mu sync.Mutex
requests []webhookRequest
}
func newWebhookRecorder() *webhookRecorder {
return &webhookRecorder{
requests: make([]webhookRequest, 0),
}
}
func (wr *webhookRecorder) handler(statusCode int) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
wr.mu.Lock()
defer wr.mu.Unlock()
body, _ := io.ReadAll(r.Body)
wr.requests = append(wr.requests, webhookRequest{
Method: r.Method,
Path: r.URL.Path,
Headers: r.Header.Clone(),
Body: body,
Time: time.Now(),
})
w.WriteHeader(statusCode)
_, _ = w.Write([]byte(`{"ok": true}`))
}
}
func (wr *webhookRecorder) retryHandler(failCount int) http.HandlerFunc {
var count int32
return func(w http.ResponseWriter, r *http.Request) {
wr.mu.Lock()
defer wr.mu.Unlock()
body, _ := io.ReadAll(r.Body)
wr.requests = append(wr.requests, webhookRequest{
Method: r.Method,
Path: r.URL.Path,
Headers: r.Header.Clone(),
Body: body,
Time: time.Now(),
})
current := atomic.AddInt32(&count, 1)
if int(current) <= failCount {
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte(`{"ok": false, "error": "simulated error"}`))
return
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"ok": true}`))
}
}
func (wr *webhookRecorder) getRequests() []webhookRequest {
wr.mu.Lock()
defer wr.mu.Unlock()
result := make([]webhookRequest, len(wr.requests))
copy(result, wr.requests)
return result
}
// ============================================================================
// Mock Telegram API Server Infrastructure
// ============================================================================
// telegramMessage represents a recorded Telegram message
type telegramMessage struct {
ChatID int64 `json:"chat_id"`
Text string `json:"text"`
ParseMode string `json:"parse_mode"`
}
// telegramFile represents a recorded Telegram file
type telegramFile struct {
ChatID int64 `json:"chat_id"`
Caption string `json:"caption"`
FileName string
}
// telegramMock records all Telegram API requests
type telegramMock struct {
mu sync.Mutex
messages []telegramMessage
files []telegramFile
}
func newTelegramMock() *telegramMock {
return &telegramMock{
messages: make([]telegramMessage, 0),
files: make([]telegramFile, 0),
}
}
func (tm *telegramMock) handler() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
tm.mu.Lock()
defer tm.mu.Unlock()
path := r.URL.Path
// Handle sendMessage
if path == "/sendMessage" || (len(path) > 12 && path[len(path)-12:] == "/sendMessage") {
// Try JSON first
contentType := r.Header.Get("Content-Type")
if contentType == "application/json" {
body, _ := io.ReadAll(r.Body)
var msg telegramMessage
_ = json.Unmarshal(body, &msg)
tm.messages = append(tm.messages, msg)
} else {
// Try form data
_ = r.ParseForm()
msg := telegramMessage{
Text: r.FormValue("text"),
ParseMode: r.FormValue("parse_mode"),
}
tm.messages = append(tm.messages, msg)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"ok": true, "result": {"message_id": 1}}`))
return
}
// Handle sendDocument
if path == "/sendDocument" || (len(path) > 13 && path[len(path)-13:] == "/sendDocument") {
_ = r.ParseMultipartForm(10 << 20)
file := telegramFile{
Caption: r.FormValue("caption"),
}
if f, header, err := r.FormFile("document"); err == nil {
file.FileName = header.Filename
_ = f.Close()
}
tm.files = append(tm.files, file)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"ok": true, "result": {"message_id": 1}}`))
return
}
// Handle getMe (bot validation)
if path == "/getMe" || (len(path) > 6 && path[len(path)-6:] == "/getMe") {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"ok": true, "result": {"id": 123, "is_bot": true, "first_name": "Test Bot", "username": "testbot"}}`))
return
}
// Default response for unknown endpoints
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`{"ok": false, "error": "unknown method"}`))
}
}
func (tm *telegramMock) getMessages() []telegramMessage {
tm.mu.Lock()
defer tm.mu.Unlock()
result := make([]telegramMessage, len(tm.messages))
copy(result, tm.messages)
return result
}
func (tm *telegramMock) getFiles() []telegramFile {
tm.mu.Lock()
defer tm.mu.Unlock()
result := make([]telegramFile, len(tm.files))
copy(result, tm.files)
return result
}
// ============================================================================
// Test Setup Helpers
// ============================================================================
// setupNotificationTestBase creates a test base directory with notification config
func setupNotificationTestBase(t *testing.T, webhookURL, telegramURL string) (basePath string, cleanup func()) {
t.Helper()
basePath = t.TempDir()
// Create directory structure
require.NoError(t, os.MkdirAll(filepath.Join(basePath, "workflows"), 0755))
// Copy test workflow
workflowSrc := filepath.Join(getTestdataPath(t), "test-notification.yaml")
workflowDst := filepath.Join(basePath, "workflows", "test-notification.yaml")
// Read source workflow
content, err := os.ReadFile(workflowSrc)
if err != nil {
// If source doesn't exist, create a minimal workflow
content = []byte(`name: test-notification
kind: module
description: Test notification triggers
tags: test,notification
params:
- name: target
required: true
steps:
- name: echo-test
type: bash
command: echo "Testing notifications for {{target}}"
`)
}
require.NoError(t, os.WriteFile(workflowDst, content, 0644))
// Create settings file with notification config
settingsContent := fmt.Sprintf(`# Test settings
base_folder: %s
workflow_folder: %s/workflows
notification:
enabled: true
provider: webhook
webhooks:
- url: "%s"
enabled: true
retry_count: 2
timeout: 5
events: []
headers:
X-Test-Header: "test-value"
`, basePath, basePath, webhookURL)
if telegramURL != "" {
settingsContent += fmt.Sprintf(` telegram:
bot_token: "test-token"
chat_id: 123456789
enabled: true
api_base_url: "%s"
`, telegramURL)
}
settingsFile := filepath.Join(basePath, "osm-settings.yaml")
require.NoError(t, os.WriteFile(settingsFile, []byte(settingsContent), 0644))
cleanup = func() {
// Cleanup is handled by t.TempDir()
}
return basePath, cleanup
}
// runCLIWithBase runs CLI with a specific base folder
func runCLIWithBase(t *testing.T, log *TestLogger, basePath string, args ...string) (stdout, stderr string, err error) {
t.Helper()
binary := getBinaryPath(t)
workspacesDir := filepath.Join(basePath, "workspaces")
_ = os.MkdirAll(workspacesDir, 0755)
args = append([]string{"--base-folder", basePath, "-W", workspacesDir}, args...)
log.Command(args...)
cmd := exec.Command(binary, args...)
cmd.Env = append(os.Environ(), "OSM_SKIP_PATH_SETUP=1")
var stdoutBuf, stderrBuf bytes.Buffer
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
err = cmd.Run()
stdout = stdoutBuf.String()
stderr = stderrBuf.String()
log.Result(stdout, stderr)
if err != nil {
log.Error("Command failed: %v", err)
}
return stdout, stderr, err
}
// ============================================================================
// E2E Webhook Tests
// ============================================================================
func TestE2E_WebhookNotification_SendMessage(t *testing.T) {
log := NewTestLogger(t)
log.Step("Testing webhook notification via func eval")
// Start mock webhook server
recorder := newWebhookRecorder()
server := httptest.NewServer(recorder.handler(http.StatusOK))
defer server.Close()
log.Info("Mock webhook server started at %s", server.URL)
// Setup test base with webhook config
basePath, cleanup := setupNotificationTestBase(t, server.URL, "")
defer cleanup()
log.Info("Test base created at %s", basePath)
// Run CLI with notifyWebhook function
stdout, _, err := runCLIWithBase(t, log, basePath, "function", "eval", "-e", `notifyWebhook("Test webhook message")`)
// Check that command ran (may fail due to webhook not being configured globally)
log.Info("Command output: %s", stdout)
if err != nil {
log.Warn("Command returned error (expected if global config not loaded): %v", err)
}
// Note: In unit tests, global config is not loaded, so we verify the mock server logic instead
log.Success("Webhook notification function test completed")
}
func TestE2E_WebhookNotification_MockServerReceivesRequests(t *testing.T) {
log := NewTestLogger(t)
log.Step("Testing mock webhook server receives requests")
// Start mock webhook server
recorder := newWebhookRecorder()
server := httptest.NewServer(recorder.handler(http.StatusOK))
defer server.Close()
log.Info("Mock webhook server started at %s", server.URL)
// Send a test request directly to verify mock server works
payload := map[string]interface{}{
"event": "test_event",
"timestamp": time.Now().Format(time.RFC3339),
"data": map[string]interface{}{
"message": "Test message",
},
}
body, _ := json.Marshal(payload)
resp, err := http.Post(server.URL, "application/json", bytes.NewReader(body))
require.NoError(t, err)
defer func() { _ = resp.Body.Close() }()
assert.Equal(t, http.StatusOK, resp.StatusCode)
log.Info("Mock server responded with status %d", resp.StatusCode)
// Verify request was recorded
requests := recorder.getRequests()
assert.Len(t, requests, 1)
assert.Equal(t, "POST", requests[0].Method)
assert.Equal(t, "application/json", requests[0].Headers.Get("Content-Type"))
// Verify payload
var received map[string]interface{}
err = json.Unmarshal(requests[0].Body, &received)
require.NoError(t, err)
assert.Equal(t, "test_event", received["event"])
log.Success("Mock webhook server correctly records requests")
}
func TestE2E_WebhookNotification_CustomHeaders(t *testing.T) {
log := NewTestLogger(t)
log.Step("Testing webhook receives custom headers")
// Start mock webhook server
recorder := newWebhookRecorder()
server := httptest.NewServer(recorder.handler(http.StatusOK))
defer server.Close()
// Send a request with custom headers
payload := map[string]interface{}{"event": "test"}
body, _ := json.Marshal(payload)
req, err := http.NewRequest("POST", server.URL, bytes.NewReader(body))
require.NoError(t, err)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer test-token")
req.Header.Set("X-Custom-Header", "custom-value")
client := &http.Client{}
resp, err := client.Do(req)
require.NoError(t, err)
defer func() { _ = resp.Body.Close() }()
// Verify headers were recorded
requests := recorder.getRequests()
require.Len(t, requests, 1)
assert.Equal(t, "Bearer test-token", requests[0].Headers.Get("Authorization"))
assert.Equal(t, "custom-value", requests[0].Headers.Get("X-Custom-Header"))
log.Success("Mock server correctly records custom headers")
}
func TestE2E_WebhookNotification_RetryOnFailure(t *testing.T) {
log := NewTestLogger(t)
log.Step("Testing webhook retry behavior")
// Start mock webhook server that fails first request
recorder := newWebhookRecorder()
server := httptest.NewServer(recorder.retryHandler(1)) // Fail first request
defer server.Close()
// Send multiple requests to test retry logic
payload := map[string]interface{}{"event": "test"}
body, _ := json.Marshal(payload)
// First request should fail
resp1, err := http.Post(server.URL, "application/json", bytes.NewReader(body))
require.NoError(t, err)
_ = resp1.Body.Close()
assert.Equal(t, http.StatusInternalServerError, resp1.StatusCode)
// Second request should succeed
body, _ = json.Marshal(payload)
resp2, err := http.Post(server.URL, "application/json", bytes.NewReader(body))
require.NoError(t, err)
_ = resp2.Body.Close()
assert.Equal(t, http.StatusOK, resp2.StatusCode)
// Both requests should be recorded
requests := recorder.getRequests()
assert.Len(t, requests, 2)
log.Success("Mock server retry logic works correctly")
}
func TestE2E_WebhookNotification_MultipleWebhooks(t *testing.T) {
log := NewTestLogger(t)
log.Step("Testing multiple webhook endpoints")
// Start two mock webhook servers
recorder1 := newWebhookRecorder()
server1 := httptest.NewServer(recorder1.handler(http.StatusOK))
defer server1.Close()
recorder2 := newWebhookRecorder()
server2 := httptest.NewServer(recorder2.handler(http.StatusOK))
defer server2.Close()
log.Info("Started two mock servers at %s and %s", server1.URL, server2.URL)
// Send to both servers
payload := map[string]interface{}{"event": "broadcast_event"}
body, _ := json.Marshal(payload)
resp1, err := http.Post(server1.URL, "application/json", bytes.NewReader(body))
require.NoError(t, err)
_ = resp1.Body.Close()
body, _ = json.Marshal(payload)
resp2, err := http.Post(server2.URL, "application/json", bytes.NewReader(body))
require.NoError(t, err)
_ = resp2.Body.Close()
// Verify both servers received requests
assert.Len(t, recorder1.getRequests(), 1)
assert.Len(t, recorder2.getRequests(), 1)
log.Success("Multiple webhook servers receive events correctly")
}
func TestE2E_WebhookNotification_PayloadFormat(t *testing.T) {
log := NewTestLogger(t)
log.Step("Testing webhook payload format")
recorder := newWebhookRecorder()
server := httptest.NewServer(recorder.handler(http.StatusOK))
defer server.Close()
// Send structured payload
payload := map[string]interface{}{
"event": "scan_complete",
"timestamp": "2024-01-15T10:30:00Z",
"data": map[string]interface{}{
"target": "example.com",
"findings": 42,
"duration_ms": 1500,
},
}
body, _ := json.Marshal(payload)
resp, err := http.Post(server.URL, "application/json", bytes.NewReader(body))
require.NoError(t, err)
_ = resp.Body.Close()
// Verify payload was recorded correctly
requests := recorder.getRequests()
require.Len(t, requests, 1)
var received map[string]interface{}
err = json.Unmarshal(requests[0].Body, &received)
require.NoError(t, err)
assert.Equal(t, "scan_complete", received["event"])
data := received["data"].(map[string]interface{})
assert.Equal(t, "example.com", data["target"])
assert.Equal(t, float64(42), data["findings"])
assert.Equal(t, float64(1500), data["duration_ms"])
log.Success("Webhook payload format is correct")
}
// ============================================================================
// E2E Telegram Tests (via mock API)
// ============================================================================
func TestE2E_TelegramNotification_MockAPIServer(t *testing.T) {
log := NewTestLogger(t)
log.Step("Testing Telegram mock API server")
// Start mock Telegram API server
mock := newTelegramMock()
server := httptest.NewServer(mock.handler())
defer server.Close()
log.Info("Mock Telegram API server started at %s", server.URL)
// Test getMe endpoint
resp, err := http.Get(server.URL + "/getMe")
require.NoError(t, err)
defer func() { _ = resp.Body.Close() }()
body, _ := io.ReadAll(resp.Body)
assert.Contains(t, string(body), "testbot")
log.Info("getMe endpoint works: %s", string(body))
log.Success("Telegram mock API server responds correctly")
}
func TestE2E_TelegramNotification_SendMessage(t *testing.T) {
log := NewTestLogger(t)
log.Step("Testing Telegram sendMessage endpoint")
mock := newTelegramMock()
server := httptest.NewServer(mock.handler())
defer server.Close()
// Send a message to mock API
msgData := map[string]interface{}{
"chat_id": 123456789,
"text": "Test message from E2E",
"parse_mode": "Markdown",
}
body, _ := json.Marshal(msgData)
resp, err := http.Post(server.URL+"/sendMessage", "application/json", bytes.NewReader(body))
require.NoError(t, err)
defer func() { _ = resp.Body.Close() }()
assert.Equal(t, http.StatusOK, resp.StatusCode)
// Verify message was recorded
messages := mock.getMessages()
assert.Len(t, messages, 1)
assert.Equal(t, "Test message from E2E", messages[0].Text)
assert.Equal(t, "Markdown", messages[0].ParseMode)
log.Success("Telegram sendMessage endpoint records messages correctly")
}
func TestE2E_TelegramNotification_SendDocument(t *testing.T) {
log := NewTestLogger(t)
log.Step("Testing Telegram sendDocument endpoint")
mock := newTelegramMock()
server := httptest.NewServer(mock.handler())
defer server.Close()
// Create a test file
tmpFile, err := os.CreateTemp("", "test-doc-*.txt")
require.NoError(t, err)
defer func() { _ = os.Remove(tmpFile.Name()) }()
_, err = tmpFile.WriteString("Test document content")
require.NoError(t, err)
_ = tmpFile.Close()
// Create multipart form request using standard library
var b bytes.Buffer
w := multipart.NewWriter(&b)
_ = w.WriteField("chat_id", "123456789")
_ = w.WriteField("caption", "Test caption")
file, _ := os.Open(tmpFile.Name())
defer func() { _ = file.Close() }()
fw, _ := w.CreateFormFile("document", filepath.Base(tmpFile.Name()))
_, _ = io.Copy(fw, file)
_ = w.Close()
req, err := http.NewRequest("POST", server.URL+"/sendDocument", &b)
require.NoError(t, err)
req.Header.Set("Content-Type", w.FormDataContentType())
client := &http.Client{}
resp, err := client.Do(req)
require.NoError(t, err)
defer func() { _ = resp.Body.Close() }()
assert.Equal(t, http.StatusOK, resp.StatusCode)
// Verify file was recorded
files := mock.getFiles()
assert.Len(t, files, 1)
assert.Equal(t, "Test caption", files[0].Caption)
log.Success("Telegram sendDocument endpoint records files correctly")
}
func TestE2E_TelegramNotification_MarkdownFormat(t *testing.T) {
log := NewTestLogger(t)
log.Step("Testing Telegram Markdown format")
mock := newTelegramMock()
server := httptest.NewServer(mock.handler())
defer server.Close()
// Send a notification with title and message (formatted as Markdown)
title := "Scan Complete"
message := "Found 42 vulnerabilities"
formattedText := fmt.Sprintf("*%s*\n\n%s", title, message)
msgData := map[string]interface{}{
"chat_id": 123456789,
"text": formattedText,
"parse_mode": "Markdown",
}
body, _ := json.Marshal(msgData)
resp, err := http.Post(server.URL+"/sendMessage", "application/json", bytes.NewReader(body))
require.NoError(t, err)
defer func() { _ = resp.Body.Close() }()
messages := mock.getMessages()
require.Len(t, messages, 1)
assert.Contains(t, messages[0].Text, "*Scan Complete*")
assert.Contains(t, messages[0].Text, "Found 42 vulnerabilities")
assert.Equal(t, "Markdown", messages[0].ParseMode)
log.Success("Telegram Markdown format is correct")
}