mirror of
https://github.com/suitenumerique/messages.git
synced 2026-08-17 21:25:41 +02:00
✨(backend) report selfcheck status to Sentry crons (#694)
Use the new (documented) env var MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG to enable
This commit is contained in:
+14
@@ -247,6 +247,20 @@ _Those settings are deprecated and will be removed in the future._
|
||||
| `NEXT_PUBLIC_SENTRY_DSN` | None | Sentry DSN for error tracking | Optional |
|
||||
| `NEXT_PUBLIC_SENTRY_ENVIRONMENT` | None | Sentry environment for error tracking | Optional ('production', 'development', 'staging') |
|
||||
|
||||
### Selfcheck
|
||||
|
||||
End-to-end mail delivery probe — see [selfcheck.md](selfcheck.md) for details.
|
||||
|
||||
| Variable | Default | Description | Required |
|
||||
|----------|---------|-------------|----------|
|
||||
| `MESSAGES_SELFCHECK_FROM` | None | Email address the selfcheck sends from. Leave unset to disable the selfcheck. | Optional |
|
||||
| `MESSAGES_SELFCHECK_TO` | None | Email address the selfcheck sends to. Leave unset to disable the selfcheck. | Optional |
|
||||
| `MESSAGES_SELFCHECK_SECRET` | `self-check-secret-for-dev` | Secret string embedded in the test message body | Optional |
|
||||
| `MESSAGES_SELFCHECK_INTERVAL` | `600` | Interval between selfcheck runs, in seconds | Optional |
|
||||
| `MESSAGES_SELFCHECK_TIMEOUT` | `60` | Timeout for message reception, in seconds | Optional |
|
||||
| `MESSAGES_SELFCHECK_WEBHOOK_URL` | None | Webhook URL POSTed on each successful selfcheck (updown.io-compatible heartbeat) | Optional |
|
||||
| `MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG` | None | Sentry cron monitor slug. When set (with `SENTRY_DSN`), each run is reported as a Sentry check-in. | Optional |
|
||||
|
||||
### Logging
|
||||
|
||||
| Variable | Default | Description | Required |
|
||||
|
||||
@@ -28,6 +28,10 @@ Optionally, to enable uptime alerting via a selfcheck webhook:
|
||||
|
||||
- `MESSAGES_SELFCHECK_WEBHOOK_URL`: URL of the selfcheck webhook endpoint (default: `None` - disabled)
|
||||
|
||||
Optionally, to report selfcheck runs to [Sentry Crons](https://docs.sentry.io/product/crons/):
|
||||
|
||||
- `MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG`: Slug of the Sentry cron monitor (default: `None` - disabled). Requires `SENTRY_DSN` to also be set.
|
||||
|
||||
## Usage
|
||||
|
||||
### Manual Execution
|
||||
@@ -95,6 +99,16 @@ The POST body includes timing data:
|
||||
{"send_time": 0.15, "reception_time": 2.34}
|
||||
```
|
||||
|
||||
### Sentry Crons
|
||||
|
||||
When `MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG` is configured (and `SENTRY_DSN` is set), each selfcheck run is reported to [Sentry Crons](https://docs.sentry.io/product/crons/):
|
||||
|
||||
- An `in_progress` check-in is opened before the test message is sent.
|
||||
- The check-in is closed with status `ok` on success or `error` on failure.
|
||||
- On success, the reported `duration` is `send_time + reception_time`, excluding the post-run cleanup pause.
|
||||
|
||||
Configure the monitor schedule (interval and grace period) in the Sentry UI to match `MESSAGES_SELFCHECK_INTERVAL`. Runs skipped because `MESSAGES_SELFCHECK_FROM` or `MESSAGES_SELFCHECK_TO` is empty do not produce a check-in.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
- The selfcheck uses dedicated test mailboxes that are separate from user data
|
||||
|
||||
@@ -13,7 +13,12 @@ from django.utils import timezone
|
||||
from core import models
|
||||
from core.mda.draft import create_draft
|
||||
from core.mda.outbound import prepare_outbound_message, send_message
|
||||
from core.mda.selfcheck_reporting import SelfCheckResult, report_selfcheck
|
||||
from core.mda.selfcheck_reporting import (
|
||||
SelfCheckResult,
|
||||
finish_sentry_checkin,
|
||||
report_selfcheck,
|
||||
start_sentry_checkin,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -247,6 +252,8 @@ def run_selfcheck() -> SelfCheckResult:
|
||||
|
||||
logger.info("Starting selfcheck: %s -> %s", from_email, to_email)
|
||||
|
||||
check_in_id = start_sentry_checkin()
|
||||
|
||||
try:
|
||||
# Step 1: Create test mailboxes
|
||||
from_mailbox, to_mailbox = _create_test_mailboxes(from_email, to_email)
|
||||
@@ -340,4 +347,6 @@ that the mail delivery pipeline is working correctly.</p>
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Failed to report selfcheck result", exc_info=True)
|
||||
|
||||
finish_sentry_checkin(check_in_id, result)
|
||||
|
||||
return result
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Selfcheck reporting: webhook and structured logging."""
|
||||
"""Selfcheck reporting: webhook, Sentry crons, and structured logging."""
|
||||
|
||||
import logging
|
||||
from typing import Optional, TypedDict
|
||||
@@ -6,6 +6,8 @@ from typing import Optional, TypedDict
|
||||
from django.conf import settings
|
||||
|
||||
import requests
|
||||
from sentry_sdk.crons import capture_checkin
|
||||
from sentry_sdk.crons.consts import MonitorStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -45,6 +47,71 @@ def log_selfcheck_result(result: SelfCheckResult):
|
||||
)
|
||||
|
||||
|
||||
def _sentry_crons_enabled() -> bool:
|
||||
"""Whether Sentry cron reporting is enabled and usable.
|
||||
|
||||
Warns once per call when a slug is configured without ``SENTRY_DSN``
|
||||
— without the DSN, ``capture_checkin`` silently no-ops, so the
|
||||
operator would otherwise see "missed" alerts in Sentry with no
|
||||
hint as to why.
|
||||
"""
|
||||
if not settings.MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG:
|
||||
return False
|
||||
if not settings.SENTRY_DSN:
|
||||
logger.warning(
|
||||
"MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG is set but SENTRY_DSN is "
|
||||
"not — selfcheck Sentry cron check-in skipped."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def start_sentry_checkin() -> Optional[str]:
|
||||
"""Open a Sentry cron check-in if configured. Returns the check_in_id."""
|
||||
if not _sentry_crons_enabled():
|
||||
return None
|
||||
|
||||
try:
|
||||
return capture_checkin(
|
||||
monitor_slug=settings.MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG,
|
||||
status=MonitorStatus.IN_PROGRESS,
|
||||
)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Failed to open Sentry selfcheck check-in", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def finish_sentry_checkin(check_in_id: Optional[str], result: SelfCheckResult):
|
||||
"""Close a previously opened Sentry cron check-in with OK/ERROR.
|
||||
|
||||
Reports an explicit duration when both send and reception times are
|
||||
known — Sentry would otherwise infer it from the check-in timestamp
|
||||
delta, which also includes the post-run cleanup sleep.
|
||||
"""
|
||||
# Short-circuit on check_in_id first so the misconfig warning from
|
||||
# _sentry_crons_enabled() fires at most once per run (from start).
|
||||
if not check_in_id or not _sentry_crons_enabled():
|
||||
return
|
||||
|
||||
status = MonitorStatus.OK if result["success"] else MonitorStatus.ERROR
|
||||
send_time = result["send_time"]
|
||||
reception_time = result["reception_time"]
|
||||
duration = (
|
||||
send_time + reception_time
|
||||
if send_time is not None and reception_time is not None
|
||||
else None
|
||||
)
|
||||
try:
|
||||
capture_checkin(
|
||||
monitor_slug=settings.MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG,
|
||||
check_in_id=check_in_id,
|
||||
status=status,
|
||||
duration=duration,
|
||||
)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Failed to close Sentry selfcheck check-in", exc_info=True)
|
||||
|
||||
|
||||
def send_selfcheck_webhook(result: SelfCheckResult):
|
||||
"""POST to selfcheck webhook on success only."""
|
||||
webhook_url = settings.MESSAGES_SELFCHECK_WEBHOOK_URL
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
"""Tests for selfcheck reporting (webhook + structured logging)."""
|
||||
|
||||
import json
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.test import TestCase, override_settings
|
||||
|
||||
import responses
|
||||
|
||||
from core.mda.selfcheck_reporting import SelfCheckResult, report_selfcheck
|
||||
from core.mda.selfcheck_reporting import (
|
||||
SelfCheckResult,
|
||||
finish_sentry_checkin,
|
||||
report_selfcheck,
|
||||
start_sentry_checkin,
|
||||
)
|
||||
|
||||
WEBHOOK_URL = "https://example.com/api/checks/xxxx/webhook"
|
||||
|
||||
@@ -112,3 +118,151 @@ class TestSendSelfcheckWebhook(TestCase):
|
||||
self.assertTrue(
|
||||
any("Failed to send selfcheck webhook" in line for line in cm.output)
|
||||
)
|
||||
|
||||
|
||||
SENTRY_SLUG = "messages-selfcheck"
|
||||
SENTRY_DSN = "https://public@sentry.example.com/1"
|
||||
|
||||
|
||||
class TestStartSentryCheckin(TestCase):
|
||||
"""Tests for opening a Sentry cron check-in."""
|
||||
|
||||
@override_settings(MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=None)
|
||||
@patch("core.mda.selfcheck_reporting.capture_checkin")
|
||||
def test_noop_when_slug_unset(self, mock_capture):
|
||||
"""No Sentry call when slug is not configured; returns None."""
|
||||
self.assertIsNone(start_sentry_checkin())
|
||||
mock_capture.assert_not_called()
|
||||
|
||||
@override_settings(
|
||||
MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG, SENTRY_DSN=None
|
||||
)
|
||||
@patch("core.mda.selfcheck_reporting.capture_checkin")
|
||||
def test_warns_and_skips_when_dsn_missing(self, mock_capture):
|
||||
"""Slug set without SENTRY_DSN: warn loudly and skip the call."""
|
||||
with self.assertLogs("core.mda.selfcheck_reporting", level="WARNING") as cm:
|
||||
self.assertIsNone(start_sentry_checkin())
|
||||
|
||||
mock_capture.assert_not_called()
|
||||
self.assertTrue(
|
||||
any(
|
||||
"MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG is set but SENTRY_DSN is not"
|
||||
in line
|
||||
for line in cm.output
|
||||
)
|
||||
)
|
||||
|
||||
@override_settings(
|
||||
MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG, SENTRY_DSN=SENTRY_DSN
|
||||
)
|
||||
@patch("core.mda.selfcheck_reporting.capture_checkin")
|
||||
def test_opens_in_progress_checkin(self, mock_capture):
|
||||
"""Sends IN_PROGRESS check-in and returns the check_in_id."""
|
||||
mock_capture.return_value = "abc123"
|
||||
|
||||
check_in_id = start_sentry_checkin()
|
||||
|
||||
self.assertEqual(check_in_id, "abc123")
|
||||
mock_capture.assert_called_once_with(
|
||||
monitor_slug=SENTRY_SLUG,
|
||||
status="in_progress",
|
||||
)
|
||||
|
||||
@override_settings(
|
||||
MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG, SENTRY_DSN=SENTRY_DSN
|
||||
)
|
||||
@patch("core.mda.selfcheck_reporting.capture_checkin")
|
||||
def test_error_swallowed(self, mock_capture):
|
||||
"""Failure in capture_checkin logs a warning and returns None."""
|
||||
mock_capture.side_effect = RuntimeError("sentry down")
|
||||
|
||||
with self.assertLogs("core.mda.selfcheck_reporting", level="WARNING") as cm:
|
||||
self.assertIsNone(start_sentry_checkin())
|
||||
|
||||
self.assertTrue(
|
||||
any(
|
||||
"Failed to open Sentry selfcheck check-in" in line for line in cm.output
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class TestFinishSentryCheckin(TestCase):
|
||||
"""Tests for closing a Sentry cron check-in."""
|
||||
|
||||
@override_settings(MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=None)
|
||||
@patch("core.mda.selfcheck_reporting.capture_checkin")
|
||||
def test_noop_when_slug_unset(self, mock_capture):
|
||||
"""No Sentry call when slug is not configured."""
|
||||
finish_sentry_checkin("abc123", SUCCESS_RESULT)
|
||||
mock_capture.assert_not_called()
|
||||
|
||||
@override_settings(
|
||||
MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG, SENTRY_DSN=SENTRY_DSN
|
||||
)
|
||||
@patch("core.mda.selfcheck_reporting.capture_checkin")
|
||||
def test_noop_when_check_in_id_missing(self, mock_capture):
|
||||
"""No Sentry call when start_sentry_checkin returned None."""
|
||||
finish_sentry_checkin(None, SUCCESS_RESULT)
|
||||
mock_capture.assert_not_called()
|
||||
|
||||
@override_settings(
|
||||
MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG, SENTRY_DSN=None
|
||||
)
|
||||
@patch("core.mda.selfcheck_reporting.capture_checkin")
|
||||
def test_skips_when_dsn_missing(self, mock_capture):
|
||||
"""Slug set without SENTRY_DSN: skip the call.
|
||||
|
||||
(No warning expected here — start_sentry_checkin already warned and
|
||||
returned None, so finish_sentry_checkin short-circuits on
|
||||
check_in_id before the misconfig check.)
|
||||
"""
|
||||
finish_sentry_checkin("abc123", SUCCESS_RESULT)
|
||||
mock_capture.assert_not_called()
|
||||
|
||||
@override_settings(
|
||||
MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG, SENTRY_DSN=SENTRY_DSN
|
||||
)
|
||||
@patch("core.mda.selfcheck_reporting.capture_checkin")
|
||||
def test_ok_with_duration_on_success(self, mock_capture):
|
||||
"""OK status with send+reception duration on success."""
|
||||
finish_sentry_checkin("abc123", SUCCESS_RESULT)
|
||||
|
||||
mock_capture.assert_called_once_with(
|
||||
monitor_slug=SENTRY_SLUG,
|
||||
check_in_id="abc123",
|
||||
status="ok",
|
||||
duration=0.150 + 2.340,
|
||||
)
|
||||
|
||||
@override_settings(
|
||||
MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG, SENTRY_DSN=SENTRY_DSN
|
||||
)
|
||||
@patch("core.mda.selfcheck_reporting.capture_checkin")
|
||||
def test_error_without_duration_on_failure(self, mock_capture):
|
||||
"""ERROR status and no duration when timing data is missing."""
|
||||
finish_sentry_checkin("abc123", FAILURE_RESULT)
|
||||
|
||||
mock_capture.assert_called_once_with(
|
||||
monitor_slug=SENTRY_SLUG,
|
||||
check_in_id="abc123",
|
||||
status="error",
|
||||
duration=None,
|
||||
)
|
||||
|
||||
@override_settings(
|
||||
MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG, SENTRY_DSN=SENTRY_DSN
|
||||
)
|
||||
@patch("core.mda.selfcheck_reporting.capture_checkin")
|
||||
def test_error_swallowed(self, mock_capture):
|
||||
"""Failure in capture_checkin logs a warning, no raise."""
|
||||
mock_capture.side_effect = RuntimeError("sentry down")
|
||||
|
||||
with self.assertLogs("core.mda.selfcheck_reporting", level="WARNING") as cm:
|
||||
finish_sentry_checkin("abc123", SUCCESS_RESULT)
|
||||
|
||||
self.assertTrue(
|
||||
any(
|
||||
"Failed to close Sentry selfcheck check-in" in line
|
||||
for line in cm.output
|
||||
)
|
||||
)
|
||||
|
||||
@@ -507,6 +507,12 @@ class Base(Configuration):
|
||||
environ_prefix=None,
|
||||
)
|
||||
|
||||
MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG = values.Value(
|
||||
None,
|
||||
environ_name="MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG",
|
||||
environ_prefix=None,
|
||||
)
|
||||
|
||||
# Manual retry settings
|
||||
MESSAGES_MANUAL_RETRY_MAX_AGE = values.PositiveIntegerValue(
|
||||
7 * 24 * 60 * 60, # 7 days in seconds
|
||||
|
||||
Reference in New Issue
Block a user