From bb3672b83e4287cdb70ba7cadfb40a0b7fa68efe Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Thu, 4 Jun 2026 00:23:52 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(backend)=20report=20selfcheck=20statu?= =?UTF-8?q?s=20to=20Sentry=20crons?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the new (documented) env var MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG to enable --- docs/env.md | 14 +++ docs/selfcheck.md | 14 +++ src/backend/core/mda/selfcheck.py | 11 +- src/backend/core/mda/selfcheck_reporting.py | 50 +++++++- .../tests/mda/test_selfcheck_reporting.py | 111 +++++++++++++++++- src/backend/messages/settings.py | 6 + 6 files changed, 203 insertions(+), 3 deletions(-) diff --git a/docs/env.md b/docs/env.md index f943baaf..e17add56 100644 --- a/docs/env.md +++ b/docs/env.md @@ -247,6 +247,20 @@ _Those settings are deprecated and will be removed in the future._ | `NEXT_PUBLIC_SENTRY_DSN` | None | Sentry DSN for error tracking | Optional | | `NEXT_PUBLIC_SENTRY_ENVIRONMENT` | None | Sentry environment for error tracking | Optional ('production', 'development', 'staging') | +### Selfcheck + +End-to-end mail delivery probe — see [selfcheck.md](selfcheck.md) for details. + +| Variable | Default | Description | Required | +|----------|---------|-------------|----------| +| `MESSAGES_SELFCHECK_FROM` | None | Email address the selfcheck sends from. Leave unset to disable the selfcheck. | Optional | +| `MESSAGES_SELFCHECK_TO` | None | Email address the selfcheck sends to. Leave unset to disable the selfcheck. | Optional | +| `MESSAGES_SELFCHECK_SECRET` | `self-check-secret-for-dev` | Secret string embedded in the test message body | Optional | +| `MESSAGES_SELFCHECK_INTERVAL` | `600` | Interval between selfcheck runs, in seconds | Optional | +| `MESSAGES_SELFCHECK_TIMEOUT` | `60` | Timeout for message reception, in seconds | Optional | +| `MESSAGES_SELFCHECK_WEBHOOK_URL` | None | Webhook URL POSTed on each successful selfcheck (updown.io-compatible heartbeat) | Optional | +| `MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG` | None | Sentry cron monitor slug. When set (with `SENTRY_DSN`), each run is reported as a Sentry check-in. | Optional | + ### Logging | Variable | Default | Description | Required | diff --git a/docs/selfcheck.md b/docs/selfcheck.md index f0968033..7f635e61 100644 --- a/docs/selfcheck.md +++ b/docs/selfcheck.md @@ -28,6 +28,10 @@ Optionally, to enable uptime alerting via a selfcheck webhook: - `MESSAGES_SELFCHECK_WEBHOOK_URL`: URL of the selfcheck webhook endpoint (default: `None` - disabled) +Optionally, to report selfcheck runs to [Sentry Crons](https://docs.sentry.io/product/crons/): + +- `MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG`: Slug of the Sentry cron monitor (default: `None` - disabled). Requires `SENTRY_DSN` to also be set. + ## Usage ### Manual Execution @@ -95,6 +99,16 @@ The POST body includes timing data: {"send_time": 0.15, "reception_time": 2.34} ``` +### Sentry Crons + +When `MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG` is configured (and `SENTRY_DSN` is set), each selfcheck run is reported to [Sentry Crons](https://docs.sentry.io/product/crons/): + +- An `in_progress` check-in is opened before the test message is sent. +- The check-in is closed with status `ok` on success or `error` on failure. +- On success, the reported `duration` is `send_time + reception_time`, excluding the post-run cleanup pause. + +Configure the monitor schedule (interval and grace period) in the Sentry UI to match `MESSAGES_SELFCHECK_INTERVAL`. Runs skipped because `MESSAGES_SELFCHECK_FROM` or `MESSAGES_SELFCHECK_TO` is empty do not produce a check-in. + ## Security Considerations - The selfcheck uses dedicated test mailboxes that are separate from user data diff --git a/src/backend/core/mda/selfcheck.py b/src/backend/core/mda/selfcheck.py index 226ab3e9..c25224da 100644 --- a/src/backend/core/mda/selfcheck.py +++ b/src/backend/core/mda/selfcheck.py @@ -13,7 +13,12 @@ from django.utils import timezone from core import models from core.mda.draft import create_draft from core.mda.outbound import prepare_outbound_message, send_message -from core.mda.selfcheck_reporting import SelfCheckResult, report_selfcheck +from core.mda.selfcheck_reporting import ( + SelfCheckResult, + finish_sentry_checkin, + report_selfcheck, + start_sentry_checkin, +) logger = logging.getLogger(__name__) @@ -247,6 +252,8 @@ def run_selfcheck() -> SelfCheckResult: logger.info("Starting selfcheck: %s -> %s", from_email, to_email) + check_in_id = start_sentry_checkin() + try: # Step 1: Create test mailboxes from_mailbox, to_mailbox = _create_test_mailboxes(from_email, to_email) @@ -340,4 +347,6 @@ that the mail delivery pipeline is working correctly.

except Exception: # pylint: disable=broad-exception-caught logger.warning("Failed to report selfcheck result", exc_info=True) + finish_sentry_checkin(check_in_id, result) + return result diff --git a/src/backend/core/mda/selfcheck_reporting.py b/src/backend/core/mda/selfcheck_reporting.py index 1f137650..5b127b97 100644 --- a/src/backend/core/mda/selfcheck_reporting.py +++ b/src/backend/core/mda/selfcheck_reporting.py @@ -1,4 +1,4 @@ -"""Selfcheck reporting: webhook and structured logging.""" +"""Selfcheck reporting: webhook, Sentry crons, and structured logging.""" import logging from typing import Optional, TypedDict @@ -6,6 +6,8 @@ from typing import Optional, TypedDict from django.conf import settings import requests +from sentry_sdk.crons import capture_checkin +from sentry_sdk.crons.consts import MonitorStatus logger = logging.getLogger(__name__) @@ -45,6 +47,52 @@ def log_selfcheck_result(result: SelfCheckResult): ) +def start_sentry_checkin() -> Optional[str]: + """Open a Sentry cron check-in if configured. Returns the check_in_id.""" + slug = settings.MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG + if not slug: + return None + + try: + return capture_checkin( + monitor_slug=slug, + status=MonitorStatus.IN_PROGRESS, + ) + except Exception: # pylint: disable=broad-exception-caught + logger.warning("Failed to open Sentry selfcheck check-in", exc_info=True) + return None + + +def finish_sentry_checkin(check_in_id: Optional[str], result: SelfCheckResult): + """Close a previously opened Sentry cron check-in with OK/ERROR. + + Reports an explicit duration when both send and reception times are + known — Sentry would otherwise infer it from the check-in timestamp + delta, which also includes the post-run cleanup sleep. + """ + slug = settings.MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG + if not slug or not check_in_id: + return + + status = MonitorStatus.OK if result["success"] else MonitorStatus.ERROR + send_time = result["send_time"] + reception_time = result["reception_time"] + duration = ( + send_time + reception_time + if send_time is not None and reception_time is not None + else None + ) + try: + capture_checkin( + monitor_slug=slug, + check_in_id=check_in_id, + status=status, + duration=duration, + ) + except Exception: # pylint: disable=broad-exception-caught + logger.warning("Failed to close Sentry selfcheck check-in", exc_info=True) + + def send_selfcheck_webhook(result: SelfCheckResult): """POST to selfcheck webhook on success only.""" webhook_url = settings.MESSAGES_SELFCHECK_WEBHOOK_URL diff --git a/src/backend/core/tests/mda/test_selfcheck_reporting.py b/src/backend/core/tests/mda/test_selfcheck_reporting.py index b26fe377..4e4238ab 100644 --- a/src/backend/core/tests/mda/test_selfcheck_reporting.py +++ b/src/backend/core/tests/mda/test_selfcheck_reporting.py @@ -1,12 +1,18 @@ """Tests for selfcheck reporting (webhook + structured logging).""" import json +from unittest.mock import patch from django.test import TestCase, override_settings import responses -from core.mda.selfcheck_reporting import SelfCheckResult, report_selfcheck +from core.mda.selfcheck_reporting import ( + SelfCheckResult, + finish_sentry_checkin, + report_selfcheck, + start_sentry_checkin, +) WEBHOOK_URL = "https://example.com/api/checks/xxxx/webhook" @@ -112,3 +118,106 @@ class TestSendSelfcheckWebhook(TestCase): self.assertTrue( any("Failed to send selfcheck webhook" in line for line in cm.output) ) + + +SENTRY_SLUG = "messages-selfcheck" + + +class TestStartSentryCheckin(TestCase): + """Tests for opening a Sentry cron check-in.""" + + @override_settings(MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=None) + @patch("core.mda.selfcheck_reporting.capture_checkin") + def test_noop_when_slug_unset(self, mock_capture): + """No Sentry call when slug is not configured; returns None.""" + self.assertIsNone(start_sentry_checkin()) + mock_capture.assert_not_called() + + @override_settings(MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG) + @patch("core.mda.selfcheck_reporting.capture_checkin") + def test_opens_in_progress_checkin(self, mock_capture): + """Sends IN_PROGRESS check-in and returns the check_in_id.""" + mock_capture.return_value = "abc123" + + check_in_id = start_sentry_checkin() + + self.assertEqual(check_in_id, "abc123") + mock_capture.assert_called_once_with( + monitor_slug=SENTRY_SLUG, + status="in_progress", + ) + + @override_settings(MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG) + @patch("core.mda.selfcheck_reporting.capture_checkin") + def test_error_swallowed(self, mock_capture): + """Failure in capture_checkin logs a warning and returns None.""" + mock_capture.side_effect = RuntimeError("sentry down") + + with self.assertLogs("core.mda.selfcheck_reporting", level="WARNING") as cm: + self.assertIsNone(start_sentry_checkin()) + + self.assertTrue( + any( + "Failed to open Sentry selfcheck check-in" in line for line in cm.output + ) + ) + + +class TestFinishSentryCheckin(TestCase): + """Tests for closing a Sentry cron check-in.""" + + @override_settings(MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=None) + @patch("core.mda.selfcheck_reporting.capture_checkin") + def test_noop_when_slug_unset(self, mock_capture): + """No Sentry call when slug is not configured.""" + finish_sentry_checkin("abc123", SUCCESS_RESULT) + mock_capture.assert_not_called() + + @override_settings(MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG) + @patch("core.mda.selfcheck_reporting.capture_checkin") + def test_noop_when_check_in_id_missing(self, mock_capture): + """No Sentry call when start_sentry_checkin returned None.""" + finish_sentry_checkin(None, SUCCESS_RESULT) + mock_capture.assert_not_called() + + @override_settings(MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG) + @patch("core.mda.selfcheck_reporting.capture_checkin") + def test_ok_with_duration_on_success(self, mock_capture): + """OK status with send+reception duration on success.""" + finish_sentry_checkin("abc123", SUCCESS_RESULT) + + mock_capture.assert_called_once_with( + monitor_slug=SENTRY_SLUG, + check_in_id="abc123", + status="ok", + duration=0.150 + 2.340, + ) + + @override_settings(MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG) + @patch("core.mda.selfcheck_reporting.capture_checkin") + def test_error_without_duration_on_failure(self, mock_capture): + """ERROR status and no duration when timing data is missing.""" + finish_sentry_checkin("abc123", FAILURE_RESULT) + + mock_capture.assert_called_once_with( + monitor_slug=SENTRY_SLUG, + check_in_id="abc123", + status="error", + duration=None, + ) + + @override_settings(MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG=SENTRY_SLUG) + @patch("core.mda.selfcheck_reporting.capture_checkin") + def test_error_swallowed(self, mock_capture): + """Failure in capture_checkin logs a warning, no raise.""" + mock_capture.side_effect = RuntimeError("sentry down") + + with self.assertLogs("core.mda.selfcheck_reporting", level="WARNING") as cm: + finish_sentry_checkin("abc123", SUCCESS_RESULT) + + self.assertTrue( + any( + "Failed to close Sentry selfcheck check-in" in line + for line in cm.output + ) + ) diff --git a/src/backend/messages/settings.py b/src/backend/messages/settings.py index 039a60a0..262e5965 100644 --- a/src/backend/messages/settings.py +++ b/src/backend/messages/settings.py @@ -507,6 +507,12 @@ class Base(Configuration): environ_prefix=None, ) + MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG = values.Value( + None, + environ_name="MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG", + environ_prefix=None, + ) + # Manual retry settings MESSAGES_MANUAL_RETRY_MAX_AGE = values.PositiveIntegerValue( 7 * 24 * 60 * 60, # 7 days in seconds