diff --git a/src/backend/core/mda/autoreply.py b/src/backend/core/mda/autoreply.py index 29b9e49c..d4584bae 100644 --- a/src/backend/core/mda/autoreply.py +++ b/src/backend/core/mda/autoreply.py @@ -82,30 +82,36 @@ def _is_auto_reply_message(headers: dict) -> bool: if not headers: return False - # Normalize header keys to lowercase for comparison + # ``headers`` follows the rfc5322 parser contract: + # - RFC max=1 headers (auto-submitted, …) are ``str`` + # - every other header is ``list[str]`` in document order + # Lowercase defensively so the function tolerates a caller that + # hasn't normalised yet (production callers go through the parser + # which already lowercases). lower_headers = {k.lower(): v for k, v in headers.items()} - # Return-Path: empty or <> means bounce (RFC 3834) - if "return-path" in lower_headers: - return_path = lower_headers["return-path"].strip() - if return_path in ("", "<>"): + # Return-Path: empty or <> means bounce (RFC 3834). Return-Path is + # not in _SCALAR_HEADERS so it's a list[str]; iterate every + # occurrence so a benign duplicate can't mask a bounce indicator. + for return_path in lower_headers.get("return-path", []): + if return_path.strip() in ("", "<>"): return True - # Auto-Submitted: anything other than "no" means auto-generated. - # RFC 3834 allows parameters after ";" (e.g. "auto-replied; owner-email=..."). + # Auto-Submitted: RFC 3834 §5 makes this max=1, so it's a str. + # Parameters after ";" (e.g. "auto-replied; owner-email=...") are + # stripped before comparison; anything other than "no" counts. auto_submitted = lower_headers.get("auto-submitted", "").strip().lower() if auto_submitted: - # Strip parameters: "auto-replied; foo=bar" -> "auto-replied" - auto_submitted_value = auto_submitted.split(";", 1)[0].strip() - if auto_submitted_value and auto_submitted_value != "no": + if auto_submitted.split(";", 1)[0].strip() not in ("", "no"): return True - # Precedence: bulk, list, junk - precedence = lower_headers.get("precedence", "").strip().lower() - if precedence in _PRECEDENCE_VALUES: - return True + # Precedence: bulk, list, junk. Repeatable per RFC 5322 (optional-field). + for precedence in lower_headers.get("precedence", []): + if precedence.strip().lower() in _PRECEDENCE_VALUES: + return True - # Presence of any loop header + # Presence of any loop header is enough (list-id, list-unsubscribe, + # x-loop, …). All of these are list-typed; truthy ⇔ non-empty. for header_name in _LOOP_HEADERS: if lower_headers.get(header_name): return True diff --git a/src/backend/core/mda/inbound_tasks.py b/src/backend/core/mda/inbound_tasks.py index 9a362664..1781e066 100644 --- a/src/backend/core/mda/inbound_tasks.py +++ b/src/backend/core/mda/inbound_tasks.py @@ -73,9 +73,12 @@ def _check_spam_with_hardcoded_rules( key = key.lower().strip() value = value.lower().strip() - # Get header value(s) - can be a string or list - header_value = headers.get(key) - if header_value is None: + # Existence check only — the actual value is read from + # headers_blocks below to apply the trusted-relays cut. + # ``headers`` follows the rfc5322 parser contract: + # str for RFC max=1 headers, list[str] otherwise; either + # shape is truthy when present, which is all we need here. + if headers.get(key) is None: continue # Use headers_blocks to identify which headers to trust based on trusted_relays config. diff --git a/src/backend/core/mda/rfc5322/parser.py b/src/backend/core/mda/rfc5322/parser.py index b2a31a18..32e3fcc8 100644 --- a/src/backend/core/mda/rfc5322/parser.py +++ b/src/backend/core/mda/rfc5322/parser.py @@ -43,6 +43,58 @@ class EmailParseError(Exception): """Exception raised for errors during email parsing.""" +# Header names (lowercase) that the relevant RFCs specify as appearing +# at most once per message. ``parsed["headers"][name]`` is ``str`` for +# these and ``list[str]`` (in document order) for every other header. +# When duplicated in the wild, the first occurrence wins — matching +# Python stdlib ``email.message.Message[name]`` semantics. +# +# Inclusion criterion: registered with max=1 in the IANA Provisional +# Message Header Field Registry (RFC 3864 / RFC 9057). The set is +# pre-emptive — many of these headers have no consumer in the code +# today, but pinning their type now means a future caller can write +# ``parsed["headers"]["x"].strip()`` without an isinstance check. +_SCALAR_HEADERS: frozenset[str] = frozenset( + { + # RFC 5322 §3.6 (origination, destination, identification, + # informational fields — all max=1): + "date", + "from", + "sender", + "reply-to", + "to", + "cc", + "bcc", + "message-id", + "in-reply-to", + "references", + "subject", + # RFC 3834 §5 — Auto-Submitted MUST NOT appear more than once: + "auto-submitted", + # RFC 2045 / 2046 / 2183 single-valued MIME container fields: + "mime-version", + "content-type", + "content-id", + "content-transfer-encoding", + "content-description", + "content-disposition", + # RFC 4021 §2.1.55 / .56 / .57 — message-class indicators: + "importance", + "priority", + "sensitivity", + # RFC 8098 §2.1 — MDN request (MUST NOT appear more than once): + "disposition-notification-to", + # RFC 3798 — MDN reporting: + "original-message-id", + # RFC 5703 §3.4 — Sieve archive original-envelope: + "original-from", + "original-subject", + # RFC 8058 §3.1 — one-click unsubscribe POST body: + "list-unsubscribe-post", + } +) + + def decode_email_header_text(header_text: str) -> str: """ Decode email header text that might be encoded (RFC 2047). @@ -724,17 +776,36 @@ def _parse_labels_header(labels_str: str) -> list: def parse_email_message(raw_email_bytes: bytes) -> Optional[Dict[str, Any]]: - """ - Parse a raw email message (bytes) into a structured dictionary following JMAP format. + """Parse a raw email message (bytes) into a structured dict (JMAP-ish). + + The returned dict carries three header views, each with a fixed + type contract so callers don't need ``isinstance`` checks: + + - ``headers`` — ``dict[str, str | list[str]]`` keyed by lowercase + name. RFC max=1 headers (see ``_SCALAR_HEADERS``) are ``str``; + first occurrence wins on duplication, matching stdlib + ``email.message.Message[name]``. Every other header is + ``list[str]`` in document order — ``received``, ``return-path``, + ``dkim-signature``, ``authentication-results``, ``arc-*``, + ``list-*``, ``comments``, ``keywords``, every ``X-*``. + - ``headers_list`` — ``list[tuple[str, str]]`` of every header in + document order, with lowercase names. Source of truth for full + occurrence + ordering. + - ``headers_blocks`` — ``list[dict[str, list[str]]]``. Each block + ends with a ``Received`` header; everything above (earlier) it + is in the same trust scope. Values inside a block are *always* + ``list[str]`` regardless of the header's spec, so trusted-relays + filters can index uniformly. Args: - raw_email_bytes: Raw email data as bytes + raw_email_bytes: Raw email data as bytes. Returns: - Dictionary containing parsed email data, or None if parsing fails fundamentally. + Dict of parsed fields, or raises ``EmailParseError`` on + fundamental parse failure. Raises: - EmailParseError: If parsing fails with a specific error we want to propagate. + EmailParseError: parsing failed. """ if not raw_email_bytes or not isinstance(raw_email_bytes, bytes): # Ensure input is non-empty bytes @@ -768,15 +839,12 @@ def parse_email_message(raw_email_bytes: bytes) -> Optional[Dict[str, Any]]: key_lower = k.lower() headers_list.append((key_lower, decoded_value)) - # Build headers dict (for compatibility) - if key_lower in headers: - current_value = headers[key_lower] - if isinstance(current_value, list): - current_value.append(decoded_value) - else: - headers[key_lower] = [current_value, decoded_value] + if key_lower in _SCALAR_HEADERS: + # First occurrence wins (RFC max=1; matches stdlib). + headers.setdefault(key_lower, decoded_value) else: - headers[key_lower] = decoded_value + # Repeatable: always list[str] in document order. + headers.setdefault(key_lower, []).append(decoded_value) # Split headers into blocks based on Received headers # Each Received header marks the END of its block - everything above it (before it in the list) is trusted @@ -804,29 +872,24 @@ def parse_email_message(raw_email_bytes: bytes) -> Optional[Dict[str, Any]]: gmail_labels = [] seen_labels = set() - # Parse X-Gmail-Labels (Google Takeout format) + # Parse X-Gmail-Labels (Google Takeout format). The header is + # not in _SCALAR_HEADERS so it's a list[str]; only the first + # occurrence is meaningful for label semantics. if "x-gmail-labels" in headers: - labels_str = headers["x-gmail-labels"] - if isinstance(labels_str, list): - labels_str = labels_str[0] # Take first value if multiple - for label in _parse_labels_header(labels_str): + for label in _parse_labels_header(headers["x-gmail-labels"][0]): if label not in seen_labels: seen_labels.add(label) gmail_labels.append(label) - # Parse X-Keywords (Dovecot/OfflineIMAP/mu4e format) + # Parse X-Keywords (Dovecot/OfflineIMAP/mu4e format). if "x-keywords" in headers: - labels_str = headers["x-keywords"] - if isinstance(labels_str, list): - labels_str = labels_str[0] # Take first value if multiple - for label in _parse_labels_header(labels_str): + for label in _parse_labels_header(headers["x-keywords"][0]): if label not in seen_labels: seen_labels.add(label) gmail_labels.append(label) subject = headers.get("subject", "") - from_header_decoded = headers.get("from", "") - from_name, from_addr = parse_email_address(from_header_decoded) + from_name, from_addr = parse_email_address(headers.get("from", "")) to_recipients = parse_email_addresses(headers.get("to", "")) cc_recipients = parse_email_addresses(headers.get("cc", "")) bcc_recipients = parse_email_addresses(headers.get("bcc", "")) diff --git a/src/backend/core/models.py b/src/backend/core/models.py index b2339a25..a722206a 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -2007,16 +2007,25 @@ class Message(BaseModel): """Get a parsed field from the parsed email data.""" return (self.get_parsed_data() or {}).get(field_name) - def get_mime_headers(self) -> Dict[str, str]: - """Get the MIME headers of the message.""" + def get_mime_headers(self) -> Dict[str, Any]: + """Get the MIME headers of the message. + + Values follow the rfc5322 parser contract: ``str`` for RFC max=1 + headers, ``list[str]`` in document order for every other header. + """ return self.get_parsed_data().get("headers", {}) def get_stmsg_headers(self) -> Dict[str, str]: - """Get the STMSG headers of the message.""" + """Get the STMSG headers of the message. + + ``X-StMsg-*`` headers are stamped by our own MTA pipeline (one + per message) and any sender-supplied copies are stripped before + parsing. They surface as single-element lists; take the first. + """ return { - k[len("x-stmsg-") :].lower(): v + k[len("x-stmsg-") :].lower(): v[0] for k, v in self.get_parsed_data().get("headers", {}).items() - if k.startswith("x-stmsg-") + if k.startswith("x-stmsg-") and v } def generate_mime_id(self) -> str: diff --git a/src/backend/core/tests/mda/test_autoreply.py b/src/backend/core/tests/mda/test_autoreply.py index 5834665c..ccea383f 100644 --- a/src/backend/core/tests/mda/test_autoreply.py +++ b/src/backend/core/tests/mda/test_autoreply.py @@ -111,42 +111,42 @@ class TestIsAutoReplyMessage: def test_precedence_bulk(self): """Precedence: bulk is detected.""" - headers = {"Precedence": "bulk"} + headers = {"Precedence": ["bulk"]} assert _is_auto_reply_message(headers) is True def test_precedence_list(self): """Precedence: list is detected.""" - headers = {"Precedence": "list"} + headers = {"Precedence": ["list"]} assert _is_auto_reply_message(headers) is True def test_precedence_junk(self): """Precedence: junk is detected.""" - headers = {"Precedence": "junk"} + headers = {"Precedence": ["junk"]} assert _is_auto_reply_message(headers) is True def test_list_id_header(self): """List-Id header is detected.""" - headers = {"List-Id": ""} + headers = {"List-Id": [""]} assert _is_auto_reply_message(headers) is True def test_list_unsubscribe_header(self): """List-Unsubscribe header is detected.""" - headers = {"List-Unsubscribe": ""} + headers = {"List-Unsubscribe": [""]} assert _is_auto_reply_message(headers) is True def test_x_auto_response_suppress(self): """X-Auto-Response-Suppress header is detected.""" - headers = {"X-Auto-Response-Suppress": "All"} + headers = {"X-Auto-Response-Suppress": ["All"]} assert _is_auto_reply_message(headers) is True def test_x_autoreply(self): """X-Autoreply header is detected.""" - headers = {"X-Autoreply": "yes"} + headers = {"X-Autoreply": ["yes"]} assert _is_auto_reply_message(headers) is True def test_x_autorespond(self): """X-Autorespond header is detected.""" - headers = {"X-Autorespond": "yes"} + headers = {"X-Autorespond": ["yes"]} assert _is_auto_reply_message(headers) is True def test_auto_submitted_with_parameters(self): @@ -165,47 +165,47 @@ class TestIsAutoReplyMessageExtended: def test_return_path_null(self): """Return-Path: <> (null sender) is detected.""" - headers = {"Return-Path": "<>"} + headers = {"Return-Path": ["<>"]} assert _is_auto_reply_message(headers) is True def test_return_path_empty(self): """Return-Path with empty value is detected.""" - headers = {"Return-Path": ""} + headers = {"Return-Path": [""]} assert _is_auto_reply_message(headers) is True def test_list_post_header(self): """List-Post header is detected.""" - headers = {"List-Post": ""} + headers = {"List-Post": [""]} assert _is_auto_reply_message(headers) is True def test_list_help_header(self): """List-Help header is detected.""" - headers = {"List-Help": ""} + headers = {"List-Help": [""]} assert _is_auto_reply_message(headers) is True def test_list_subscribe_header(self): """List-Subscribe header is detected.""" - headers = {"List-Subscribe": ""} + headers = {"List-Subscribe": [""]} assert _is_auto_reply_message(headers) is True def test_list_owner_header(self): """List-Owner header is detected.""" - headers = {"List-Owner": ""} + headers = {"List-Owner": [""]} assert _is_auto_reply_message(headers) is True def test_list_archive_header(self): """List-Archive header is detected.""" - headers = {"List-Archive": ""} + headers = {"List-Archive": [""]} assert _is_auto_reply_message(headers) is True def test_x_loop_header(self): """X-Loop header is detected.""" - headers = {"X-Loop": "yes"} + headers = {"X-Loop": ["yes"]} assert _is_auto_reply_message(headers) is True def test_feedback_id_header(self): """Feedback-ID header is detected (Gmail newsletters).""" - headers = {"Feedback-ID": "123:campaign:gmail"} + headers = {"Feedback-ID": ["123:campaign:gmail"]} assert _is_auto_reply_message(headers) is True diff --git a/src/backend/core/tests/mda/test_inbound_auth.py b/src/backend/core/tests/mda/test_inbound_auth.py index e4a9c21c..d4cdfda9 100644 --- a/src/backend/core/tests/mda/test_inbound_auth.py +++ b/src/backend/core/tests/mda/test_inbound_auth.py @@ -519,7 +519,7 @@ class TestProcessInboundMessageAuthIntegration: call_kwargs = mock_create_message.call_args[1] assert call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth: none\r\n") parsed = call_kwargs["parsed_email"] - assert parsed["headers"].get("x-stmsg-sender-auth") == "none" + assert parsed["headers"].get("x-stmsg-sender-auth") == ["none"] @override_settings(SPAM_CONFIG={"inbound_auth": "rspamd"}) @patch("core.mda.inbound_tasks.check_inbound_authentication") @@ -541,7 +541,7 @@ class TestProcessInboundMessageAuthIntegration: call_kwargs = mock_create_message.call_args[1] assert call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth: fail\r\n") parsed = call_kwargs["parsed_email"] - assert parsed["headers"].get("x-stmsg-sender-auth") == "fail" + assert parsed["headers"].get("x-stmsg-sender-auth") == ["fail"] @override_settings(SPAM_CONFIG={"inbound_auth": "native"}) @patch("core.mda.inbound_tasks.check_inbound_authentication") @@ -683,7 +683,7 @@ class TestProcessInboundMessageAuthIntegration: """After prepending, the parser exposes the header via x-stmsg-*.""" tagged = b"X-StMsg-Sender-Auth: fail\r\n" + RAW_EMAIL parsed = parse_email_message(tagged) - assert parsed["headers"].get("x-stmsg-sender-auth") == "fail" + assert parsed["headers"].get("x-stmsg-sender-auth") == ["fail"] @override_settings(SPAM_CONFIG={"inbound_auth": "native"}) @patch("core.mda.inbound_tasks.parse_email_message") diff --git a/src/backend/core/tests/mda/test_rfc5322_parser.py b/src/backend/core/tests/mda/test_rfc5322_parser.py index 8e62f76e..e6cb37fd 100644 --- a/src/backend/core/tests/mda/test_rfc5322_parser.py +++ b/src/backend/core/tests/mda/test_rfc5322_parser.py @@ -851,10 +851,11 @@ Text part. parsed = parse_email_message(message.to_string().encode("utf-8")) assert parsed is not None assert parsed["subject"] == "Custom Headers" - assert "x-custom-header" in parsed["headers"] - assert parsed["headers"]["x-custom-header"] == "Custom Value" - assert parsed["headers"]["x-priority"] == "1" - assert parsed["headers"]["x-mailer"] == "Custom Mailer v1.0" + # Non-scalar headers (X-*, optional-field per RFC 5322 §3.6.8) + # are stored as list[str] in document order. + assert parsed["headers"]["x-custom-header"] == ["Custom Value"] + assert parsed["headers"]["x-priority"] == ["1"] + assert parsed["headers"]["x-mailer"] == ["Custom Mailer v1.0"] # Check headers_list contains custom headers in order assert "headers_list" in parsed @@ -2169,5 +2170,745 @@ Content-Type: text/html; charset="utf-8" assert "htmlBody" in parsed +class TestUnrecognisedMessageSubtypeRobustness: + """Coverage for inbound bounces and any ``message/*`` content type + the underlying MIME engine has no dedicated branch for. + + RFC 5337 / RFC 6533 i18n DSN variants + (``message/global-delivery-status``, + ``message/global-disposition-notification``, + ``message/global-headers``), plus other standardised ``message/*`` + subtypes (``message/partial`` RFC 2046, ``message/imdn+xml`` RFC 5438, + ``message/sip`` RFC 3261, ``message/cpim`` RFC 3862, …) and any + vendor / unknown subtype must all parse successfully when nested + inside a ``multipart/report`` bounce. + + Tests exercise the public ``parse_email_message`` API only — no + MIME-engine internals — so they remain valid if the parser backend + is swapped (e.g. flanker → ``email.parser``). The behavioural + contract: + + 1. Parsing must not raise ``EmailParseError`` on any well-formed + multipart/report whose status part uses an unrecognised + ``message/*`` subtype. + 2. The human-readable notification text part must survive parsing + so the bounce remains useful to the recipient. + 3. Legacy bounce part types must continue to parse correctly. + 4. Top-level unrecognised ``message/*`` content types are accepted + too, not only the multipart-nested case. + """ + + BOUNCE_BOUNDARY = "boundary-dsn" + NOTIFICATION_TEXT = "Your message could not be delivered to one or more recipients." + + @classmethod + def _build_dsn(cls, status_content_type: str, status_body: bytes) -> bytes: + """Build a realistic multipart/report bounce wrapping ``status_body``.""" + boundary = cls.BOUNCE_BOUNDARY.encode() + return ( + b"From: MAILER-DAEMON@mta.example.com (Mail Delivery System)\r\n" + b"To: sender@example.org\r\n" + b"Subject: Undelivered Mail Returned to Sender\r\n" + b"Date: Thu, 4 Jun 2026 02:30:20 +0200 (CEST)\r\n" + b"MIME-Version: 1.0\r\n" + b"Content-Type: multipart/report; report-type=delivery-status;\r\n" + b'\tboundary="' + boundary + b'"\r\n' + b"\r\n" + b"This is a MIME-encapsulated message.\r\n" + b"\r\n" + b"--" + boundary + b"\r\n" + b"Content-Description: Notification\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"\r\n" + cls.NOTIFICATION_TEXT.encode() + b"\r\n" + b"\r\n" + b"--" + boundary + b"\r\n" + b"Content-Description: Delivery report\r\n" + b"Content-Type: " + status_content_type.encode() + b"\r\n" + b"\r\n" + status_body + b"\r\n" + b"\r\n" + b"--" + boundary + b"--\r\n" + ) + + @pytest.mark.parametrize( + "content_type", + [ + # RFC 5337 / 6533 i18n equivalents of the legacy bounce / MDN / + # rfc822-headers parts. These need predicate-level recognition + # to be classified correctly (not just treated as opaque). + "message/global-delivery-status", + "message/global-disposition-notification", + "message/global-headers", + # Other standardised ``message/*`` subtypes — any unknown + # subtype must be tolerated, not only the i18n ones. + "message/partial", # RFC 2046 §5.2.2 fragmented messages + "message/imdn+xml", # RFC 5438 IMDN + "message/sip", # RFC 3261 SIP signalling + "message/sipfrag", # RFC 3420 partial SIP messages + "message/cpim", # RFC 3862 Common Presence and IM + # Vendor / unknown subtypes must also be tolerated. + "message/x-vendor-future", + "message/x-totally-unknown", + ], + ) + def test_dsn_with_unrecognised_message_subtype(self, content_type): + """A multipart/report bounce whose status part uses any unrecognised + ``message/*`` subtype must parse without raising, and the + human-readable notification text part must be preserved.""" + raw = self._build_dsn( + content_type, + b"Reporting-MTA: dns; mta.example.com\r\n" + b"\r\n" + b"Final-Recipient: rfc822; recipient@example.com\r\n" + b"Action: failed\r\n" + b"Status: 5.0.0", + ) + parsed = parse_email_message(raw) + assert parsed is not None + assert parsed["subject"] == "Undelivered Mail Returned to Sender" + assert parsed["from"]["email"] == "MAILER-DAEMON@mta.example.com" + assert any( + self.NOTIFICATION_TEXT in part["content"] for part in parsed["textBody"] + ), ( + f"notification text missing from textBody for {content_type}; " + f"got parts: {[p.get('content', '')[:60] for p in parsed['textBody']]}" + ) + + @pytest.mark.parametrize( + "content_type", + [ + "message/delivery-status", + "message/disposition-notification", + "text/rfc822-headers", + ], + ) + def test_dsn_with_legacy_recognised_subtype(self, content_type): + """Legacy bounce part types must keep parsing correctly.""" + raw = self._build_dsn( + content_type, + b"Reporting-MTA: dns; mta.example.com\r\n" + b"\r\n" + b"Final-Recipient: rfc822; recipient@example.com\r\n" + b"Action: failed\r\n" + b"Status: 5.0.0", + ) + parsed = parse_email_message(raw) + assert parsed is not None + assert parsed["subject"] == "Undelivered Mail Returned to Sender" + assert any( + self.NOTIFICATION_TEXT in part["content"] for part in parsed["textBody"] + ) + + def test_postfix_i18n_dsn_full_shape(self): + """End-to-end shape: a Postfix bounce wrapping RFC 6533 + ``message/global-delivery-status`` inside ``multipart/report``, + with the full set of headers Postfix typically emits.""" + raw = self._build_dsn( + "message/global-delivery-status", + b"Reporting-MTA: dns; mta.example.com\r\n" + b"X-Postfix-Queue-ID: ABC123\r\n" + b"X-Postfix-Sender: rfc822; sender@example.org\r\n" + b"\r\n" + b"Final-Recipient: rfc822; user@example.com\r\n" + b"Original-Recipient: rfc822;user@example.com\r\n" + b"Action: failed\r\n" + b"Status: 5.0.0", + ) + parsed = parse_email_message(raw) + assert parsed["subject"] == "Undelivered Mail Returned to Sender" + # Sanity-check the headers that downstream bounce handlers read. + assert parsed["headers"].get("from", "").startswith("MAILER-DAEMON") + assert "multipart/report" in parsed["headers"]["content-type"] + + def test_top_level_unrecognised_message_subtype(self): + """An email whose ROOT Content-Type is an unrecognised + ``message/*`` subtype must parse — the fallback applies at the + root, not only nested inside a multipart.""" + raw = ( + b"From: sender@example.com\r\n" + b"To: recipient@example.com\r\n" + b"Subject: Weird top-level subtype\r\n" + b"MIME-Version: 1.0\r\n" + b"Content-Type: message/x-something-vendor\r\n" + b"\r\n" + b"opaque payload bytes\r\n" + ) + # The contract is "do not raise". Whether the body bytes are + # surfaced verbatim in textBody is implementation-defined and + # not asserted here, so the test survives a backend swap. + parsed = parse_email_message(raw) + assert parsed is not None + assert parsed["subject"] == "Weird top-level subtype" + assert parsed["from"]["email"] == "sender@example.com" + + def test_unrecognised_subtype_nested_in_message_container(self): + """A ``message/rfc822``-wrapped inner message with its own + unrecognised ``message/*`` content type must parse — the outer + container should not be affected by an unknown inner subtype.""" + raw = ( + b"From: x@example.com\r\n" + b"To: y@example.com\r\n" + b"Subject: Forwarded weird\r\n" + b"MIME-Version: 1.0\r\n" + b'Content-Type: multipart/mixed; boundary="OUTER"\r\n' + b"\r\n" + b"--OUTER\r\n" + b"Content-Type: text/plain\r\n" + b"\r\n" + b"See the attached forwarded message.\r\n" + b"\r\n" + b"--OUTER\r\n" + b"Content-Type: message/rfc822\r\n" + b"\r\n" + b"From: inner@example.com\r\n" + b"Subject: Inner\r\n" + b"Content-Type: message/x-unknown-vendor\r\n" + b"\r\n" + b"opaque body\r\n" + b"--OUTER--\r\n" + ) + parsed = parse_email_message(raw) + assert parsed is not None + assert parsed["subject"] == "Forwarded weird" + assert any("See the attached" in part["content"] for part in parsed["textBody"]) + + def test_multiple_unrecognised_subtypes_in_same_report(self): + """A bounce can contain multiple status sub-parts with mixed + legacy and i18n content types. None of them should crash and + the notification text must survive.""" + raw = ( + b"From: MAILER-DAEMON@mta.example.com\r\n" + b"To: sender@example.org\r\n" + b"Subject: Mixed bounce\r\n" + b"MIME-Version: 1.0\r\n" + b"Content-Type: multipart/report; report-type=delivery-status;\r\n" + b'\tboundary="MULTI"\r\n' + b"\r\n" + b"--MULTI\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"\r\n" + self.NOTIFICATION_TEXT.encode() + b"\r\n" + b"--MULTI\r\n" + b"Content-Type: message/delivery-status\r\n" + b"\r\n" + b"Reporting-MTA: dns; legacy.example.com\r\n" + b"--MULTI\r\n" + b"Content-Type: message/global-delivery-status\r\n" + b"\r\n" + b"Reporting-MTA: dns; i18n.example.com\r\n" + b"--MULTI\r\n" + b"Content-Type: message/x-something-unknown\r\n" + b"\r\n" + b"opaque\r\n" + b"--MULTI--\r\n" + ) + parsed = parse_email_message(raw) + assert parsed is not None + assert parsed["subject"] == "Mixed bounce" + assert any( + self.NOTIFICATION_TEXT in part["content"] for part in parsed["textBody"] + ) + + +class TestScalarHeaderDuplicates: + """RFC 5322 §3.6 makes every scalar header — Date, From, Sender, + Reply-To, To, Cc, Bcc, Message-ID, In-Reply-To, References, Subject — + appear at most once. Real-world senders sometimes emit duplicates + anyway, and the parser must tolerate that. + + Behaviour matches the Python stdlib's + ``email.message.Message.__getitem__``: when a header is repeated, + the parser silently uses the first occurrence. Tests pin that + contract through the public ``parse_email_message`` API so they + survive a swap from flanker to ``email.parser``. + """ + + BASE_HEADERS = { + "From": "sender@example.com", + "To": "recipient@example.com", + "Subject": "hello", + "Date": "Thu, 4 Jun 2026 00:47:09 +0000", + "Message-ID": "", + } + + @classmethod + def _build(cls, **overrides: object) -> bytes: + """Build a minimal email. ``overrides`` may map a header name to + a single str (default behaviour) or a list[str] (the header is + emitted multiple times, in that order).""" + headers = {**cls.BASE_HEADERS, **overrides} + lines: list[str] = [] + for name, value in headers.items(): + if isinstance(value, list): + for v in value: + lines.append(f"{name}: {v}") + else: + lines.append(f"{name}: {value}") + lines += ["", "body"] + return ("\r\n".join(lines)).encode("utf-8") + + @pytest.mark.parametrize( + "header_name,values,result_key,expected_first", + [ + # Identification: Message-ID and In-Reply-To strip <> on the + # first value; the second one must not leak into the result. + ( + "Message-ID", + ["", ""], + "message_id", + "first@example.com", + ), + ( + "In-Reply-To", + ["", ""], + "in_reply_to", + "parent-first@example.com", + ), + ( + "References", + ["", ""], + "references", + "", + ), + ( + "Subject", + ["first subject", "second subject"], + "subject", + "first subject", + ), + ], + ) + def test_duplicate_scalar_header_takes_first( + self, header_name, values, result_key, expected_first + ): + """A duplicated scalar header must surface only its first + occurrence in the structured result.""" + raw = self._build(**{header_name: values}) + parsed = parse_email_message(raw) + assert parsed is not None + assert parsed[result_key] == expected_first + + def test_duplicate_from_takes_first(self): + """``From`` is parsed into ``{name, email}`` — first occurrence wins.""" + raw = self._build(**{"From": ["first@example.com", "second@example.com"]}) + parsed = parse_email_message(raw) + assert parsed["from"]["email"] == "first@example.com" + + @pytest.mark.parametrize( + "header_name,result_key", + [("To", "to"), ("Cc", "cc"), ("Bcc", "bcc")], + ) + def test_duplicate_address_list_header_takes_first(self, header_name, result_key): + """Only the first occurrence's addresses should appear in the + structured result for duplicated address-list headers.""" + raw = self._build(**{header_name: ["first@example.com", "second@example.com"]}) + parsed = parse_email_message(raw) + assert len(parsed[result_key]) == 1 + assert parsed[result_key][0]["email"] == "first@example.com" + + def test_duplicate_date_takes_first(self): + """The first ``Date`` header wins.""" + first = "Thu, 4 Jun 2026 00:47:09 +0000" + second = "Fri, 5 Jun 2026 00:00:00 +0000" + raw = self._build(Date=[first, second]) + parsed = parse_email_message(raw) + # Assert only the day-of-month — uniquely identifies the choice + # without depending on tz formatting which can differ across + # parser backends. + assert parsed["date"] is not None + assert parsed["date"].day == 4 + + def test_dual_message_id_returns_first(self): + """A message with two ``Message-ID`` headers must yield a single + deterministic Message-ID downstream so the ``mime_id``-based + duplicate-message check in inbound delivery has something + stable to compare against.""" + raw = self._build( + **{ + "Message-ID": [ + "<0S7NGNc8g9oEF8bStCvPthDYCCU0T9dnM20qLmmECY@example.com>", + "", + ] + } + ) + parsed = parse_email_message(raw) + assert isinstance(parsed["message_id"], str) + assert ( + parsed["message_id"] + == "0S7NGNc8g9oEF8bStCvPthDYCCU0T9dnM20qLmmECY@example.com" + ) + + def test_no_duplication_still_works(self): + """Regression guard: emails without any duplicated header must + keep producing the same scalar fields after the helper change.""" + raw = self._build() + parsed = parse_email_message(raw) + assert parsed["subject"] == "hello" + assert parsed["from"]["email"] == "sender@example.com" + assert parsed["to"][0]["email"] == "recipient@example.com" + assert parsed["message_id"] == "canonical@example.com" + + def test_every_scalar_header_duplicated_simultaneously(self): + """Stress test: every scalar header per RFC 5322 §3.6 emitted + twice in the same email must not crash the parser.""" + raw = self._build( + **{ + "From": ["first-from@example.com", "second-from@example.com"], + "To": ["first-to@example.com", "second-to@example.com"], + "Cc": ["first-cc@example.com", "second-cc@example.com"], + "Bcc": ["first-bcc@example.com", "second-bcc@example.com"], + "Subject": ["first subj", "second subj"], + "Date": [ + "Thu, 4 Jun 2026 00:47:09 +0000", + "Fri, 5 Jun 2026 00:00:00 +0000", + ], + "Message-ID": ["", ""], + "In-Reply-To": ["", ""], + "References": ["", ""], + "Reply-To": ["first-rt@example.com", "second-rt@example.com"], + "Sender": ["first-sender@example.com", "second-sender@example.com"], + } + ) + parsed = parse_email_message(raw) + # Every scalar field must be a plain str (or scalar-shaped + # dict for ``from``) — never a list. + assert isinstance(parsed["subject"], str) + assert isinstance(parsed["from"]["email"], str) + assert isinstance(parsed["message_id"], str) + assert isinstance(parsed["in_reply_to"], str) + assert isinstance(parsed["references"], str) + # And the first values won. + assert parsed["subject"] == "first subj" + assert parsed["from"]["email"] == "first-from@example.com" + assert parsed["message_id"] == "id-a@example.com" + assert parsed["in_reply_to"] == "irt-a@example.com" + assert parsed["references"] == "" + + +class TestParsedHeadersTypeContract: + """The ``parsed["headers"]`` dict follows a fixed per-header contract: + + - Headers registered with max=1 in the IANA Provisional Message + Header Field Registry (RFC 5322 §3.6, RFC 3834 §5, RFC 2045 / + 2046 / 2183, RFC 4021, RFC 3798, RFC 5703, RFC 8058, RFC 8098) + are stored as ``str``. On duplication, the first occurrence + wins — matching stdlib ``email.message.Message[name]`` semantics. + - Every other header (``received``, ``return-path``, ``precedence``, + ``dkim-signature``, ``authentication-results``, ``arc-*``, + ``list-*``, ``comments``, ``keywords``, every ``X-*`` / + optional-field) is stored as ``list[str]`` in document order. + + Consumers can therefore rely on the type at the call site based on + the header name alone — no runtime ``isinstance`` checks. Tests + here exercise the public API only so they survive a swap to the + Python stdlib parser. + """ + + @pytest.mark.parametrize( + "header_name", + [ + # RFC 5322 §3.6 + "subject", + "from", + "sender", + "reply-to", + "to", + "cc", + "bcc", + "date", + "message-id", + "in-reply-to", + "references", + # RFC 3834 §5 + "auto-submitted", + # RFC 2045 / 2046 / 2183 + "mime-version", + "content-type", + "content-transfer-encoding", + # RFC 4021 — message-class indicators + "importance", + "priority", + "sensitivity", + # RFC 8098 — MDN request + "disposition-notification-to", + # RFC 3798 — MDN reporting + "original-message-id", + # RFC 5703 — Sieve archive + "original-from", + "original-subject", + # RFC 8058 — one-click unsubscribe + "list-unsubscribe-post", + ], + ) + def test_scalar_header_is_str(self, header_name): + """Every header registered with max=1 in the IANA registry must + be ``str`` in ``parsed["headers"]``, even when the input had + only one occurrence. Pre-emptive coverage: a future consumer + reading any of these can rely on the type without + ``isinstance``.""" + raw = ( + b"From: sender@example.com\r\n" + b"To: recipient@example.com\r\n" + b"Date: Thu, 4 Jun 2026 00:47:09 +0000\r\n" + b"Subject: hi\r\n" + b"Sender: real-sender@example.com\r\n" + b"Reply-To: replies@example.com\r\n" + b"Cc: cc@example.com\r\n" + b"Bcc: bcc@example.com\r\n" + b"Message-ID: \r\n" + b"In-Reply-To: \r\n" + b"References: \r\n" + b"Auto-Submitted: no\r\n" + b"Importance: high\r\n" + b"Priority: urgent\r\n" + b"Sensitivity: Personal\r\n" + b"Disposition-Notification-To: mdn@example.com\r\n" + b"Original-Message-ID: \r\n" + b"Original-From: orig@example.com\r\n" + b"Original-Subject: original subject\r\n" + b"List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n" + b"MIME-Version: 1.0\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Transfer-Encoding: 7bit\r\n" + b"\r\n" + b"body\r\n" + ) + parsed = parse_email_message(raw) + value = parsed["headers"].get(header_name) + assert isinstance(value, str), ( + f"{header_name} expected str, got {type(value).__name__}" + ) + + @pytest.mark.parametrize( + "header_name,header_value", + [ + # Trace fields — explicitly unlimited per RFC 5322 §3.6.7. + ("received", "from a.example.com by b.example.com"), + ("return-path", ""), + # RFC 5322 §3.6.5 — unlimited. + ("comments", "a comment"), + ("keywords", "tag1, tag2"), + # Optional-field — RFC 5322 §3.6.8 unlimited. + ("precedence", "bulk"), + ("x-mailer", "PHPMailer 7.1.1"), + ("x-priority", "1"), + # RFC 6376 — multiple signatures expected. + ("dkim-signature", "v=1; a=rsa-sha256; d=example.com; ..."), + # RFC 8601 — one per authserv-id, repeatable. + ("authentication-results", "mta.example.com; spf=pass"), + # RFC 2369 list-* — repeatable in practice. + ("list-id", ""), + ("list-unsubscribe", ""), + ], + ) + def test_repeatable_header_is_list(self, header_name, header_value): + """Every header outside ``_SCALAR_HEADERS`` must be ``list[str]`` + even when it appears only once.""" + raw = ( + b"From: sender@example.com\r\n" + b"Subject: hi\r\n" + + header_name.encode("ascii") + + b": " + + header_value.encode("ascii") + + b"\r\n" + b"\r\n" + b"body\r\n" + ) + parsed = parse_email_message(raw) + value = parsed["headers"].get(header_name) + assert isinstance(value, list), ( + f"{header_name} expected list, got {type(value).__name__}" + ) + assert value == [header_value] + + def test_multiple_received_preserved_in_order(self): + """``Received`` is repeatable — every hop must appear in + document order so spam / forensics consumers can walk the chain.""" + raw = ( + b"Received: from hop3 by hop4\r\n" + b"Received: from hop2 by hop3\r\n" + b"Received: from hop1 by hop2\r\n" + b"From: sender@example.com\r\n" + b"Subject: traced\r\n" + b"\r\n" + b"body\r\n" + ) + parsed = parse_email_message(raw) + received = parsed["headers"]["received"] + assert isinstance(received, list) + assert len(received) == 3 + assert "hop3 by hop4" in received[0] + assert "hop2 by hop3" in received[1] + assert "hop1 by hop2" in received[2] + + def test_multiple_dkim_signatures_preserved(self): + """Multiple ``DKIM-Signature`` headers (RFC 6376) — each relay + can add its own. All must survive parsing as list entries.""" + sig_a = "v=1; a=rsa-sha256; d=wanadoo.fr; s=t20230301; bh=AAA" + sig_b = "v=1; a=rsa-sha256; d=ac-limoges.fr; s=default; bh=BBB" + raw = ( + b"From: sender@example.com\r\n" + b"Subject: dkim test\r\n" + b"DKIM-Signature: " + sig_a.encode() + b"\r\n" + b"DKIM-Signature: " + sig_b.encode() + b"\r\n" + b"\r\n" + b"body\r\n" + ) + parsed = parse_email_message(raw) + signatures = parsed["headers"]["dkim-signature"] + assert isinstance(signatures, list) + assert len(signatures) == 2 + assert "wanadoo.fr" in signatures[0] + assert "ac-limoges.fr" in signatures[1] + + @pytest.mark.parametrize( + "header_name", + [ + "received", + "dkim-signature", + "authentication-results", + "comments", + "keywords", + "resent-from", + "resent-to", + "list-id", + "x-custom", + ], + ) + def test_repeatable_header_preserves_all_occurrences(self, header_name): + """Every occurrence of a repeatable header must be retained in + document order — no silent deduplication or truncation.""" + raw = ( + b"From: sender@example.com\r\nSubject: hi\r\n" + + (header_name.encode() + b": value-A\r\n") + + (header_name.encode() + b": value-B\r\n") + + (header_name.encode() + b": value-C\r\n") + + b"\r\nbody\r\n" + ) + parsed = parse_email_message(raw) + values = parsed["headers"][header_name] + assert values == ["value-A", "value-B", "value-C"], ( + f"{header_name}: expected all three preserved in order, got {values!r}" + ) + + def test_headers_blocks_always_uses_list_values(self): + """Inside ``headers_blocks`` every value is ``list[str]``, + even for scalar headers like Subject — block consumers index + uniformly so the trusted-relays cut stays simple. + + ``parsed["headers"]`` and ``parsed["headers_blocks"]`` have + intentionally different shapes; pin both so a future refactor + doesn't quietly unify them. + """ + raw = ( + b"Received: from hop1 by hop2\r\n" + b"From: sender@example.com\r\n" + b"Subject: scalar in block\r\n" + b"\r\nbody\r\n" + ) + parsed = parse_email_message(raw) + # parsed["headers"]["subject"] is the str scalar. + assert parsed["headers"]["subject"] == "scalar in block" + # parsed["headers_blocks"][N]["subject"] is the same value + # wrapped in a single-element list. + for block in parsed["headers_blocks"]: + if "subject" in block: + assert isinstance(block["subject"], list) + assert block["subject"] == ["scalar in block"] + break + else: # pragma: no cover — fail loudly if no block had Subject + pytest.fail("Subject not found in any header block") + + @pytest.mark.parametrize("case_variant", ["From", "FROM", "from", "FrOm"]) + def test_scalar_header_lookup_is_case_insensitive(self, case_variant): + """Header names in input are case-insensitive per RFC 5322 + §3.6.8; ``parsed["headers"]`` keys are always lowercase and + callers must reach values via the lowercase form regardless of + the wire casing.""" + raw = ( + case_variant.encode("ascii") + b": sender@example.com\r\n" + b"Subject: hi\r\n\r\nbody\r\n" + ) + parsed = parse_email_message(raw) + assert parsed["headers"]["from"] == "sender@example.com" + + def test_auto_submitted_duplicate_first_wins_per_rfc_3834(self): + """RFC 3834 §5 makes Auto-Submitted max=1; on duplication we + take the first value — matching stdlib semantics. A sender + that emits ``Auto-Submitted: no`` first and + ``Auto-Submitted: auto-replied`` second is non-compliant; we + intentionally trust the first occurrence.""" + raw = ( + b"From: sender@example.com\r\n" + b"Subject: hi\r\n" + b"Auto-Submitted: no\r\n" + b"Auto-Submitted: auto-replied\r\n" + b"\r\nbody\r\n" + ) + parsed = parse_email_message(raw) + assert parsed["headers"]["auto-submitted"] == "no" + + def test_precedence_poison_first_still_detected_as_auto_reply(self): + """Defence-in-depth for loop detection. ``Precedence`` is + repeatable per RFC 5322 §3.6.8 (optional-field); the autoreply + check iterates every occurrence, so a non-bulk value cannot + mask a later ``bulk`` one and provoke a reply loop.""" + raw = ( + b"From: list@example.com\r\nSubject: newsletter\r\n" + b"Precedence: not-bulk\r\n" + b"Precedence: bulk\r\n" + b"\r\nbody\r\n" + ) + parsed = parse_email_message(raw) + from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel + _is_auto_reply_message, + ) + + assert _is_auto_reply_message(parsed["headers"]) is True + + def test_return_path_poison_first_still_detected_as_bounce(self): + """Same defence-in-depth as Precedence: a duplicate + ``Return-Path`` with a non-bounce value first must not mask a + bounce indicator (``<>`` / empty) later.""" + raw = ( + b"From: MAILER-DAEMON@example.com\r\nSubject: bounce\r\n" + b"Return-Path: \r\n" + b"Return-Path: <>\r\n" + b"\r\nbody\r\n" + ) + parsed = parse_email_message(raw) + from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel + _is_auto_reply_message, + ) + + assert _is_auto_reply_message(parsed["headers"]) is True + + def test_headers_dict_type_matches_call_site_expectation(self): + """Real-world inbound with multiple legitimately-repeated + ``DKIM-Signature`` and ``Received`` headers (RFC 6376 / + RFC 5322) must pass through the autoreply detection logic — + which does ``.strip().lower()`` on scalar lookups — without + raising.""" + raw = ( + b"Received: from hop2 by hop3\r\n" + b"Received: from hop1 by hop2\r\n" + b"DKIM-Signature: v=1; a=rsa-sha256; d=a.example.com\r\n" + b"DKIM-Signature: v=1; a=rsa-sha256; d=b.example.com\r\n" + b"From: sender@example.com\r\n" + b"To: recipient@example.com\r\n" + b"Subject: legitimate forwarded mail\r\n" + b"Precedence: bulk\r\n" + b"\r\n" + b"body\r\n" + ) + parsed = parse_email_message(raw) + # Defer the autoreply import to keep this test isolated from + # the autoreply module's import-time side effects. + from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel + _is_auto_reply_message, + ) + + # Must not raise — and the bulk Precedence must be detected. + assert _is_auto_reply_message(parsed["headers"]) is True + + if __name__ == "__main__": pytest.main() diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 88939066..063de269 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -49,7 +49,7 @@ dependencies = [ "drf_spectacular==0.29.0", "opensearch-py==2.8.0", "factory_boy==3.3.3", - "flanker@git+https://github.com/sylvinus/flanker@a4248826794d446b07fb26719479c1c355411f5a", + "flanker@git+https://github.com/sylvinus/flanker@77f4582044f1a8549d49333d50d9bded1176ca09", "gunicorn==25.1.0", "icalendar==7.0.3", "jsonschema==4.26.0", diff --git a/src/backend/uv.lock b/src/backend/uv.lock index 8bfd8ec0..7e17cba0 100644 --- a/src/backend/uv.lock +++ b/src/backend/uv.lock @@ -778,7 +778,7 @@ wheels = [ [[package]] name = "flanker" version = "0.9.11" -source = { git = "https://github.com/sylvinus/flanker?rev=a4248826794d446b07fb26719479c1c355411f5a#a4248826794d446b07fb26719479c1c355411f5a" } +source = { git = "https://github.com/sylvinus/flanker?rev=77f4582044f1a8549d49333d50d9bded1176ca09#77f4582044f1a8549d49333d50d9bded1176ca09" } dependencies = [ { name = "attrs" }, { name = "chardet" }, @@ -1215,7 +1215,7 @@ requires-dist = [ { name = "drf-spectacular", specifier = "==0.29.0" }, { name = "drf-spectacular-sidecar", marker = "extra == 'dev'", specifier = "==2026.1.1" }, { name = "factory-boy", specifier = "==3.3.3" }, - { name = "flanker", git = "https://github.com/sylvinus/flanker?rev=a4248826794d446b07fb26719479c1c355411f5a" }, + { name = "flanker", git = "https://github.com/sylvinus/flanker?rev=77f4582044f1a8549d49333d50d9bded1176ca09" }, { name = "flower", marker = "extra == 'dev'", specifier = "==2.0.1" }, { name = "gunicorn", specifier = "==25.1.0" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = "==6.151.9" },