mirror of
https://github.com/suitenumerique/messages.git
synced 2026-08-17 21:25:41 +02:00
🐛(backend) harden inbound email parsing (#695)
Two distinct AttributeError crashes were firing on legitimate inbound
mail and producing 5xx-equivalent failures (parse aborts, autoreply
skipped).
1. Flanker scanner crashed on `multipart/report` bounces whose status
part used an RFC 6533 i18n content type (`message/global-delivery-
status` and siblings). The same crash path was reachable for every
`message/*` subtype not in flanker's hardcoded list — `partial`,
`imdn+xml`, `sip`, `cpim`, future / vendor subtypes. Fixed in the
flanker fork (i18n predicates + traverse fallback + make_part guard);
pin bumped to 77f4582044f1a8549d49333d50d9bded1176ca09.
2. `parsed["headers"]` returned `str` for single occurrences and
`list[str]` when duplicated. Every scalar consumer (`.strip()`,
`.lower()`, `.startswith()` on Subject / Message-ID / Precedence /
Return-Path / etc.) crashed the moment a header was repeated. The
parser now applies a fixed per-header type contract driven by the
IANA Provisional Message Header Field Registry: headers registered
with max=1 (RFC 5322 §3.6, RFC 3834, RFC 2045/2046/2183, RFC 4021,
RFC 3798, RFC 5703, RFC 8058, RFC 8098) are `str` with first-wins
semantics matching stdlib `email.message.Message[name]`; every
other header is `list[str]` in document order. `headers_blocks`
stays uniformly list-typed for the trusted-relays cut.
This commit is contained in:
Sylvain Zimmer
@@ -82,30 +82,36 @@ def _is_auto_reply_message(headers: dict) -> bool:
|
||||
if not headers:
|
||||
return False
|
||||
|
||||
# Normalize header keys to lowercase for comparison
|
||||
# ``headers`` follows the rfc5322 parser contract:
|
||||
# - RFC max=1 headers (auto-submitted, …) are ``str``
|
||||
# - every other header is ``list[str]`` in document order
|
||||
# Lowercase defensively so the function tolerates a caller that
|
||||
# hasn't normalised yet (production callers go through the parser
|
||||
# which already lowercases).
|
||||
lower_headers = {k.lower(): v for k, v in headers.items()}
|
||||
|
||||
# Return-Path: empty or <> means bounce (RFC 3834)
|
||||
if "return-path" in lower_headers:
|
||||
return_path = lower_headers["return-path"].strip()
|
||||
if return_path in ("", "<>"):
|
||||
# Return-Path: empty or <> means bounce (RFC 3834). Return-Path is
|
||||
# not in _SCALAR_HEADERS so it's a list[str]; iterate every
|
||||
# occurrence so a benign duplicate can't mask a bounce indicator.
|
||||
for return_path in lower_headers.get("return-path", []):
|
||||
if return_path.strip() in ("", "<>"):
|
||||
return True
|
||||
|
||||
# Auto-Submitted: anything other than "no" means auto-generated.
|
||||
# RFC 3834 allows parameters after ";" (e.g. "auto-replied; owner-email=...").
|
||||
# Auto-Submitted: RFC 3834 §5 makes this max=1, so it's a str.
|
||||
# Parameters after ";" (e.g. "auto-replied; owner-email=...") are
|
||||
# stripped before comparison; anything other than "no" counts.
|
||||
auto_submitted = lower_headers.get("auto-submitted", "").strip().lower()
|
||||
if auto_submitted:
|
||||
# Strip parameters: "auto-replied; foo=bar" -> "auto-replied"
|
||||
auto_submitted_value = auto_submitted.split(";", 1)[0].strip()
|
||||
if auto_submitted_value and auto_submitted_value != "no":
|
||||
if auto_submitted.split(";", 1)[0].strip() not in ("", "no"):
|
||||
return True
|
||||
|
||||
# Precedence: bulk, list, junk
|
||||
precedence = lower_headers.get("precedence", "").strip().lower()
|
||||
if precedence in _PRECEDENCE_VALUES:
|
||||
return True
|
||||
# Precedence: bulk, list, junk. Repeatable per RFC 5322 (optional-field).
|
||||
for precedence in lower_headers.get("precedence", []):
|
||||
if precedence.strip().lower() in _PRECEDENCE_VALUES:
|
||||
return True
|
||||
|
||||
# Presence of any loop header
|
||||
# Presence of any loop header is enough (list-id, list-unsubscribe,
|
||||
# x-loop, …). All of these are list-typed; truthy ⇔ non-empty.
|
||||
for header_name in _LOOP_HEADERS:
|
||||
if lower_headers.get(header_name):
|
||||
return True
|
||||
|
||||
@@ -73,9 +73,12 @@ def _check_spam_with_hardcoded_rules(
|
||||
key = key.lower().strip()
|
||||
value = value.lower().strip()
|
||||
|
||||
# Get header value(s) - can be a string or list
|
||||
header_value = headers.get(key)
|
||||
if header_value is None:
|
||||
# Existence check only — the actual value is read from
|
||||
# headers_blocks below to apply the trusted-relays cut.
|
||||
# ``headers`` follows the rfc5322 parser contract:
|
||||
# str for RFC max=1 headers, list[str] otherwise; either
|
||||
# shape is truthy when present, which is all we need here.
|
||||
if headers.get(key) is None:
|
||||
continue
|
||||
|
||||
# Use headers_blocks to identify which headers to trust based on trusted_relays config.
|
||||
|
||||
@@ -43,6 +43,58 @@ class EmailParseError(Exception):
|
||||
"""Exception raised for errors during email parsing."""
|
||||
|
||||
|
||||
# Header names (lowercase) that the relevant RFCs specify as appearing
|
||||
# at most once per message. ``parsed["headers"][name]`` is ``str`` for
|
||||
# these and ``list[str]`` (in document order) for every other header.
|
||||
# When duplicated in the wild, the first occurrence wins — matching
|
||||
# Python stdlib ``email.message.Message[name]`` semantics.
|
||||
#
|
||||
# Inclusion criterion: registered with max=1 in the IANA Provisional
|
||||
# Message Header Field Registry (RFC 3864 / RFC 9057). The set is
|
||||
# pre-emptive — many of these headers have no consumer in the code
|
||||
# today, but pinning their type now means a future caller can write
|
||||
# ``parsed["headers"]["x"].strip()`` without an isinstance check.
|
||||
_SCALAR_HEADERS: frozenset[str] = frozenset(
|
||||
{
|
||||
# RFC 5322 §3.6 (origination, destination, identification,
|
||||
# informational fields — all max=1):
|
||||
"date",
|
||||
"from",
|
||||
"sender",
|
||||
"reply-to",
|
||||
"to",
|
||||
"cc",
|
||||
"bcc",
|
||||
"message-id",
|
||||
"in-reply-to",
|
||||
"references",
|
||||
"subject",
|
||||
# RFC 3834 §5 — Auto-Submitted MUST NOT appear more than once:
|
||||
"auto-submitted",
|
||||
# RFC 2045 / 2046 / 2183 single-valued MIME container fields:
|
||||
"mime-version",
|
||||
"content-type",
|
||||
"content-id",
|
||||
"content-transfer-encoding",
|
||||
"content-description",
|
||||
"content-disposition",
|
||||
# RFC 4021 §2.1.55 / .56 / .57 — message-class indicators:
|
||||
"importance",
|
||||
"priority",
|
||||
"sensitivity",
|
||||
# RFC 8098 §2.1 — MDN request (MUST NOT appear more than once):
|
||||
"disposition-notification-to",
|
||||
# RFC 3798 — MDN reporting:
|
||||
"original-message-id",
|
||||
# RFC 5703 §3.4 — Sieve archive original-envelope:
|
||||
"original-from",
|
||||
"original-subject",
|
||||
# RFC 8058 §3.1 — one-click unsubscribe POST body:
|
||||
"list-unsubscribe-post",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def decode_email_header_text(header_text: str) -> str:
|
||||
"""
|
||||
Decode email header text that might be encoded (RFC 2047).
|
||||
@@ -724,17 +776,36 @@ def _parse_labels_header(labels_str: str) -> list:
|
||||
|
||||
|
||||
def parse_email_message(raw_email_bytes: bytes) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Parse a raw email message (bytes) into a structured dictionary following JMAP format.
|
||||
"""Parse a raw email message (bytes) into a structured dict (JMAP-ish).
|
||||
|
||||
The returned dict carries three header views, each with a fixed
|
||||
type contract so callers don't need ``isinstance`` checks:
|
||||
|
||||
- ``headers`` — ``dict[str, str | list[str]]`` keyed by lowercase
|
||||
name. RFC max=1 headers (see ``_SCALAR_HEADERS``) are ``str``;
|
||||
first occurrence wins on duplication, matching stdlib
|
||||
``email.message.Message[name]``. Every other header is
|
||||
``list[str]`` in document order — ``received``, ``return-path``,
|
||||
``dkim-signature``, ``authentication-results``, ``arc-*``,
|
||||
``list-*``, ``comments``, ``keywords``, every ``X-*``.
|
||||
- ``headers_list`` — ``list[tuple[str, str]]`` of every header in
|
||||
document order, with lowercase names. Source of truth for full
|
||||
occurrence + ordering.
|
||||
- ``headers_blocks`` — ``list[dict[str, list[str]]]``. Each block
|
||||
ends with a ``Received`` header; everything above (earlier) it
|
||||
is in the same trust scope. Values inside a block are *always*
|
||||
``list[str]`` regardless of the header's spec, so trusted-relays
|
||||
filters can index uniformly.
|
||||
|
||||
Args:
|
||||
raw_email_bytes: Raw email data as bytes
|
||||
raw_email_bytes: Raw email data as bytes.
|
||||
|
||||
Returns:
|
||||
Dictionary containing parsed email data, or None if parsing fails fundamentally.
|
||||
Dict of parsed fields, or raises ``EmailParseError`` on
|
||||
fundamental parse failure.
|
||||
|
||||
Raises:
|
||||
EmailParseError: If parsing fails with a specific error we want to propagate.
|
||||
EmailParseError: parsing failed.
|
||||
"""
|
||||
if not raw_email_bytes or not isinstance(raw_email_bytes, bytes):
|
||||
# Ensure input is non-empty bytes
|
||||
@@ -768,15 +839,12 @@ def parse_email_message(raw_email_bytes: bytes) -> Optional[Dict[str, Any]]:
|
||||
key_lower = k.lower()
|
||||
headers_list.append((key_lower, decoded_value))
|
||||
|
||||
# Build headers dict (for compatibility)
|
||||
if key_lower in headers:
|
||||
current_value = headers[key_lower]
|
||||
if isinstance(current_value, list):
|
||||
current_value.append(decoded_value)
|
||||
else:
|
||||
headers[key_lower] = [current_value, decoded_value]
|
||||
if key_lower in _SCALAR_HEADERS:
|
||||
# First occurrence wins (RFC max=1; matches stdlib).
|
||||
headers.setdefault(key_lower, decoded_value)
|
||||
else:
|
||||
headers[key_lower] = decoded_value
|
||||
# Repeatable: always list[str] in document order.
|
||||
headers.setdefault(key_lower, []).append(decoded_value)
|
||||
|
||||
# Split headers into blocks based on Received headers
|
||||
# Each Received header marks the END of its block - everything above it (before it in the list) is trusted
|
||||
@@ -804,29 +872,24 @@ def parse_email_message(raw_email_bytes: bytes) -> Optional[Dict[str, Any]]:
|
||||
gmail_labels = []
|
||||
seen_labels = set()
|
||||
|
||||
# Parse X-Gmail-Labels (Google Takeout format)
|
||||
# Parse X-Gmail-Labels (Google Takeout format). The header is
|
||||
# not in _SCALAR_HEADERS so it's a list[str]; only the first
|
||||
# occurrence is meaningful for label semantics.
|
||||
if "x-gmail-labels" in headers:
|
||||
labels_str = headers["x-gmail-labels"]
|
||||
if isinstance(labels_str, list):
|
||||
labels_str = labels_str[0] # Take first value if multiple
|
||||
for label in _parse_labels_header(labels_str):
|
||||
for label in _parse_labels_header(headers["x-gmail-labels"][0]):
|
||||
if label not in seen_labels:
|
||||
seen_labels.add(label)
|
||||
gmail_labels.append(label)
|
||||
|
||||
# Parse X-Keywords (Dovecot/OfflineIMAP/mu4e format)
|
||||
# Parse X-Keywords (Dovecot/OfflineIMAP/mu4e format).
|
||||
if "x-keywords" in headers:
|
||||
labels_str = headers["x-keywords"]
|
||||
if isinstance(labels_str, list):
|
||||
labels_str = labels_str[0] # Take first value if multiple
|
||||
for label in _parse_labels_header(labels_str):
|
||||
for label in _parse_labels_header(headers["x-keywords"][0]):
|
||||
if label not in seen_labels:
|
||||
seen_labels.add(label)
|
||||
gmail_labels.append(label)
|
||||
|
||||
subject = headers.get("subject", "")
|
||||
from_header_decoded = headers.get("from", "")
|
||||
from_name, from_addr = parse_email_address(from_header_decoded)
|
||||
from_name, from_addr = parse_email_address(headers.get("from", ""))
|
||||
to_recipients = parse_email_addresses(headers.get("to", ""))
|
||||
cc_recipients = parse_email_addresses(headers.get("cc", ""))
|
||||
bcc_recipients = parse_email_addresses(headers.get("bcc", ""))
|
||||
|
||||
@@ -2007,16 +2007,25 @@ class Message(BaseModel):
|
||||
"""Get a parsed field from the parsed email data."""
|
||||
return (self.get_parsed_data() or {}).get(field_name)
|
||||
|
||||
def get_mime_headers(self) -> Dict[str, str]:
|
||||
"""Get the MIME headers of the message."""
|
||||
def get_mime_headers(self) -> Dict[str, Any]:
|
||||
"""Get the MIME headers of the message.
|
||||
|
||||
Values follow the rfc5322 parser contract: ``str`` for RFC max=1
|
||||
headers, ``list[str]`` in document order for every other header.
|
||||
"""
|
||||
return self.get_parsed_data().get("headers", {})
|
||||
|
||||
def get_stmsg_headers(self) -> Dict[str, str]:
|
||||
"""Get the STMSG headers of the message."""
|
||||
"""Get the STMSG headers of the message.
|
||||
|
||||
``X-StMsg-*`` headers are stamped by our own MTA pipeline (one
|
||||
per message) and any sender-supplied copies are stripped before
|
||||
parsing. They surface as single-element lists; take the first.
|
||||
"""
|
||||
return {
|
||||
k[len("x-stmsg-") :].lower(): v
|
||||
k[len("x-stmsg-") :].lower(): v[0]
|
||||
for k, v in self.get_parsed_data().get("headers", {}).items()
|
||||
if k.startswith("x-stmsg-")
|
||||
if k.startswith("x-stmsg-") and v
|
||||
}
|
||||
|
||||
def generate_mime_id(self) -> str:
|
||||
|
||||
@@ -111,42 +111,42 @@ class TestIsAutoReplyMessage:
|
||||
|
||||
def test_precedence_bulk(self):
|
||||
"""Precedence: bulk is detected."""
|
||||
headers = {"Precedence": "bulk"}
|
||||
headers = {"Precedence": ["bulk"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_precedence_list(self):
|
||||
"""Precedence: list is detected."""
|
||||
headers = {"Precedence": "list"}
|
||||
headers = {"Precedence": ["list"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_precedence_junk(self):
|
||||
"""Precedence: junk is detected."""
|
||||
headers = {"Precedence": "junk"}
|
||||
headers = {"Precedence": ["junk"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_list_id_header(self):
|
||||
"""List-Id header is detected."""
|
||||
headers = {"List-Id": "<list.example.com>"}
|
||||
headers = {"List-Id": ["<list.example.com>"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_list_unsubscribe_header(self):
|
||||
"""List-Unsubscribe header is detected."""
|
||||
headers = {"List-Unsubscribe": "<mailto:unsub@example.com>"}
|
||||
headers = {"List-Unsubscribe": ["<mailto:unsub@example.com>"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_x_auto_response_suppress(self):
|
||||
"""X-Auto-Response-Suppress header is detected."""
|
||||
headers = {"X-Auto-Response-Suppress": "All"}
|
||||
headers = {"X-Auto-Response-Suppress": ["All"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_x_autoreply(self):
|
||||
"""X-Autoreply header is detected."""
|
||||
headers = {"X-Autoreply": "yes"}
|
||||
headers = {"X-Autoreply": ["yes"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_x_autorespond(self):
|
||||
"""X-Autorespond header is detected."""
|
||||
headers = {"X-Autorespond": "yes"}
|
||||
headers = {"X-Autorespond": ["yes"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_auto_submitted_with_parameters(self):
|
||||
@@ -165,47 +165,47 @@ class TestIsAutoReplyMessageExtended:
|
||||
|
||||
def test_return_path_null(self):
|
||||
"""Return-Path: <> (null sender) is detected."""
|
||||
headers = {"Return-Path": "<>"}
|
||||
headers = {"Return-Path": ["<>"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_return_path_empty(self):
|
||||
"""Return-Path with empty value is detected."""
|
||||
headers = {"Return-Path": ""}
|
||||
headers = {"Return-Path": [""]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_list_post_header(self):
|
||||
"""List-Post header is detected."""
|
||||
headers = {"List-Post": "<mailto:list@example.com>"}
|
||||
headers = {"List-Post": ["<mailto:list@example.com>"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_list_help_header(self):
|
||||
"""List-Help header is detected."""
|
||||
headers = {"List-Help": "<mailto:help@example.com>"}
|
||||
headers = {"List-Help": ["<mailto:help@example.com>"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_list_subscribe_header(self):
|
||||
"""List-Subscribe header is detected."""
|
||||
headers = {"List-Subscribe": "<mailto:sub@example.com>"}
|
||||
headers = {"List-Subscribe": ["<mailto:sub@example.com>"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_list_owner_header(self):
|
||||
"""List-Owner header is detected."""
|
||||
headers = {"List-Owner": "<mailto:owner@example.com>"}
|
||||
headers = {"List-Owner": ["<mailto:owner@example.com>"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_list_archive_header(self):
|
||||
"""List-Archive header is detected."""
|
||||
headers = {"List-Archive": "<https://archive.example.com>"}
|
||||
headers = {"List-Archive": ["<https://archive.example.com>"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_x_loop_header(self):
|
||||
"""X-Loop header is detected."""
|
||||
headers = {"X-Loop": "yes"}
|
||||
headers = {"X-Loop": ["yes"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
def test_feedback_id_header(self):
|
||||
"""Feedback-ID header is detected (Gmail newsletters)."""
|
||||
headers = {"Feedback-ID": "123:campaign:gmail"}
|
||||
headers = {"Feedback-ID": ["123:campaign:gmail"]}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
|
||||
|
||||
|
||||
@@ -519,7 +519,7 @@ class TestProcessInboundMessageAuthIntegration:
|
||||
call_kwargs = mock_create_message.call_args[1]
|
||||
assert call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth: none\r\n")
|
||||
parsed = call_kwargs["parsed_email"]
|
||||
assert parsed["headers"].get("x-stmsg-sender-auth") == "none"
|
||||
assert parsed["headers"].get("x-stmsg-sender-auth") == ["none"]
|
||||
|
||||
@override_settings(SPAM_CONFIG={"inbound_auth": "rspamd"})
|
||||
@patch("core.mda.inbound_tasks.check_inbound_authentication")
|
||||
@@ -541,7 +541,7 @@ class TestProcessInboundMessageAuthIntegration:
|
||||
call_kwargs = mock_create_message.call_args[1]
|
||||
assert call_kwargs["raw_data"].startswith(b"X-StMsg-Sender-Auth: fail\r\n")
|
||||
parsed = call_kwargs["parsed_email"]
|
||||
assert parsed["headers"].get("x-stmsg-sender-auth") == "fail"
|
||||
assert parsed["headers"].get("x-stmsg-sender-auth") == ["fail"]
|
||||
|
||||
@override_settings(SPAM_CONFIG={"inbound_auth": "native"})
|
||||
@patch("core.mda.inbound_tasks.check_inbound_authentication")
|
||||
@@ -683,7 +683,7 @@ class TestProcessInboundMessageAuthIntegration:
|
||||
"""After prepending, the parser exposes the header via x-stmsg-*."""
|
||||
tagged = b"X-StMsg-Sender-Auth: fail\r\n" + RAW_EMAIL
|
||||
parsed = parse_email_message(tagged)
|
||||
assert parsed["headers"].get("x-stmsg-sender-auth") == "fail"
|
||||
assert parsed["headers"].get("x-stmsg-sender-auth") == ["fail"]
|
||||
|
||||
@override_settings(SPAM_CONFIG={"inbound_auth": "native"})
|
||||
@patch("core.mda.inbound_tasks.parse_email_message")
|
||||
|
||||
@@ -851,10 +851,11 @@ Text part.
|
||||
parsed = parse_email_message(message.to_string().encode("utf-8"))
|
||||
assert parsed is not None
|
||||
assert parsed["subject"] == "Custom Headers"
|
||||
assert "x-custom-header" in parsed["headers"]
|
||||
assert parsed["headers"]["x-custom-header"] == "Custom Value"
|
||||
assert parsed["headers"]["x-priority"] == "1"
|
||||
assert parsed["headers"]["x-mailer"] == "Custom Mailer v1.0"
|
||||
# Non-scalar headers (X-*, optional-field per RFC 5322 §3.6.8)
|
||||
# are stored as list[str] in document order.
|
||||
assert parsed["headers"]["x-custom-header"] == ["Custom Value"]
|
||||
assert parsed["headers"]["x-priority"] == ["1"]
|
||||
assert parsed["headers"]["x-mailer"] == ["Custom Mailer v1.0"]
|
||||
|
||||
# Check headers_list contains custom headers in order
|
||||
assert "headers_list" in parsed
|
||||
@@ -2169,5 +2170,745 @@ Content-Type: text/html; charset="utf-8"
|
||||
assert "htmlBody" in parsed
|
||||
|
||||
|
||||
class TestUnrecognisedMessageSubtypeRobustness:
|
||||
"""Coverage for inbound bounces and any ``message/*`` content type
|
||||
the underlying MIME engine has no dedicated branch for.
|
||||
|
||||
RFC 5337 / RFC 6533 i18n DSN variants
|
||||
(``message/global-delivery-status``,
|
||||
``message/global-disposition-notification``,
|
||||
``message/global-headers``), plus other standardised ``message/*``
|
||||
subtypes (``message/partial`` RFC 2046, ``message/imdn+xml`` RFC 5438,
|
||||
``message/sip`` RFC 3261, ``message/cpim`` RFC 3862, …) and any
|
||||
vendor / unknown subtype must all parse successfully when nested
|
||||
inside a ``multipart/report`` bounce.
|
||||
|
||||
Tests exercise the public ``parse_email_message`` API only — no
|
||||
MIME-engine internals — so they remain valid if the parser backend
|
||||
is swapped (e.g. flanker → ``email.parser``). The behavioural
|
||||
contract:
|
||||
|
||||
1. Parsing must not raise ``EmailParseError`` on any well-formed
|
||||
multipart/report whose status part uses an unrecognised
|
||||
``message/*`` subtype.
|
||||
2. The human-readable notification text part must survive parsing
|
||||
so the bounce remains useful to the recipient.
|
||||
3. Legacy bounce part types must continue to parse correctly.
|
||||
4. Top-level unrecognised ``message/*`` content types are accepted
|
||||
too, not only the multipart-nested case.
|
||||
"""
|
||||
|
||||
BOUNCE_BOUNDARY = "boundary-dsn"
|
||||
NOTIFICATION_TEXT = "Your message could not be delivered to one or more recipients."
|
||||
|
||||
@classmethod
|
||||
def _build_dsn(cls, status_content_type: str, status_body: bytes) -> bytes:
|
||||
"""Build a realistic multipart/report bounce wrapping ``status_body``."""
|
||||
boundary = cls.BOUNCE_BOUNDARY.encode()
|
||||
return (
|
||||
b"From: MAILER-DAEMON@mta.example.com (Mail Delivery System)\r\n"
|
||||
b"To: sender@example.org\r\n"
|
||||
b"Subject: Undelivered Mail Returned to Sender\r\n"
|
||||
b"Date: Thu, 4 Jun 2026 02:30:20 +0200 (CEST)\r\n"
|
||||
b"MIME-Version: 1.0\r\n"
|
||||
b"Content-Type: multipart/report; report-type=delivery-status;\r\n"
|
||||
b'\tboundary="' + boundary + b'"\r\n'
|
||||
b"\r\n"
|
||||
b"This is a MIME-encapsulated message.\r\n"
|
||||
b"\r\n"
|
||||
b"--" + boundary + b"\r\n"
|
||||
b"Content-Description: Notification\r\n"
|
||||
b"Content-Type: text/plain; charset=utf-8\r\n"
|
||||
b"\r\n" + cls.NOTIFICATION_TEXT.encode() + b"\r\n"
|
||||
b"\r\n"
|
||||
b"--" + boundary + b"\r\n"
|
||||
b"Content-Description: Delivery report\r\n"
|
||||
b"Content-Type: " + status_content_type.encode() + b"\r\n"
|
||||
b"\r\n" + status_body + b"\r\n"
|
||||
b"\r\n"
|
||||
b"--" + boundary + b"--\r\n"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content_type",
|
||||
[
|
||||
# RFC 5337 / 6533 i18n equivalents of the legacy bounce / MDN /
|
||||
# rfc822-headers parts. These need predicate-level recognition
|
||||
# to be classified correctly (not just treated as opaque).
|
||||
"message/global-delivery-status",
|
||||
"message/global-disposition-notification",
|
||||
"message/global-headers",
|
||||
# Other standardised ``message/*`` subtypes — any unknown
|
||||
# subtype must be tolerated, not only the i18n ones.
|
||||
"message/partial", # RFC 2046 §5.2.2 fragmented messages
|
||||
"message/imdn+xml", # RFC 5438 IMDN
|
||||
"message/sip", # RFC 3261 SIP signalling
|
||||
"message/sipfrag", # RFC 3420 partial SIP messages
|
||||
"message/cpim", # RFC 3862 Common Presence and IM
|
||||
# Vendor / unknown subtypes must also be tolerated.
|
||||
"message/x-vendor-future",
|
||||
"message/x-totally-unknown",
|
||||
],
|
||||
)
|
||||
def test_dsn_with_unrecognised_message_subtype(self, content_type):
|
||||
"""A multipart/report bounce whose status part uses any unrecognised
|
||||
``message/*`` subtype must parse without raising, and the
|
||||
human-readable notification text part must be preserved."""
|
||||
raw = self._build_dsn(
|
||||
content_type,
|
||||
b"Reporting-MTA: dns; mta.example.com\r\n"
|
||||
b"\r\n"
|
||||
b"Final-Recipient: rfc822; recipient@example.com\r\n"
|
||||
b"Action: failed\r\n"
|
||||
b"Status: 5.0.0",
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed is not None
|
||||
assert parsed["subject"] == "Undelivered Mail Returned to Sender"
|
||||
assert parsed["from"]["email"] == "MAILER-DAEMON@mta.example.com"
|
||||
assert any(
|
||||
self.NOTIFICATION_TEXT in part["content"] for part in parsed["textBody"]
|
||||
), (
|
||||
f"notification text missing from textBody for {content_type}; "
|
||||
f"got parts: {[p.get('content', '')[:60] for p in parsed['textBody']]}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content_type",
|
||||
[
|
||||
"message/delivery-status",
|
||||
"message/disposition-notification",
|
||||
"text/rfc822-headers",
|
||||
],
|
||||
)
|
||||
def test_dsn_with_legacy_recognised_subtype(self, content_type):
|
||||
"""Legacy bounce part types must keep parsing correctly."""
|
||||
raw = self._build_dsn(
|
||||
content_type,
|
||||
b"Reporting-MTA: dns; mta.example.com\r\n"
|
||||
b"\r\n"
|
||||
b"Final-Recipient: rfc822; recipient@example.com\r\n"
|
||||
b"Action: failed\r\n"
|
||||
b"Status: 5.0.0",
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed is not None
|
||||
assert parsed["subject"] == "Undelivered Mail Returned to Sender"
|
||||
assert any(
|
||||
self.NOTIFICATION_TEXT in part["content"] for part in parsed["textBody"]
|
||||
)
|
||||
|
||||
def test_postfix_i18n_dsn_full_shape(self):
|
||||
"""End-to-end shape: a Postfix bounce wrapping RFC 6533
|
||||
``message/global-delivery-status`` inside ``multipart/report``,
|
||||
with the full set of headers Postfix typically emits."""
|
||||
raw = self._build_dsn(
|
||||
"message/global-delivery-status",
|
||||
b"Reporting-MTA: dns; mta.example.com\r\n"
|
||||
b"X-Postfix-Queue-ID: ABC123\r\n"
|
||||
b"X-Postfix-Sender: rfc822; sender@example.org\r\n"
|
||||
b"\r\n"
|
||||
b"Final-Recipient: rfc822; user@example.com\r\n"
|
||||
b"Original-Recipient: rfc822;user@example.com\r\n"
|
||||
b"Action: failed\r\n"
|
||||
b"Status: 5.0.0",
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed["subject"] == "Undelivered Mail Returned to Sender"
|
||||
# Sanity-check the headers that downstream bounce handlers read.
|
||||
assert parsed["headers"].get("from", "").startswith("MAILER-DAEMON")
|
||||
assert "multipart/report" in parsed["headers"]["content-type"]
|
||||
|
||||
def test_top_level_unrecognised_message_subtype(self):
|
||||
"""An email whose ROOT Content-Type is an unrecognised
|
||||
``message/*`` subtype must parse — the fallback applies at the
|
||||
root, not only nested inside a multipart."""
|
||||
raw = (
|
||||
b"From: sender@example.com\r\n"
|
||||
b"To: recipient@example.com\r\n"
|
||||
b"Subject: Weird top-level subtype\r\n"
|
||||
b"MIME-Version: 1.0\r\n"
|
||||
b"Content-Type: message/x-something-vendor\r\n"
|
||||
b"\r\n"
|
||||
b"opaque payload bytes\r\n"
|
||||
)
|
||||
# The contract is "do not raise". Whether the body bytes are
|
||||
# surfaced verbatim in textBody is implementation-defined and
|
||||
# not asserted here, so the test survives a backend swap.
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed is not None
|
||||
assert parsed["subject"] == "Weird top-level subtype"
|
||||
assert parsed["from"]["email"] == "sender@example.com"
|
||||
|
||||
def test_unrecognised_subtype_nested_in_message_container(self):
|
||||
"""A ``message/rfc822``-wrapped inner message with its own
|
||||
unrecognised ``message/*`` content type must parse — the outer
|
||||
container should not be affected by an unknown inner subtype."""
|
||||
raw = (
|
||||
b"From: x@example.com\r\n"
|
||||
b"To: y@example.com\r\n"
|
||||
b"Subject: Forwarded weird\r\n"
|
||||
b"MIME-Version: 1.0\r\n"
|
||||
b'Content-Type: multipart/mixed; boundary="OUTER"\r\n'
|
||||
b"\r\n"
|
||||
b"--OUTER\r\n"
|
||||
b"Content-Type: text/plain\r\n"
|
||||
b"\r\n"
|
||||
b"See the attached forwarded message.\r\n"
|
||||
b"\r\n"
|
||||
b"--OUTER\r\n"
|
||||
b"Content-Type: message/rfc822\r\n"
|
||||
b"\r\n"
|
||||
b"From: inner@example.com\r\n"
|
||||
b"Subject: Inner\r\n"
|
||||
b"Content-Type: message/x-unknown-vendor\r\n"
|
||||
b"\r\n"
|
||||
b"opaque body\r\n"
|
||||
b"--OUTER--\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed is not None
|
||||
assert parsed["subject"] == "Forwarded weird"
|
||||
assert any("See the attached" in part["content"] for part in parsed["textBody"])
|
||||
|
||||
def test_multiple_unrecognised_subtypes_in_same_report(self):
|
||||
"""A bounce can contain multiple status sub-parts with mixed
|
||||
legacy and i18n content types. None of them should crash and
|
||||
the notification text must survive."""
|
||||
raw = (
|
||||
b"From: MAILER-DAEMON@mta.example.com\r\n"
|
||||
b"To: sender@example.org\r\n"
|
||||
b"Subject: Mixed bounce\r\n"
|
||||
b"MIME-Version: 1.0\r\n"
|
||||
b"Content-Type: multipart/report; report-type=delivery-status;\r\n"
|
||||
b'\tboundary="MULTI"\r\n'
|
||||
b"\r\n"
|
||||
b"--MULTI\r\n"
|
||||
b"Content-Type: text/plain; charset=utf-8\r\n"
|
||||
b"\r\n" + self.NOTIFICATION_TEXT.encode() + b"\r\n"
|
||||
b"--MULTI\r\n"
|
||||
b"Content-Type: message/delivery-status\r\n"
|
||||
b"\r\n"
|
||||
b"Reporting-MTA: dns; legacy.example.com\r\n"
|
||||
b"--MULTI\r\n"
|
||||
b"Content-Type: message/global-delivery-status\r\n"
|
||||
b"\r\n"
|
||||
b"Reporting-MTA: dns; i18n.example.com\r\n"
|
||||
b"--MULTI\r\n"
|
||||
b"Content-Type: message/x-something-unknown\r\n"
|
||||
b"\r\n"
|
||||
b"opaque\r\n"
|
||||
b"--MULTI--\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed is not None
|
||||
assert parsed["subject"] == "Mixed bounce"
|
||||
assert any(
|
||||
self.NOTIFICATION_TEXT in part["content"] for part in parsed["textBody"]
|
||||
)
|
||||
|
||||
|
||||
class TestScalarHeaderDuplicates:
|
||||
"""RFC 5322 §3.6 makes every scalar header — Date, From, Sender,
|
||||
Reply-To, To, Cc, Bcc, Message-ID, In-Reply-To, References, Subject —
|
||||
appear at most once. Real-world senders sometimes emit duplicates
|
||||
anyway, and the parser must tolerate that.
|
||||
|
||||
Behaviour matches the Python stdlib's
|
||||
``email.message.Message.__getitem__``: when a header is repeated,
|
||||
the parser silently uses the first occurrence. Tests pin that
|
||||
contract through the public ``parse_email_message`` API so they
|
||||
survive a swap from flanker to ``email.parser``.
|
||||
"""
|
||||
|
||||
BASE_HEADERS = {
|
||||
"From": "sender@example.com",
|
||||
"To": "recipient@example.com",
|
||||
"Subject": "hello",
|
||||
"Date": "Thu, 4 Jun 2026 00:47:09 +0000",
|
||||
"Message-ID": "<canonical@example.com>",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _build(cls, **overrides: object) -> bytes:
|
||||
"""Build a minimal email. ``overrides`` may map a header name to
|
||||
a single str (default behaviour) or a list[str] (the header is
|
||||
emitted multiple times, in that order)."""
|
||||
headers = {**cls.BASE_HEADERS, **overrides}
|
||||
lines: list[str] = []
|
||||
for name, value in headers.items():
|
||||
if isinstance(value, list):
|
||||
for v in value:
|
||||
lines.append(f"{name}: {v}")
|
||||
else:
|
||||
lines.append(f"{name}: {value}")
|
||||
lines += ["", "body"]
|
||||
return ("\r\n".join(lines)).encode("utf-8")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"header_name,values,result_key,expected_first",
|
||||
[
|
||||
# Identification: Message-ID and In-Reply-To strip <> on the
|
||||
# first value; the second one must not leak into the result.
|
||||
(
|
||||
"Message-ID",
|
||||
["<first@example.com>", "<second@example.com>"],
|
||||
"message_id",
|
||||
"first@example.com",
|
||||
),
|
||||
(
|
||||
"In-Reply-To",
|
||||
["<parent-first@example.com>", "<parent-second@example.com>"],
|
||||
"in_reply_to",
|
||||
"parent-first@example.com",
|
||||
),
|
||||
(
|
||||
"References",
|
||||
["<r1@example.com>", "<r2@example.com>"],
|
||||
"references",
|
||||
"<r1@example.com>",
|
||||
),
|
||||
(
|
||||
"Subject",
|
||||
["first subject", "second subject"],
|
||||
"subject",
|
||||
"first subject",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_duplicate_scalar_header_takes_first(
|
||||
self, header_name, values, result_key, expected_first
|
||||
):
|
||||
"""A duplicated scalar header must surface only its first
|
||||
occurrence in the structured result."""
|
||||
raw = self._build(**{header_name: values})
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed is not None
|
||||
assert parsed[result_key] == expected_first
|
||||
|
||||
def test_duplicate_from_takes_first(self):
|
||||
"""``From`` is parsed into ``{name, email}`` — first occurrence wins."""
|
||||
raw = self._build(**{"From": ["first@example.com", "second@example.com"]})
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed["from"]["email"] == "first@example.com"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"header_name,result_key",
|
||||
[("To", "to"), ("Cc", "cc"), ("Bcc", "bcc")],
|
||||
)
|
||||
def test_duplicate_address_list_header_takes_first(self, header_name, result_key):
|
||||
"""Only the first occurrence's addresses should appear in the
|
||||
structured result for duplicated address-list headers."""
|
||||
raw = self._build(**{header_name: ["first@example.com", "second@example.com"]})
|
||||
parsed = parse_email_message(raw)
|
||||
assert len(parsed[result_key]) == 1
|
||||
assert parsed[result_key][0]["email"] == "first@example.com"
|
||||
|
||||
def test_duplicate_date_takes_first(self):
|
||||
"""The first ``Date`` header wins."""
|
||||
first = "Thu, 4 Jun 2026 00:47:09 +0000"
|
||||
second = "Fri, 5 Jun 2026 00:00:00 +0000"
|
||||
raw = self._build(Date=[first, second])
|
||||
parsed = parse_email_message(raw)
|
||||
# Assert only the day-of-month — uniquely identifies the choice
|
||||
# without depending on tz formatting which can differ across
|
||||
# parser backends.
|
||||
assert parsed["date"] is not None
|
||||
assert parsed["date"].day == 4
|
||||
|
||||
def test_dual_message_id_returns_first(self):
|
||||
"""A message with two ``Message-ID`` headers must yield a single
|
||||
deterministic Message-ID downstream so the ``mime_id``-based
|
||||
duplicate-message check in inbound delivery has something
|
||||
stable to compare against."""
|
||||
raw = self._build(
|
||||
**{
|
||||
"Message-ID": [
|
||||
"<0S7NGNc8g9oEF8bStCvPthDYCCU0T9dnM20qLmmECY@example.com>",
|
||||
"<cdd440bf7fd91e2e23ac53136b0b860d@example.com>",
|
||||
]
|
||||
}
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
assert isinstance(parsed["message_id"], str)
|
||||
assert (
|
||||
parsed["message_id"]
|
||||
== "0S7NGNc8g9oEF8bStCvPthDYCCU0T9dnM20qLmmECY@example.com"
|
||||
)
|
||||
|
||||
def test_no_duplication_still_works(self):
|
||||
"""Regression guard: emails without any duplicated header must
|
||||
keep producing the same scalar fields after the helper change."""
|
||||
raw = self._build()
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed["subject"] == "hello"
|
||||
assert parsed["from"]["email"] == "sender@example.com"
|
||||
assert parsed["to"][0]["email"] == "recipient@example.com"
|
||||
assert parsed["message_id"] == "canonical@example.com"
|
||||
|
||||
def test_every_scalar_header_duplicated_simultaneously(self):
|
||||
"""Stress test: every scalar header per RFC 5322 §3.6 emitted
|
||||
twice in the same email must not crash the parser."""
|
||||
raw = self._build(
|
||||
**{
|
||||
"From": ["first-from@example.com", "second-from@example.com"],
|
||||
"To": ["first-to@example.com", "second-to@example.com"],
|
||||
"Cc": ["first-cc@example.com", "second-cc@example.com"],
|
||||
"Bcc": ["first-bcc@example.com", "second-bcc@example.com"],
|
||||
"Subject": ["first subj", "second subj"],
|
||||
"Date": [
|
||||
"Thu, 4 Jun 2026 00:47:09 +0000",
|
||||
"Fri, 5 Jun 2026 00:00:00 +0000",
|
||||
],
|
||||
"Message-ID": ["<id-a@example.com>", "<id-b@example.com>"],
|
||||
"In-Reply-To": ["<irt-a@example.com>", "<irt-b@example.com>"],
|
||||
"References": ["<ref-a@example.com>", "<ref-b@example.com>"],
|
||||
"Reply-To": ["first-rt@example.com", "second-rt@example.com"],
|
||||
"Sender": ["first-sender@example.com", "second-sender@example.com"],
|
||||
}
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
# Every scalar field must be a plain str (or scalar-shaped
|
||||
# dict for ``from``) — never a list.
|
||||
assert isinstance(parsed["subject"], str)
|
||||
assert isinstance(parsed["from"]["email"], str)
|
||||
assert isinstance(parsed["message_id"], str)
|
||||
assert isinstance(parsed["in_reply_to"], str)
|
||||
assert isinstance(parsed["references"], str)
|
||||
# And the first values won.
|
||||
assert parsed["subject"] == "first subj"
|
||||
assert parsed["from"]["email"] == "first-from@example.com"
|
||||
assert parsed["message_id"] == "id-a@example.com"
|
||||
assert parsed["in_reply_to"] == "irt-a@example.com"
|
||||
assert parsed["references"] == "<ref-a@example.com>"
|
||||
|
||||
|
||||
class TestParsedHeadersTypeContract:
|
||||
"""The ``parsed["headers"]`` dict follows a fixed per-header contract:
|
||||
|
||||
- Headers registered with max=1 in the IANA Provisional Message
|
||||
Header Field Registry (RFC 5322 §3.6, RFC 3834 §5, RFC 2045 /
|
||||
2046 / 2183, RFC 4021, RFC 3798, RFC 5703, RFC 8058, RFC 8098)
|
||||
are stored as ``str``. On duplication, the first occurrence
|
||||
wins — matching stdlib ``email.message.Message[name]`` semantics.
|
||||
- Every other header (``received``, ``return-path``, ``precedence``,
|
||||
``dkim-signature``, ``authentication-results``, ``arc-*``,
|
||||
``list-*``, ``comments``, ``keywords``, every ``X-*`` /
|
||||
optional-field) is stored as ``list[str]`` in document order.
|
||||
|
||||
Consumers can therefore rely on the type at the call site based on
|
||||
the header name alone — no runtime ``isinstance`` checks. Tests
|
||||
here exercise the public API only so they survive a swap to the
|
||||
Python stdlib parser.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"header_name",
|
||||
[
|
||||
# RFC 5322 §3.6
|
||||
"subject",
|
||||
"from",
|
||||
"sender",
|
||||
"reply-to",
|
||||
"to",
|
||||
"cc",
|
||||
"bcc",
|
||||
"date",
|
||||
"message-id",
|
||||
"in-reply-to",
|
||||
"references",
|
||||
# RFC 3834 §5
|
||||
"auto-submitted",
|
||||
# RFC 2045 / 2046 / 2183
|
||||
"mime-version",
|
||||
"content-type",
|
||||
"content-transfer-encoding",
|
||||
# RFC 4021 — message-class indicators
|
||||
"importance",
|
||||
"priority",
|
||||
"sensitivity",
|
||||
# RFC 8098 — MDN request
|
||||
"disposition-notification-to",
|
||||
# RFC 3798 — MDN reporting
|
||||
"original-message-id",
|
||||
# RFC 5703 — Sieve archive
|
||||
"original-from",
|
||||
"original-subject",
|
||||
# RFC 8058 — one-click unsubscribe
|
||||
"list-unsubscribe-post",
|
||||
],
|
||||
)
|
||||
def test_scalar_header_is_str(self, header_name):
|
||||
"""Every header registered with max=1 in the IANA registry must
|
||||
be ``str`` in ``parsed["headers"]``, even when the input had
|
||||
only one occurrence. Pre-emptive coverage: a future consumer
|
||||
reading any of these can rely on the type without
|
||||
``isinstance``."""
|
||||
raw = (
|
||||
b"From: sender@example.com\r\n"
|
||||
b"To: recipient@example.com\r\n"
|
||||
b"Date: Thu, 4 Jun 2026 00:47:09 +0000\r\n"
|
||||
b"Subject: hi\r\n"
|
||||
b"Sender: real-sender@example.com\r\n"
|
||||
b"Reply-To: replies@example.com\r\n"
|
||||
b"Cc: cc@example.com\r\n"
|
||||
b"Bcc: bcc@example.com\r\n"
|
||||
b"Message-ID: <id@example.com>\r\n"
|
||||
b"In-Reply-To: <parent@example.com>\r\n"
|
||||
b"References: <ref@example.com>\r\n"
|
||||
b"Auto-Submitted: no\r\n"
|
||||
b"Importance: high\r\n"
|
||||
b"Priority: urgent\r\n"
|
||||
b"Sensitivity: Personal\r\n"
|
||||
b"Disposition-Notification-To: mdn@example.com\r\n"
|
||||
b"Original-Message-ID: <orig@example.com>\r\n"
|
||||
b"Original-From: orig@example.com\r\n"
|
||||
b"Original-Subject: original subject\r\n"
|
||||
b"List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n"
|
||||
b"MIME-Version: 1.0\r\n"
|
||||
b"Content-Type: text/plain\r\n"
|
||||
b"Content-Transfer-Encoding: 7bit\r\n"
|
||||
b"\r\n"
|
||||
b"body\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
value = parsed["headers"].get(header_name)
|
||||
assert isinstance(value, str), (
|
||||
f"{header_name} expected str, got {type(value).__name__}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"header_name,header_value",
|
||||
[
|
||||
# Trace fields — explicitly unlimited per RFC 5322 §3.6.7.
|
||||
("received", "from a.example.com by b.example.com"),
|
||||
("return-path", "<sender@example.com>"),
|
||||
# RFC 5322 §3.6.5 — unlimited.
|
||||
("comments", "a comment"),
|
||||
("keywords", "tag1, tag2"),
|
||||
# Optional-field — RFC 5322 §3.6.8 unlimited.
|
||||
("precedence", "bulk"),
|
||||
("x-mailer", "PHPMailer 7.1.1"),
|
||||
("x-priority", "1"),
|
||||
# RFC 6376 — multiple signatures expected.
|
||||
("dkim-signature", "v=1; a=rsa-sha256; d=example.com; ..."),
|
||||
# RFC 8601 — one per authserv-id, repeatable.
|
||||
("authentication-results", "mta.example.com; spf=pass"),
|
||||
# RFC 2369 list-* — repeatable in practice.
|
||||
("list-id", "<list.example.com>"),
|
||||
("list-unsubscribe", "<mailto:u@example.com>"),
|
||||
],
|
||||
)
|
||||
def test_repeatable_header_is_list(self, header_name, header_value):
|
||||
"""Every header outside ``_SCALAR_HEADERS`` must be ``list[str]``
|
||||
even when it appears only once."""
|
||||
raw = (
|
||||
b"From: sender@example.com\r\n"
|
||||
b"Subject: hi\r\n"
|
||||
+ header_name.encode("ascii")
|
||||
+ b": "
|
||||
+ header_value.encode("ascii")
|
||||
+ b"\r\n"
|
||||
b"\r\n"
|
||||
b"body\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
value = parsed["headers"].get(header_name)
|
||||
assert isinstance(value, list), (
|
||||
f"{header_name} expected list, got {type(value).__name__}"
|
||||
)
|
||||
assert value == [header_value]
|
||||
|
||||
def test_multiple_received_preserved_in_order(self):
|
||||
"""``Received`` is repeatable — every hop must appear in
|
||||
document order so spam / forensics consumers can walk the chain."""
|
||||
raw = (
|
||||
b"Received: from hop3 by hop4\r\n"
|
||||
b"Received: from hop2 by hop3\r\n"
|
||||
b"Received: from hop1 by hop2\r\n"
|
||||
b"From: sender@example.com\r\n"
|
||||
b"Subject: traced\r\n"
|
||||
b"\r\n"
|
||||
b"body\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
received = parsed["headers"]["received"]
|
||||
assert isinstance(received, list)
|
||||
assert len(received) == 3
|
||||
assert "hop3 by hop4" in received[0]
|
||||
assert "hop2 by hop3" in received[1]
|
||||
assert "hop1 by hop2" in received[2]
|
||||
|
||||
def test_multiple_dkim_signatures_preserved(self):
|
||||
"""Multiple ``DKIM-Signature`` headers (RFC 6376) — each relay
|
||||
can add its own. All must survive parsing as list entries."""
|
||||
sig_a = "v=1; a=rsa-sha256; d=wanadoo.fr; s=t20230301; bh=AAA"
|
||||
sig_b = "v=1; a=rsa-sha256; d=ac-limoges.fr; s=default; bh=BBB"
|
||||
raw = (
|
||||
b"From: sender@example.com\r\n"
|
||||
b"Subject: dkim test\r\n"
|
||||
b"DKIM-Signature: " + sig_a.encode() + b"\r\n"
|
||||
b"DKIM-Signature: " + sig_b.encode() + b"\r\n"
|
||||
b"\r\n"
|
||||
b"body\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
signatures = parsed["headers"]["dkim-signature"]
|
||||
assert isinstance(signatures, list)
|
||||
assert len(signatures) == 2
|
||||
assert "wanadoo.fr" in signatures[0]
|
||||
assert "ac-limoges.fr" in signatures[1]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"header_name",
|
||||
[
|
||||
"received",
|
||||
"dkim-signature",
|
||||
"authentication-results",
|
||||
"comments",
|
||||
"keywords",
|
||||
"resent-from",
|
||||
"resent-to",
|
||||
"list-id",
|
||||
"x-custom",
|
||||
],
|
||||
)
|
||||
def test_repeatable_header_preserves_all_occurrences(self, header_name):
|
||||
"""Every occurrence of a repeatable header must be retained in
|
||||
document order — no silent deduplication or truncation."""
|
||||
raw = (
|
||||
b"From: sender@example.com\r\nSubject: hi\r\n"
|
||||
+ (header_name.encode() + b": value-A\r\n")
|
||||
+ (header_name.encode() + b": value-B\r\n")
|
||||
+ (header_name.encode() + b": value-C\r\n")
|
||||
+ b"\r\nbody\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
values = parsed["headers"][header_name]
|
||||
assert values == ["value-A", "value-B", "value-C"], (
|
||||
f"{header_name}: expected all three preserved in order, got {values!r}"
|
||||
)
|
||||
|
||||
def test_headers_blocks_always_uses_list_values(self):
|
||||
"""Inside ``headers_blocks`` every value is ``list[str]``,
|
||||
even for scalar headers like Subject — block consumers index
|
||||
uniformly so the trusted-relays cut stays simple.
|
||||
|
||||
``parsed["headers"]`` and ``parsed["headers_blocks"]`` have
|
||||
intentionally different shapes; pin both so a future refactor
|
||||
doesn't quietly unify them.
|
||||
"""
|
||||
raw = (
|
||||
b"Received: from hop1 by hop2\r\n"
|
||||
b"From: sender@example.com\r\n"
|
||||
b"Subject: scalar in block\r\n"
|
||||
b"\r\nbody\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
# parsed["headers"]["subject"] is the str scalar.
|
||||
assert parsed["headers"]["subject"] == "scalar in block"
|
||||
# parsed["headers_blocks"][N]["subject"] is the same value
|
||||
# wrapped in a single-element list.
|
||||
for block in parsed["headers_blocks"]:
|
||||
if "subject" in block:
|
||||
assert isinstance(block["subject"], list)
|
||||
assert block["subject"] == ["scalar in block"]
|
||||
break
|
||||
else: # pragma: no cover — fail loudly if no block had Subject
|
||||
pytest.fail("Subject not found in any header block")
|
||||
|
||||
@pytest.mark.parametrize("case_variant", ["From", "FROM", "from", "FrOm"])
|
||||
def test_scalar_header_lookup_is_case_insensitive(self, case_variant):
|
||||
"""Header names in input are case-insensitive per RFC 5322
|
||||
§3.6.8; ``parsed["headers"]`` keys are always lowercase and
|
||||
callers must reach values via the lowercase form regardless of
|
||||
the wire casing."""
|
||||
raw = (
|
||||
case_variant.encode("ascii") + b": sender@example.com\r\n"
|
||||
b"Subject: hi\r\n\r\nbody\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed["headers"]["from"] == "sender@example.com"
|
||||
|
||||
def test_auto_submitted_duplicate_first_wins_per_rfc_3834(self):
|
||||
"""RFC 3834 §5 makes Auto-Submitted max=1; on duplication we
|
||||
take the first value — matching stdlib semantics. A sender
|
||||
that emits ``Auto-Submitted: no`` first and
|
||||
``Auto-Submitted: auto-replied`` second is non-compliant; we
|
||||
intentionally trust the first occurrence."""
|
||||
raw = (
|
||||
b"From: sender@example.com\r\n"
|
||||
b"Subject: hi\r\n"
|
||||
b"Auto-Submitted: no\r\n"
|
||||
b"Auto-Submitted: auto-replied\r\n"
|
||||
b"\r\nbody\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
assert parsed["headers"]["auto-submitted"] == "no"
|
||||
|
||||
def test_precedence_poison_first_still_detected_as_auto_reply(self):
|
||||
"""Defence-in-depth for loop detection. ``Precedence`` is
|
||||
repeatable per RFC 5322 §3.6.8 (optional-field); the autoreply
|
||||
check iterates every occurrence, so a non-bulk value cannot
|
||||
mask a later ``bulk`` one and provoke a reply loop."""
|
||||
raw = (
|
||||
b"From: list@example.com\r\nSubject: newsletter\r\n"
|
||||
b"Precedence: not-bulk\r\n"
|
||||
b"Precedence: bulk\r\n"
|
||||
b"\r\nbody\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel
|
||||
_is_auto_reply_message,
|
||||
)
|
||||
|
||||
assert _is_auto_reply_message(parsed["headers"]) is True
|
||||
|
||||
def test_return_path_poison_first_still_detected_as_bounce(self):
|
||||
"""Same defence-in-depth as Precedence: a duplicate
|
||||
``Return-Path`` with a non-bounce value first must not mask a
|
||||
bounce indicator (``<>`` / empty) later."""
|
||||
raw = (
|
||||
b"From: MAILER-DAEMON@example.com\r\nSubject: bounce\r\n"
|
||||
b"Return-Path: <victim@anywhere.com>\r\n"
|
||||
b"Return-Path: <>\r\n"
|
||||
b"\r\nbody\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel
|
||||
_is_auto_reply_message,
|
||||
)
|
||||
|
||||
assert _is_auto_reply_message(parsed["headers"]) is True
|
||||
|
||||
def test_headers_dict_type_matches_call_site_expectation(self):
|
||||
"""Real-world inbound with multiple legitimately-repeated
|
||||
``DKIM-Signature`` and ``Received`` headers (RFC 6376 /
|
||||
RFC 5322) must pass through the autoreply detection logic —
|
||||
which does ``.strip().lower()`` on scalar lookups — without
|
||||
raising."""
|
||||
raw = (
|
||||
b"Received: from hop2 by hop3\r\n"
|
||||
b"Received: from hop1 by hop2\r\n"
|
||||
b"DKIM-Signature: v=1; a=rsa-sha256; d=a.example.com\r\n"
|
||||
b"DKIM-Signature: v=1; a=rsa-sha256; d=b.example.com\r\n"
|
||||
b"From: sender@example.com\r\n"
|
||||
b"To: recipient@example.com\r\n"
|
||||
b"Subject: legitimate forwarded mail\r\n"
|
||||
b"Precedence: bulk\r\n"
|
||||
b"\r\n"
|
||||
b"body\r\n"
|
||||
)
|
||||
parsed = parse_email_message(raw)
|
||||
# Defer the autoreply import to keep this test isolated from
|
||||
# the autoreply module's import-time side effects.
|
||||
from core.mda.autoreply import ( # pylint: disable=import-outside-toplevel
|
||||
_is_auto_reply_message,
|
||||
)
|
||||
|
||||
# Must not raise — and the bulk Precedence must be detected.
|
||||
assert _is_auto_reply_message(parsed["headers"]) is True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main()
|
||||
|
||||
@@ -49,7 +49,7 @@ dependencies = [
|
||||
"drf_spectacular==0.29.0",
|
||||
"opensearch-py==2.8.0",
|
||||
"factory_boy==3.3.3",
|
||||
"flanker@git+https://github.com/sylvinus/flanker@a4248826794d446b07fb26719479c1c355411f5a",
|
||||
"flanker@git+https://github.com/sylvinus/flanker@77f4582044f1a8549d49333d50d9bded1176ca09",
|
||||
"gunicorn==25.1.0",
|
||||
"icalendar==7.0.3",
|
||||
"jsonschema==4.26.0",
|
||||
|
||||
Generated
+2
-2
@@ -778,7 +778,7 @@ wheels = [
|
||||
[[package]]
|
||||
name = "flanker"
|
||||
version = "0.9.11"
|
||||
source = { git = "https://github.com/sylvinus/flanker?rev=a4248826794d446b07fb26719479c1c355411f5a#a4248826794d446b07fb26719479c1c355411f5a" }
|
||||
source = { git = "https://github.com/sylvinus/flanker?rev=77f4582044f1a8549d49333d50d9bded1176ca09#77f4582044f1a8549d49333d50d9bded1176ca09" }
|
||||
dependencies = [
|
||||
{ name = "attrs" },
|
||||
{ name = "chardet" },
|
||||
@@ -1215,7 +1215,7 @@ requires-dist = [
|
||||
{ name = "drf-spectacular", specifier = "==0.29.0" },
|
||||
{ name = "drf-spectacular-sidecar", marker = "extra == 'dev'", specifier = "==2026.1.1" },
|
||||
{ name = "factory-boy", specifier = "==3.3.3" },
|
||||
{ name = "flanker", git = "https://github.com/sylvinus/flanker?rev=a4248826794d446b07fb26719479c1c355411f5a" },
|
||||
{ name = "flanker", git = "https://github.com/sylvinus/flanker?rev=77f4582044f1a8549d49333d50d9bded1176ca09" },
|
||||
{ name = "flower", marker = "extra == 'dev'", specifier = "==2.0.1" },
|
||||
{ name = "gunicorn", specifier = "==25.1.0" },
|
||||
{ name = "hypothesis", marker = "extra == 'dev'", specifier = "==6.151.9" },
|
||||
|
||||
Reference in New Issue
Block a user