mirror of
https://github.com/suitenumerique/messages.git
synced 2026-08-17 21:25:41 +02:00
Merge branch 'development' into mta-in-py
This commit is contained in:
+1
-2
@@ -34,5 +34,4 @@ db.sqlite3
|
||||
|
||||
# Frontend
|
||||
node_modules
|
||||
out
|
||||
.next
|
||||
dist
|
||||
|
||||
@@ -8,6 +8,11 @@ name: Lint and tests
|
||||
branches:
|
||||
- '*'
|
||||
|
||||
# Default to least-privilege at the workflow scope. Jobs that need
|
||||
# more (e.g. opening / editing comments) opt in explicitly.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
COMPOSE_BAKE: true
|
||||
|
||||
@@ -24,6 +29,17 @@ jobs:
|
||||
run: make lint-back
|
||||
|
||||
|
||||
typecheck-jmap-email:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
- name: Create env files
|
||||
run: make create-env-files
|
||||
- name: Run pyright (strict) on the jmap-email library
|
||||
run: make typecheck-jmap-email
|
||||
|
||||
|
||||
test-back:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
|
||||
+5
-6
@@ -14,8 +14,8 @@ dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
/lib/
|
||||
/lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
@@ -27,7 +27,6 @@ share/python-wheels/
|
||||
*.egg
|
||||
MANIFEST
|
||||
.DS_Store
|
||||
.next/
|
||||
|
||||
# Translations # Translations
|
||||
*.pot
|
||||
@@ -46,12 +45,12 @@ env.d/terraform
|
||||
|
||||
# npm
|
||||
node_modules
|
||||
src/frontend/out/
|
||||
src/frontend/dist/
|
||||
tsconfig.tsbuildinfo
|
||||
src/frontend/.config
|
||||
src/frontend/.next
|
||||
src/frontend/next-env.d.ts
|
||||
src/frontend/src/routes.gen.ts
|
||||
.vite
|
||||
.tanstack
|
||||
|
||||
# Mails
|
||||
src/backend/core/templates/mail/
|
||||
|
||||
+130
-1
@@ -8,6 +8,132 @@ and this project adheres to
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
- Bump keycloak to 26.6.3
|
||||
|
||||
## [0.8.0] - 2026-06-18
|
||||
|
||||
### Added
|
||||
|
||||
- Allow permanently deleting drafts and improve draft edition
|
||||
- Allow passwordless mailbox creation when identity sync is off #707
|
||||
- Gather mailbox settings into a dialog
|
||||
- Translate template placeholder and add `user_name` builtin variable
|
||||
- Report selfcheck status to Sentry crons #694
|
||||
|
||||
### Changed
|
||||
|
||||
- Drop Next.js for Vite + TanStack Router #675
|
||||
- Move email parser & composer to new `jmap-email` library #700
|
||||
- Add PyPI release scripts for `jmap-email`
|
||||
- Use `LaGaufreV2` component
|
||||
- Improve thread navigation a11y and multiselect UX #708
|
||||
- Refine mailbox dropdown menu #705
|
||||
- New homepage illustration #702
|
||||
- Internationalize missing strings
|
||||
- Wrap autoreply date column
|
||||
- Bump `dompurify` to 3.4.11
|
||||
- Bump `django-lasuite` to 0.0.26 #689
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix composer issues
|
||||
- Add `To` header to outbound mails missing one #712
|
||||
- Manage message/delivery-status attachments at compose
|
||||
- Persist mailbox name when contact is missing
|
||||
- Fix order and default calendar selection when RSVPing #699
|
||||
- Fix display of recurring events with exceptions #686
|
||||
- Fix opportunistic TLS against MXes with mismatched certs #687
|
||||
- Fix mbox detection as `text/html` with some libmagic versions #696
|
||||
- Complete PST email folder prefixes list
|
||||
- Fix milter socket permission race on startup #693
|
||||
|
||||
### Security
|
||||
|
||||
- Add some defense-in-depth bits #706
|
||||
- Harden SMTP connection & proxies config
|
||||
- Harden inbound email parsing #695
|
||||
|
||||
## [0.7.0] - 2026-05-28
|
||||
|
||||
### Added
|
||||
|
||||
- Attachments preview #676
|
||||
- Add link to a CalDAV instance to accept events directly #584
|
||||
|
||||
### Changed
|
||||
|
||||
- Improve sending experience #681
|
||||
- Remove deprecated model fields from tiered storage migration #678
|
||||
|
||||
### Fixed
|
||||
|
||||
- Unmount thread view immediately on unselect thread #680
|
||||
- Prevent refetch thread messages on draft deletion #682
|
||||
|
||||
## [0.6.0] - 2026-05-20
|
||||
|
||||
### Added
|
||||
|
||||
- Add thread assignation feature #645
|
||||
- Add mention notifications via UserEvent #621
|
||||
- Allow sending internal messages through ThreadEvent #566
|
||||
- Add thread deep linking #664
|
||||
- Add label assignment with archive and bulk label widget
|
||||
- Enable inviting users that haven't logged in yet #644
|
||||
- Add configurable inbound auth backends #636
|
||||
- Add encryption, custom scopes, levels and auditing on channels #599
|
||||
- Add recursive SPF check and optional send-time validation #625
|
||||
- Add tiered storage and refactor blobs/attachments
|
||||
- Add mandatory TOTP field and search field in admin #667
|
||||
- Add silent login support
|
||||
- Make panel sections resizable
|
||||
- Add read/unread action on thread action bar #659
|
||||
- Add lprobe healthchecks and checksum verification for lprobe + Caddy #600
|
||||
|
||||
### Changed
|
||||
|
||||
- Improve message composer
|
||||
- Switch back to Python's stdlib for email composition
|
||||
- Put split thread feature behind a feature flag #624
|
||||
- Show tooltip to confirm mailbox refresh
|
||||
- Disable application menu when no option is available
|
||||
- Focus `to` field on forward
|
||||
- Align send button on the left
|
||||
- Upgrade Cunningham and ui-kit
|
||||
- Localize attachment separator
|
||||
- Force default language on frontend #647
|
||||
- Add DNS propagation delay info #654
|
||||
- Allow specifying a channel id for the home feedback widget #655
|
||||
- Support legacy and new widget attribute #650
|
||||
- Update widget logic to latest version #649
|
||||
- Refactor thread query cache management #642
|
||||
- Allow reindexing from a given date
|
||||
- Defer indexation tasks for better throughput
|
||||
- Improve `search_reindex` bulk payload
|
||||
- Move imports and reindex worker queues to dedicated containers #643
|
||||
- Bump Keycloak to 26.6.1 #637
|
||||
|
||||
### Fixed
|
||||
|
||||
- Improve PST import logic
|
||||
- Allow thread editor to destroy thread accesses #668
|
||||
- Enforce full edit rights on thread mutations
|
||||
- Fix race condition in last-editor deletion guard
|
||||
- Fix thread panel header with nested label #658
|
||||
- Fix label popup stacking with create-label modal #635
|
||||
- Fix email parsing edge cases with UTF-8 in flanker #656
|
||||
- Fix threads ordering #617
|
||||
- Fix double request and flickering on search #596
|
||||
- Handle non-serializable Celery task errors and stop infinite polling #633
|
||||
- Quote error field and log SOCKS proxy in outbound delivery #626
|
||||
- Do not mark thread as read when sending autoreply #594
|
||||
|
||||
### Security
|
||||
|
||||
- Stop flagging inbound `From=To` mails as `is_sender` #652
|
||||
- Force including special characters in generated passwords #640
|
||||
- Factorize SSRF code and allow redirects in image proxy #631
|
||||
|
||||
## [0.5.0] - 2026-03-16
|
||||
|
||||
### Added
|
||||
@@ -165,7 +291,10 @@ and this project adheres to
|
||||
- Exclude `is_trashed` and `is_spam` threads from search results by default
|
||||
- `to` search modifier now looks for messages where recipient fields (to, cc, bcc) contain the given email address.
|
||||
|
||||
[unreleased]: https://github.com/suitenumerique/messages/compare/v0.5.0...main
|
||||
[unreleased]: https://github.com/suitenumerique/messages/compare/v0.8.0...main
|
||||
[0.8.0]: https://github.com/suitenumerique/messages/compare/v0.7.0...v0.8.0
|
||||
[0.7.0]: https://github.com/suitenumerique/messages/compare/v0.6.0...v0.7.0
|
||||
[0.6.0]: https://github.com/suitenumerique/messages/compare/v0.5.0...v0.6.0
|
||||
[0.5.0]: https://github.com/suitenumerique/messages/compare/v0.4.0...v0.5.0
|
||||
[0.4.0]: https://github.com/suitenumerique/messages/compare/v0.3.0...v0.4.0
|
||||
[0.3.0]: https://github.com/suitenumerique/messages/compare/v0.2.0...v0.3.0
|
||||
|
||||
@@ -115,7 +115,9 @@ build: ## build the project containers
|
||||
.PHONY: build
|
||||
|
||||
build-back-distroless: ## build the distroless production image
|
||||
@docker build --target runtime-distroless-prod -t messages-distroless -f src/backend/Dockerfile src/backend/
|
||||
@docker buildx build --load --target runtime-distroless-prod -t messages-distroless \
|
||||
-f src/backend/Dockerfile \
|
||||
src/backend/
|
||||
.PHONY: build-back-distroless
|
||||
|
||||
test-back-distroless: build-back-distroless ## build and smoke-test the distroless production image
|
||||
@@ -227,6 +229,10 @@ analyze-back: ## analyze back-end python sources
|
||||
@$(COMPOSE_RUN_APP_TOOLS) sh -c "pylint ."
|
||||
.PHONY: analyze-back
|
||||
|
||||
analyze-front: ## analyze frontend bundle sizes (per-chunk + per-package breakdown)
|
||||
@$(COMPOSE) run --rm frontend-tools npm run analyze
|
||||
.PHONY: analyze-front
|
||||
|
||||
typecheck-front: ## run the frontend type checker
|
||||
@$(COMPOSE) run --rm frontend-tools npm run ts:check
|
||||
.PHONY: typecheck-front
|
||||
@@ -314,6 +320,28 @@ test-mpa: ## run the mpa tests
|
||||
@$(COMPOSE) run --build --rm mpa-test
|
||||
.PHONY: test-mpa
|
||||
|
||||
test-jmap-email: ## run the jmap-email package tests (zero infrastructure deps)
|
||||
@$(COMPOSE) run --build --rm jmap-email-test
|
||||
.PHONY: test-jmap-email
|
||||
|
||||
fuzz-jmap-email: ## run the jmap-email Hypothesis fuzz suite
|
||||
@$(COMPOSE) run --build --rm jmap-email-test pytest -m fuzz tests/
|
||||
.PHONY: fuzz-jmap-email
|
||||
|
||||
lint-jmap-email: ## lint the jmap-email library (ruff check + format check + pylint)
|
||||
@$(COMPOSE) run --build --rm --entrypoint ruff jmap-email-test check jmap_email tests
|
||||
@$(COMPOSE) run --build --rm --entrypoint ruff jmap-email-test format --check jmap_email tests
|
||||
@$(COMPOSE) run --build --rm --entrypoint pylint jmap-email-test jmap_email tests
|
||||
.PHONY: lint-jmap-email
|
||||
|
||||
typecheck-jmap-email: ## type-check the jmap-email library with ty (Astral, Rust)
|
||||
@$(COMPOSE) run --build --rm --entrypoint ty jmap-email-test check
|
||||
.PHONY: typecheck-jmap-email
|
||||
|
||||
release-jmap-email: ## publish jmap-email to PyPI (interactive: TestPyPI → smoke install → PyPI)
|
||||
@bin/release-jmap-email.sh
|
||||
.PHONY: release-jmap-email
|
||||
|
||||
test-socks-proxy: ## run the socks-proxy tests
|
||||
@$(COMPOSE) run --build --rm socks-proxy-test
|
||||
.PHONY: test-socks-proxy
|
||||
|
||||
@@ -38,7 +38,7 @@ Messages is the all-in-one collaborative inbox for [La Suite territoriale](https
|
||||
|
||||
Messages is a full communication platform enabling teams to collaborate on emails through shared or personal mailboxes.
|
||||
|
||||
It features a [MTA](https://en.wikipedia.org/wiki/Message_transfer_agent) based on [Postfix](https://www.postfix.org/), a custom [MDA](https://en.wikipedia.org/wiki/Message_delivery_agent) built on top of [Django Rest Framework](https://www.django-rest-framework.org/) and a frontend using [Next.js](https://nextjs.org/) and [BlockNote.js](https://www.blocknotejs.org/).
|
||||
It features a [MTA](https://en.wikipedia.org/wiki/Message_transfer_agent) based on [Postfix](https://www.postfix.org/), a custom [MDA](https://en.wikipedia.org/wiki/Message_delivery_agent) built on top of [Django Rest Framework](https://www.django-rest-framework.org/) and a frontend using [Vite](https://vite.dev/), [TanStack Router](https://tanstack.com/router/) and [BlockNote.js](https://www.blocknotejs.org/).
|
||||
|
||||
### Familiar messaging features
|
||||
* 📝 Receive, draft and send emails.
|
||||
|
||||
Executable
+350
@@ -0,0 +1,350 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Interactive PyPI release for the jmap-email package.
|
||||
#
|
||||
# Hermetic: every step runs inside the python:3.14.5-slim image, so a
|
||||
# clean VM only needs Docker. The host never touches pip or twine.
|
||||
#
|
||||
# Flow:
|
||||
# 1. Run ``make lint-jmap-email typecheck-jmap-email test-jmap-email``
|
||||
# 2. Build sdist + wheel inside Docker, run ``twine check``
|
||||
# 3. Auto-inspect wheel/sdist contents and METADATA fields
|
||||
# 4. Prompt for TestPyPI API token, upload, smoke-install in a
|
||||
# throwaway container
|
||||
# 5. Prompt for PyPI API token, upload
|
||||
#
|
||||
# Each gate is interactive (y/N). Bail out anytime with Ctrl-C.
|
||||
#
|
||||
# Set ``SKIP_GATES=1`` to skip lint/typecheck/tests on retry.
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PKG_DIR="${REPO_DIR}/src/jmap-email"
|
||||
PYTHON_IMAGE="python:3.14.5-slim"
|
||||
|
||||
BOLD=$'\033[1m'
|
||||
GREEN=$'\033[1;32m'
|
||||
BLUE=$'\033[1;34m'
|
||||
RED=$'\033[1;31m'
|
||||
YELLOW=$'\033[1;33m'
|
||||
RESET=$'\033[0m'
|
||||
|
||||
say() { printf '%s%s%s\n' "${BLUE}" "$*" "${RESET}"; }
|
||||
ok() { printf '%s✓ %s%s\n' "${GREEN}" "$*" "${RESET}"; }
|
||||
warn() { printf '%s%s%s\n' "${YELLOW}" "$*" "${RESET}"; }
|
||||
die() { printf '%s✗ %s%s\n' "${RED}" "$*" "${RESET}" >&2; exit 1; }
|
||||
|
||||
confirm() {
|
||||
local prompt="$1"
|
||||
local ans
|
||||
read -r -p "${BOLD}${prompt} [y/N]${RESET} " ans
|
||||
[[ "${ans}" =~ ^[Yy]$ ]] || die "aborted"
|
||||
}
|
||||
|
||||
read_token() {
|
||||
# $1 = label, $2 = name of variable to assign into
|
||||
local label="$1" varname="$2" token
|
||||
read -r -s -p "${BOLD}${label} API token (pypi-…):${RESET} " token
|
||||
echo
|
||||
[[ -n "${token}" ]] || die "empty token"
|
||||
[[ "${token}" == pypi-* ]] || warn "token does not start with 'pypi-' — continuing anyway"
|
||||
printf -v "${varname}" '%s' "${token}"
|
||||
}
|
||||
|
||||
# ── pre-flight ────────────────────────────────────────────────────────────
|
||||
command -v docker >/dev/null || die "docker not found in PATH"
|
||||
[[ -f "${PKG_DIR}/pyproject.toml" ]] || die "no pyproject.toml at ${PKG_DIR}"
|
||||
|
||||
VERSION="$(awk -F'"' '/^version = /{print $2; exit}' "${PKG_DIR}/pyproject.toml")"
|
||||
[[ -n "${VERSION}" ]] || die "could not read version from pyproject.toml"
|
||||
|
||||
printf '\n%s════════════════════════════════════════════════════════════%s\n' "${BLUE}" "${RESET}"
|
||||
printf '%s Release jmap-email %s%s\n' "${BLUE}" "${VERSION}" "${RESET}"
|
||||
printf '%s════════════════════════════════════════════════════════════%s\n\n' "${BLUE}" "${RESET}"
|
||||
say "Image: ${PYTHON_IMAGE}"
|
||||
say "Package: ${PKG_DIR}"
|
||||
say "Flow: lint+typecheck+tests → build → inspect → TestPyPI → smoke install → PyPI"
|
||||
[[ "${SKIP_GATES:-0}" == "1" ]] && warn "SKIP_GATES=1 — lint/typecheck/tests will be skipped"
|
||||
echo
|
||||
confirm "Proceed?"
|
||||
|
||||
# ── 1. lint, typecheck, tests ─────────────────────────────────────────────
|
||||
if [[ "${SKIP_GATES:-0}" == "1" ]]; then
|
||||
warn "skipping lint/typecheck/tests (SKIP_GATES=1)"
|
||||
else
|
||||
say "→ make lint-jmap-email"
|
||||
make -C "${REPO_DIR}" lint-jmap-email
|
||||
|
||||
say "→ make typecheck-jmap-email"
|
||||
make -C "${REPO_DIR}" typecheck-jmap-email
|
||||
|
||||
say "→ make test-jmap-email"
|
||||
make -C "${REPO_DIR}" test-jmap-email
|
||||
|
||||
ok "Lint + typecheck + tests passed"
|
||||
fi
|
||||
|
||||
# ── 2. build + check ──────────────────────────────────────────────────────
|
||||
say "→ Cleaning previous artifacts"
|
||||
rm -rf "${PKG_DIR}/dist" "${PKG_DIR}/build"
|
||||
|
||||
say "→ Building sdist + wheel inside ${PYTHON_IMAGE}"
|
||||
docker run --rm -t \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
-v "${PKG_DIR}:/pkg" \
|
||||
-w /pkg \
|
||||
-e HOME=/tmp \
|
||||
"${PYTHON_IMAGE}" \
|
||||
bash -c '
|
||||
set -eo pipefail
|
||||
pip install --quiet --no-cache-dir --root-user-action=ignore --target /tmp/pip build twine
|
||||
export PYTHONPATH=/tmp/pip
|
||||
python -m build --outdir dist
|
||||
python -m twine check dist/*
|
||||
'
|
||||
|
||||
echo
|
||||
ls -lh "${PKG_DIR}/dist/"
|
||||
echo
|
||||
ok "Build + twine check passed"
|
||||
|
||||
# ── 3. inspect artifact contents ──────────────────────────────────────────
|
||||
say "→ Inspecting wheel + sdist contents and METADATA"
|
||||
docker run --rm -i \
|
||||
-v "${PKG_DIR}/dist:/dist:ro" \
|
||||
"${PYTHON_IMAGE}" \
|
||||
python - "${VERSION}" <<'PYEOF'
|
||||
import re
|
||||
import sys
|
||||
import tarfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
VERSION = sys.argv[1]
|
||||
DIST = Path("/dist")
|
||||
WHEEL = DIST / f"jmap_email-{VERSION}-py3-none-any.whl"
|
||||
SDIST = DIST / f"jmap_email-{VERSION}.tar.gz"
|
||||
|
||||
errors, warnings = [], []
|
||||
|
||||
# ── wheel: file list ─────────────────────────────────────────────
|
||||
if not WHEEL.exists():
|
||||
print(f"FATAL: wheel missing: {WHEEL}")
|
||||
sys.exit(1)
|
||||
with zipfile.ZipFile(WHEEL) as zf:
|
||||
wheel_names = set(zf.namelist())
|
||||
metadata = zf.read(f"jmap_email-{VERSION}.dist-info/METADATA").decode()
|
||||
wheel_file_count = sum(1 for n in wheel_names if not n.endswith("/"))
|
||||
|
||||
expected_wheel = {
|
||||
"jmap_email/__init__.py",
|
||||
"jmap_email/composer.py",
|
||||
"jmap_email/helpers.py",
|
||||
"jmap_email/limits.py",
|
||||
"jmap_email/parser.py",
|
||||
"jmap_email/types.py",
|
||||
"jmap_email/py.typed",
|
||||
f"jmap_email-{VERSION}.dist-info/METADATA",
|
||||
f"jmap_email-{VERSION}.dist-info/WHEEL",
|
||||
f"jmap_email-{VERSION}.dist-info/RECORD",
|
||||
}
|
||||
missing = expected_wheel - wheel_names
|
||||
if missing:
|
||||
errors.append(f"wheel missing required files: {sorted(missing)}")
|
||||
|
||||
forbidden = [
|
||||
("tests/", lambda n: n.startswith("tests/")),
|
||||
("examples/", lambda n: n.startswith("examples/")),
|
||||
(".pyc files", lambda n: n.endswith(".pyc")),
|
||||
("__pycache__", lambda n: "__pycache__" in n),
|
||||
("Dockerfile", lambda n: n.endswith("Dockerfile")),
|
||||
(".pytest_cache",lambda n: ".pytest_cache" in n),
|
||||
]
|
||||
for label, pred in forbidden:
|
||||
bad = [n for n in wheel_names if pred(n)]
|
||||
if bad:
|
||||
errors.append(f"wheel contains forbidden {label}: {bad[:3]}")
|
||||
|
||||
if not any("LICENSE" in n for n in wheel_names):
|
||||
errors.append("wheel does not bundle LICENSE")
|
||||
|
||||
# ── wheel: METADATA ──────────────────────────────────────────────
|
||||
def meta(key):
|
||||
m = re.search(rf"^{re.escape(key)}: (.+)$", metadata, re.MULTILINE)
|
||||
return m.group(1).strip() if m else None
|
||||
|
||||
if meta("Name") != "jmap-email":
|
||||
errors.append(f"METADATA Name: expected 'jmap-email', got {meta('Name')!r}")
|
||||
if meta("Version") != VERSION:
|
||||
errors.append(f"METADATA Version: expected {VERSION!r}, got {meta('Version')!r}")
|
||||
|
||||
rp = meta("Requires-Python")
|
||||
if not rp or "3.14" not in rp:
|
||||
errors.append(f"METADATA Requires-Python missing or not 3.14+: {rp!r}")
|
||||
|
||||
# Modern PEP 639 uses License-Expression; older hatchling emits License.
|
||||
if not (meta("License-Expression") or meta("License")):
|
||||
errors.append("METADATA has no License or License-Expression")
|
||||
|
||||
dct = meta("Description-Content-Type")
|
||||
if not dct or "markdown" not in dct.lower():
|
||||
errors.append(f"METADATA Description-Content-Type isn't markdown: {dct!r}")
|
||||
|
||||
if "# jmap-email" not in metadata:
|
||||
warnings.append("METADATA description doesn't contain '# jmap-email' — README may not have been embedded")
|
||||
|
||||
for url in ("Homepage", "Repository"):
|
||||
if f"Project-URL: {url}," not in metadata:
|
||||
warnings.append(f"METADATA missing Project-URL: {url}")
|
||||
|
||||
if "Topic :: Communications :: Email" not in metadata:
|
||||
warnings.append("METADATA missing 'Topic :: Communications :: Email' classifier")
|
||||
|
||||
# ── sdist ────────────────────────────────────────────────────────
|
||||
if not SDIST.exists():
|
||||
print(f"FATAL: sdist missing: {SDIST}")
|
||||
sys.exit(1)
|
||||
with tarfile.open(SDIST) as tf:
|
||||
sdist_names = tf.getnames()
|
||||
|
||||
prefix = f"jmap_email-{VERSION}/"
|
||||
sdist_rel = {n[len(prefix):] for n in sdist_names if n.startswith(prefix)}
|
||||
|
||||
expected_sdist = {
|
||||
"pyproject.toml",
|
||||
"README.md",
|
||||
"LICENSE",
|
||||
"CHANGELOG.md",
|
||||
"PKG-INFO",
|
||||
"jmap_email/__init__.py",
|
||||
"jmap_email/parser.py",
|
||||
"jmap_email/composer.py",
|
||||
"jmap_email/py.typed",
|
||||
}
|
||||
missing_sdist = expected_sdist - sdist_rel
|
||||
if missing_sdist:
|
||||
errors.append(f"sdist missing required files: {sorted(missing_sdist)}")
|
||||
|
||||
if not any(n.startswith("tests/") and n.endswith(".py") for n in sdist_rel):
|
||||
warnings.append("sdist contains no tests/*.py — pyproject.toml asked for tests/**/*.py")
|
||||
if not any(n.startswith("examples/") and n.endswith(".py") for n in sdist_rel):
|
||||
warnings.append("sdist contains no examples/*.py — pyproject.toml asked for examples/**/*.py")
|
||||
|
||||
bad_sdist = [n for n in sdist_rel if n.endswith(".pyc") or "__pycache__" in n]
|
||||
if bad_sdist:
|
||||
errors.append(f"sdist contains bytecode: {bad_sdist[:3]}")
|
||||
|
||||
# ── report ──────────────────────────────────────────────────────
|
||||
print()
|
||||
print(f" Wheel: {WHEEL.name} ({WHEEL.stat().st_size // 1024} KB, {wheel_file_count} files)")
|
||||
print(f" Sdist: {SDIST.name} ({SDIST.stat().st_size // 1024} KB, {len(sdist_rel)} entries)")
|
||||
print()
|
||||
print(" METADATA highlights:")
|
||||
for key in ("Name", "Version", "Requires-Python",
|
||||
"License-Expression", "License", "Description-Content-Type"):
|
||||
v = meta(key)
|
||||
if v:
|
||||
print(f" {key:28s} {v}")
|
||||
print()
|
||||
|
||||
if warnings:
|
||||
print(" ⚠ Warnings:")
|
||||
for w in warnings:
|
||||
print(f" {w}")
|
||||
print()
|
||||
|
||||
if errors:
|
||||
print(" ✗ Errors:")
|
||||
for e in errors:
|
||||
print(f" {e}")
|
||||
print()
|
||||
sys.exit(1)
|
||||
|
||||
print(" ✓ All artifact checks passed")
|
||||
PYEOF
|
||||
|
||||
echo
|
||||
confirm "Artifacts look right?"
|
||||
|
||||
# ── 4. TestPyPI ───────────────────────────────────────────────────────────
|
||||
say "→ TestPyPI upload"
|
||||
echo "Get a token at https://test.pypi.org/manage/account/token/"
|
||||
echo "(Account-scoped on first release; project-scoped after.)"
|
||||
read_token "TestPyPI" TESTPYPI_TOKEN
|
||||
|
||||
docker run --rm -t \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
-v "${PKG_DIR}:/pkg" \
|
||||
-w /pkg \
|
||||
-e HOME=/tmp \
|
||||
-e TWINE_USERNAME=__token__ \
|
||||
-e TWINE_PASSWORD="${TESTPYPI_TOKEN}" \
|
||||
"${PYTHON_IMAGE}" \
|
||||
bash -c '
|
||||
set -eo pipefail
|
||||
pip install --quiet --no-cache-dir --root-user-action=ignore --target /tmp/pip twine
|
||||
export PYTHONPATH=/tmp/pip
|
||||
python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/*
|
||||
'
|
||||
ok "Uploaded to TestPyPI"
|
||||
echo " https://test.pypi.org/project/jmap-email/${VERSION}/"
|
||||
|
||||
# ── 5. smoke install ──────────────────────────────────────────────────────
|
||||
say "→ Smoke-installing jmap-email==${VERSION} from TestPyPI"
|
||||
# TestPyPI's index has limited transitive coverage; jmap-email has zero
|
||||
# runtime deps so a bare TestPyPI install is fine. The retry loop covers
|
||||
# the index-propagation lag (~30s after upload).
|
||||
docker run --rm -t \
|
||||
"${PYTHON_IMAGE}" \
|
||||
bash -c "
|
||||
set -eo pipefail
|
||||
for i in 1 2 3 4 5; do
|
||||
if pip install --quiet --no-cache-dir \
|
||||
--index-url https://test.pypi.org/simple/ \
|
||||
jmap-email==${VERSION}; then
|
||||
break
|
||||
fi
|
||||
echo 'index not yet propagated, retrying in 10s…'
|
||||
sleep 10
|
||||
done
|
||||
python -c '
|
||||
import jmap_email
|
||||
assert jmap_email.__version__ == \"${VERSION}\", jmap_email.__version__
|
||||
e = jmap_email.parse_email(b\"From: a@b\r\nSubject: t\r\n\r\nhi\")
|
||||
assert e is not None and e[\"subject\"] == \"t\"
|
||||
print(\"smoke-install ok — version\", jmap_email.__version__)
|
||||
'
|
||||
"
|
||||
ok "Smoke install passed"
|
||||
|
||||
# ── 6. real PyPI ──────────────────────────────────────────────────────────
|
||||
echo
|
||||
warn "Next step is irreversible: PyPI version numbers cannot be reused."
|
||||
confirm "Publish jmap-email ${VERSION} to real PyPI?"
|
||||
|
||||
say "→ PyPI upload"
|
||||
echo "Get a token at https://pypi.org/manage/account/token/"
|
||||
read_token "PyPI" PYPI_TOKEN
|
||||
|
||||
docker run --rm -t \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
-v "${PKG_DIR}:/pkg" \
|
||||
-w /pkg \
|
||||
-e HOME=/tmp \
|
||||
-e TWINE_USERNAME=__token__ \
|
||||
-e TWINE_PASSWORD="${PYPI_TOKEN}" \
|
||||
"${PYTHON_IMAGE}" \
|
||||
bash -c '
|
||||
set -eo pipefail
|
||||
pip install --quiet --no-cache-dir --root-user-action=ignore --target /tmp/pip twine
|
||||
export PYTHONPATH=/tmp/pip
|
||||
python -m twine upload dist/*
|
||||
'
|
||||
|
||||
echo
|
||||
ok "jmap-email ${VERSION} released to PyPI"
|
||||
echo " https://pypi.org/project/jmap-email/${VERSION}/"
|
||||
echo
|
||||
echo "Next: tag the release in git when you're ready:"
|
||||
echo " git tag jmap-email-${VERSION} && git push origin jmap-email-${VERSION}"
|
||||
@@ -7,7 +7,7 @@ echo "-----> Running post-frontend script"
|
||||
|
||||
# Move the frontend build to the app root and clean up
|
||||
mkdir -p build/
|
||||
mv src/frontend/out build/frontend-out
|
||||
mv src/frontend/dist build/frontend-out
|
||||
|
||||
mv src/backend/* ./
|
||||
mkdir -p messages_backend && touch messages_backend/__init__.py
|
||||
|
||||
@@ -11,10 +11,10 @@ gunicorn -b :8000 messages.wsgi:application --log-file - &
|
||||
# Start the Caddy server
|
||||
bin/caddy run --config Caddyfile --adapter caddyfile &
|
||||
|
||||
# if the current shell is killed, also terminate all its children
|
||||
trap "pkill SIGTERM -P $$" SIGTERM
|
||||
|
||||
# wait for a single child to finish,
|
||||
wait -n
|
||||
# then kill all the other tasks
|
||||
# if the current shell is killed, also terminate all its children
|
||||
trap "pkill SIGTERM -P $$" SIGTERM
|
||||
|
||||
# wait for a single child to finish,
|
||||
wait -n
|
||||
# then kill all the other tasks
|
||||
pkill -P $$
|
||||
|
||||
+25
-2
@@ -90,6 +90,14 @@ services:
|
||||
volumes:
|
||||
- ./src/backend:/app
|
||||
- ./data/static:/data/static
|
||||
# Dev-only override: live-mount the jmap-email working tree over the
|
||||
# package installed from PyPI so local source edits propagate without a
|
||||
# rebuild. The wheel installed at
|
||||
# ``/venv/lib/${PYTHON_VERSION}/site-packages/jmap_email`` is overlaid
|
||||
# with the working-tree source. Override ``PYTHON_VERSION`` in the
|
||||
# environment when the backend's Python floor moves. Comment this mount
|
||||
# out to run exactly what CI/prod install from PyPI.
|
||||
- ./src/jmap-email/jmap_email:/venv/lib/${PYTHON_VERSION:-python3.14}/site-packages/jmap_email
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request as u; u.urlopen('http://localhost:8000/__heartbeat__/', timeout=1)"]
|
||||
interval: 3s
|
||||
@@ -143,7 +151,7 @@ services:
|
||||
volumes:
|
||||
- ./src/backend:/app
|
||||
build:
|
||||
context: src/backend/
|
||||
context: src/backend
|
||||
target: uv
|
||||
pull_policy: build
|
||||
|
||||
@@ -401,7 +409,7 @@ services:
|
||||
pull_policy: build
|
||||
|
||||
keycloak:
|
||||
image: quay.io/keycloak/keycloak:26.6.1
|
||||
image: quay.io/keycloak/keycloak:26.6.3
|
||||
volumes:
|
||||
- ./src/keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro
|
||||
- ./src/keycloak/themes/dsfr-2.2.1.jar:/opt/keycloak/providers/keycloak-theme.jar:ro
|
||||
@@ -455,5 +463,20 @@ services:
|
||||
redis:
|
||||
condition: service_started
|
||||
|
||||
# Self-contained jmap-email package tests. Zero infrastructure
|
||||
# dependencies (no DB, no opensearch, no redis) — the library has
|
||||
# no runtime deps. Source is mounted for instant feedback during
|
||||
# development.
|
||||
jmap-email-test:
|
||||
profiles:
|
||||
- tools
|
||||
build:
|
||||
context: src/jmap-email
|
||||
command: pytest -q tests/
|
||||
volumes:
|
||||
- ./src/jmap-email/jmap_email:/app/jmap_email
|
||||
- ./src/jmap-email/tests:/app/tests
|
||||
- ./src/jmap-email/pyproject.toml:/app/pyproject.toml
|
||||
|
||||
volumes:
|
||||
objectstorage-data:
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
### Frontend App
|
||||
|
||||
- **Next.js Application**: React-based SPA with TypeScript
|
||||
- **React Application**: React-based SPA with TypeScript, Vite, Tanstack Router and React Query
|
||||
- **Auto-generated API Client**: Generated from OpenAPI schema using Orval
|
||||
- **Multi-panel Interface**: Mailbox panel, thread list, and message view
|
||||
- **Real-time Updates**: Using TanStack Query for efficient state management
|
||||
|
||||
+16
-1
@@ -93,7 +93,7 @@ The application uses a new environment file structure with `.defaults` and `.loc
|
||||
| `MTA_OUT_RELAY_PASSWORD` | `pass` | Outbound SMTP password for relay mode | Optional |
|
||||
| `MTA_OUT_DIRECT_PROXIES` | `[]` | List of SOCKS proxy URLs (randomly chosen when non-empty; used in direct mode) | Optional |
|
||||
| `MTA_OUT_DIRECT_PORT` | `25` | TCP port for direct mode on remote MX servers | Optional |
|
||||
| `MTA_OUT_SMTP_TLS_SECURITY_LEVEL` | `may` | SMTP TLS security level ("none", "may") | Optional |
|
||||
| `MTA_OUT_SMTP_TLS_SECURITY_LEVEL` | `may` | SMTP TLS security level: `none`, `may` (opportunistic, no cert check, matches Postfix), or `secure` (mandatory TLS + CA chain + hostname check). Applied to both direct and relay modes — set to `secure` when running against a controlled relay with a valid cert. | Optional |
|
||||
| `MDA_API_SECRET` | `my-shared-secret-mda` | Shared secret for MDA API | Required |
|
||||
| `MDA_API_BASE_URL` | `http://backend-dev:8000/api/v1.0/` | Base URL for MDA API | Dev |
|
||||
|
||||
@@ -208,6 +208,7 @@ blob stays in PG.
|
||||
| `OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION` | `True` | Use email as fallback identifier | Optional |
|
||||
| `OIDC_ALLOW_DUPLICATE_EMAILS` | `False` | Allow duplicate emails (⚠️ Security risk) | Optional |
|
||||
| `OIDC_AUTH_REQUEST_EXTRA_PARAMS` | `{"acr_values": "eidas1"}` | Extra parameters for auth requests | Optional |
|
||||
| `OIDC_AUTH_REQUEST_FORWARDED_PARAMS` | `["login_hint"]` | Forwarded parameters for auth requests | Optional |
|
||||
|
||||
### User Mapping (⚠️ DEPRECATED)
|
||||
_Those settings are deprecated and will be removed in the future._
|
||||
@@ -247,6 +248,20 @@ _Those settings are deprecated and will be removed in the future._
|
||||
| `NEXT_PUBLIC_SENTRY_DSN` | None | Sentry DSN for error tracking | Optional |
|
||||
| `NEXT_PUBLIC_SENTRY_ENVIRONMENT` | None | Sentry environment for error tracking | Optional ('production', 'development', 'staging') |
|
||||
|
||||
### Selfcheck
|
||||
|
||||
End-to-end mail delivery probe — see [selfcheck.md](selfcheck.md) for details.
|
||||
|
||||
| Variable | Default | Description | Required |
|
||||
|----------|---------|-------------|----------|
|
||||
| `MESSAGES_SELFCHECK_FROM` | None | Email address the selfcheck sends from. Leave unset to disable the selfcheck. | Optional |
|
||||
| `MESSAGES_SELFCHECK_TO` | None | Email address the selfcheck sends to. Leave unset to disable the selfcheck. | Optional |
|
||||
| `MESSAGES_SELFCHECK_SECRET` | `self-check-secret-for-dev` | Secret string embedded in the test message body | Optional |
|
||||
| `MESSAGES_SELFCHECK_INTERVAL` | `600` | Interval between selfcheck runs, in seconds | Optional |
|
||||
| `MESSAGES_SELFCHECK_TIMEOUT` | `60` | Timeout for message reception, in seconds | Optional |
|
||||
| `MESSAGES_SELFCHECK_WEBHOOK_URL` | None | Webhook URL POSTed on each successful selfcheck (updown.io-compatible heartbeat) | Optional |
|
||||
| `MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG` | None | Sentry cron monitor slug. When set (with `SENTRY_DSN`), each run is reported as a Sentry check-in. | Optional |
|
||||
|
||||
### Logging
|
||||
|
||||
| Variable | Default | Description | Required |
|
||||
|
||||
@@ -54,12 +54,12 @@ Example of a valid JSON Schema:
|
||||
"minLength": 3,
|
||||
"x-i18n": {
|
||||
"title": {
|
||||
"fr": "Fonction",
|
||||
"en": "Job title"
|
||||
"fr-Fr": "Fonction",
|
||||
"en-US": "Job title"
|
||||
},
|
||||
"description": {
|
||||
"fr": "Le nom de la fonction de l'utilisateur",
|
||||
"en": "The job name of the user"
|
||||
"fr-Fr": "Le nom de la fonction de l'utilisateur",
|
||||
"en-US": "The job name of the user"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -70,11 +70,11 @@ Example of a valid JSON Schema:
|
||||
"description": "Whether the user is elected",
|
||||
"x-i18n": {
|
||||
"title": {
|
||||
"fr": "Est élu",
|
||||
"en": "Is elected"
|
||||
"fr-Fr": "Est élu",
|
||||
"en-US": "Is elected"
|
||||
},
|
||||
"description": {
|
||||
"fr": "Indique si l'utilisateur est élu"
|
||||
"fr-Fr": "Indique si l'utilisateur est élu",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,10 @@ Optionally, to enable uptime alerting via a selfcheck webhook:
|
||||
|
||||
- `MESSAGES_SELFCHECK_WEBHOOK_URL`: URL of the selfcheck webhook endpoint (default: `None` - disabled)
|
||||
|
||||
Optionally, to report selfcheck runs to [Sentry Crons](https://docs.sentry.io/product/crons/):
|
||||
|
||||
- `MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG`: Slug of the Sentry cron monitor (default: `None` - disabled). Requires `SENTRY_DSN` to also be set.
|
||||
|
||||
## Usage
|
||||
|
||||
### Manual Execution
|
||||
@@ -95,6 +99,16 @@ The POST body includes timing data:
|
||||
{"send_time": 0.15, "reception_time": 2.34}
|
||||
```
|
||||
|
||||
### Sentry Crons
|
||||
|
||||
When `MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG` is configured (and `SENTRY_DSN` is set), each selfcheck run is reported to [Sentry Crons](https://docs.sentry.io/product/crons/):
|
||||
|
||||
- An `in_progress` check-in is opened before the test message is sent.
|
||||
- The check-in is closed with status `ok` on success or `error` on failure.
|
||||
- On success, the reported `duration` is `send_time + reception_time`, excluding the post-run cleanup pause.
|
||||
|
||||
Configure the monitor schedule (interval and grace period) in the Sentry UI to match `MESSAGES_SELFCHECK_INTERVAL`. Runs skipped because `MESSAGES_SELFCHECK_FROM` or `MESSAGES_SELFCHECK_TO` is empty do not produce a check-in.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
- The selfcheck uses dedicated test mailboxes that are separate from user data
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
NEXT_PUBLIC_API_ORIGIN=http://localhost:8901
|
||||
NEXT_TELEMETRY_DISABLED=1
|
||||
NEXT_PUBLIC_FEEDBACK_WIDGET_API_URL=
|
||||
NEXT_PUBLIC_FEEDBACK_WIDGET_PATH=
|
||||
NEXT_PUBLIC_FEEDBACK_WIDGET_CHANNEL=
|
||||
|
||||
@@ -18,9 +18,8 @@ WORKDIR /app
|
||||
FROM base AS uv
|
||||
|
||||
# Pin uv by SHA256 digest for supply chain security.
|
||||
# Verify with: gh attestation verify --owner astral-sh oci://ghcr.io/astral-sh/uv:0.11.10
|
||||
# SLSA provenance: https://github.com/astral-sh/uv/attestations/26524427
|
||||
COPY --from=ghcr.io/astral-sh/uv@sha256:bca7f6959666f3524e0c42129f9d8bbcfb0c180d847f5187846b98ff06125ead /uv /uvx /bin/
|
||||
# Verify with: gh attestation verify --owner astral-sh oci://ghcr.io/astral-sh/uv:0.11.19
|
||||
COPY --from=ghcr.io/astral-sh/uv@sha256:b46b03ddfcfbf8f547af7e9eaefdf8a39c8cebcba7c98858d3162bd28cf536f6 /uv /uvx /bin/
|
||||
|
||||
RUN <<EOR
|
||||
apt-get update
|
||||
@@ -43,7 +42,7 @@ ENV UV_PROJECT_ENVIRONMENT=/venv
|
||||
|
||||
# Install Python via uv — integrity verified against SHA256 checksums embedded in the uv binary.
|
||||
# Uses python-build-standalone: most C deps (ssl, ffi, sqlite, zlib, lzma, bz2) are statically linked.
|
||||
RUN uv python install 3.14.4
|
||||
RUN uv python install 3.14.5
|
||||
|
||||
# ---- Production dependencies ----
|
||||
FROM uv AS base-with-deps
|
||||
@@ -79,7 +78,7 @@ EOR
|
||||
FROM uv AS python-runtime
|
||||
RUN <<EOR
|
||||
set -e
|
||||
PYDIR=$(dirname $(dirname $(uv python find 3.14.4)))
|
||||
PYDIR=$(dirname $(dirname $(uv python find 3.14.5)))
|
||||
rm -rf \
|
||||
$PYDIR/bin/idle* $PYDIR/bin/pip* $PYDIR/bin/pydoc* $PYDIR/bin/*-config \
|
||||
$PYDIR/include $PYDIR/share \
|
||||
|
||||
@@ -1103,7 +1103,6 @@ class AttachmentAdmin(admin.ModelAdmin):
|
||||
search_fields = ("name", "mailbox__local_part", "mailbox__domain__name")
|
||||
autocomplete_fields = ("mailbox",)
|
||||
raw_id_fields = ("blob", "message")
|
||||
exclude = ("_deprecated_messages",)
|
||||
|
||||
|
||||
class AttachmentInline(admin.TabularInline):
|
||||
@@ -1113,7 +1112,6 @@ class AttachmentInline(admin.TabularInline):
|
||||
fk_name = "message"
|
||||
raw_id_fields = ("blob",)
|
||||
autocomplete_fields = ("mailbox",)
|
||||
exclude = ("_deprecated_messages",)
|
||||
extra = 0
|
||||
|
||||
|
||||
@@ -1418,7 +1416,6 @@ class BlobAdmin(admin.ModelAdmin):
|
||||
"created_at",
|
||||
"updated_at",
|
||||
)
|
||||
exclude = ("_deprecated_mailbox", "_deprecated_maildomain")
|
||||
change_form_template = "admin/core/blob/change_form.html"
|
||||
|
||||
def get_queryset(self, request):
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -205,78 +205,6 @@ class IsAllowedToAccess(IsAuthenticated):
|
||||
return False
|
||||
|
||||
|
||||
class IsAllowedToCreateMessage(IsAuthenticated):
|
||||
"""Permission class for access to create a message."""
|
||||
|
||||
def has_permission(self, request, view):
|
||||
"""Check if user is allowed to create a message."""
|
||||
|
||||
if not IsAuthenticated.has_permission(self, request, view):
|
||||
return False
|
||||
|
||||
# a sender mailbox is required to create/send a message
|
||||
sender_id = request.data.get("senderId")
|
||||
parent_id = request.data.get("parentId")
|
||||
if not sender_id:
|
||||
return False
|
||||
|
||||
# get mailbox instance from sender id
|
||||
try:
|
||||
# Store mailbox on the view for later use (e.g., in the view logic)
|
||||
view.mailbox = models.Mailbox.objects.get(id=sender_id)
|
||||
except models.Mailbox.DoesNotExist:
|
||||
return False # Invalid senderId
|
||||
|
||||
# Check if user has required role on the sender Mailbox
|
||||
has_edit_role = view.mailbox.accesses.filter(
|
||||
user=request.user,
|
||||
role__in=enums.MAILBOX_ROLES_CAN_EDIT,
|
||||
).exists()
|
||||
|
||||
# if user does not have edit role with this sender mailbox, return False
|
||||
if not has_edit_role:
|
||||
return False
|
||||
|
||||
# --- Additional check for replies ---
|
||||
# If creating a reply (parentId is provided), check access to the parent thread
|
||||
if parent_id:
|
||||
try:
|
||||
parent_message = models.Message.objects.select_related("thread").get(
|
||||
id=parent_id
|
||||
)
|
||||
# Check if the user has access to the thread they are replying to
|
||||
if models.ThreadAccess.objects.filter(
|
||||
thread=parent_message.thread,
|
||||
mailbox=view.mailbox,
|
||||
role=enums.ThreadAccessRoleChoices.EDITOR,
|
||||
).exists():
|
||||
return True
|
||||
except models.Message.DoesNotExist:
|
||||
return False # Treat invalid parentId as permission failure
|
||||
|
||||
# --- Additional check for updating existing draft ---
|
||||
# If updating (messageId is provided), check access to the draft's thread
|
||||
message_id = request.data.get("messageId")
|
||||
if message_id and request.method == "PUT": # Check only needed for updates
|
||||
try:
|
||||
draft_message = models.Message.objects.select_related("thread").get(
|
||||
id=message_id, is_draft=True
|
||||
)
|
||||
# Check if the user has access to the thread of the draft being updated
|
||||
if not models.ThreadAccess.objects.filter(
|
||||
thread=draft_message.thread,
|
||||
mailbox=view.mailbox,
|
||||
role=enums.ThreadAccessRoleChoices.EDITOR,
|
||||
).exists():
|
||||
return False
|
||||
except models.Message.DoesNotExist:
|
||||
# Let the view handle invalid messageId
|
||||
return False # Treat invalid messageId as permission failure
|
||||
|
||||
# If all checks pass
|
||||
return True
|
||||
|
||||
|
||||
def _user_can_manage_thread_access(user, thread_id):
|
||||
"""True if ``user`` has full edit rights on the thread.
|
||||
|
||||
@@ -447,6 +375,40 @@ class IsMailboxAdmin(permissions.BasePermission):
|
||||
return is_domain_admin
|
||||
|
||||
|
||||
class IsMailboxAdminObject(permissions.BasePermission):
|
||||
"""Object-level permission granting access on a Mailbox instance to its
|
||||
admins (MailboxAccess role ADMIN), its domain admins, or superusers.
|
||||
|
||||
Unlike :class:`IsMailboxAdmin`, this works on the resolved Mailbox object
|
||||
(e.g. ``/mailboxes/{pk}/``) rather than a nested ``mailbox_id`` URL kwarg.
|
||||
"""
|
||||
|
||||
message = "You do not have administrative rights for this mailbox or its domain."
|
||||
|
||||
def has_object_permission(self, request, view, obj):
|
||||
user = request.user
|
||||
if not user or not user.is_authenticated:
|
||||
return False
|
||||
|
||||
if user.is_superuser:
|
||||
return True
|
||||
|
||||
is_mailbox_admin = models.MailboxAccess.objects.filter(
|
||||
user=user, mailbox=obj, role=models.MailboxRoleChoices.ADMIN
|
||||
).exists()
|
||||
if is_mailbox_admin:
|
||||
return True
|
||||
|
||||
if obj.domain:
|
||||
return models.MailDomainAccess.objects.filter(
|
||||
user=user,
|
||||
maildomain=obj.domain,
|
||||
role=models.MailDomainAccessRoleChoices.ADMIN,
|
||||
).exists()
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class HasChannelScope(permissions.BasePermission):
|
||||
"""Scope-based permission for service calls authenticated as a Channel.
|
||||
|
||||
@@ -691,3 +653,22 @@ class HasAccessToMailbox(IsAuthenticated):
|
||||
return models.MailboxAccess.objects.filter(
|
||||
user=request.user, mailbox=view.kwargs.get("mailbox_id")
|
||||
).exists()
|
||||
|
||||
|
||||
class HasWriteAccessToMailbox(IsAuthenticated):
|
||||
"""Allows access only to users with an editor-or-above role on the mailbox.
|
||||
|
||||
Use for state-changing endpoints whose effect is observable beyond the
|
||||
mailbox itself (e.g. writing to the mailbox's CalDAV calendar, which a
|
||||
VIEWER access shouldn't be able to do).
|
||||
"""
|
||||
|
||||
def has_permission(self, request, view):
|
||||
if not super().has_permission(request, view):
|
||||
return False
|
||||
|
||||
return models.MailboxAccess.objects.filter(
|
||||
user=request.user,
|
||||
mailbox=view.kwargs.get("mailbox_id"),
|
||||
role__in=enums.MAILBOX_ROLES_CAN_EDIT,
|
||||
).exists()
|
||||
|
||||
@@ -15,7 +15,7 @@ from rest_framework import serializers
|
||||
from rest_framework.exceptions import PermissionDenied
|
||||
|
||||
from core import enums, models
|
||||
from core.mda.rfc5322 import extract_base64_images_from_html
|
||||
from core.mda.inline_images import extract_inline_images_html
|
||||
from core.services.blob_gc import schedule_for_gc
|
||||
from core.services.identity import keycloak as keycloak_service
|
||||
|
||||
@@ -220,6 +220,23 @@ class IntegerChoicesField(serializers.ChoiceField):
|
||||
super().fail(key, **kwargs)
|
||||
|
||||
|
||||
def nullable_choices_schema(choices_class):
|
||||
"""Nullable enum schema for a ``SerializerMethodField`` that may return
|
||||
``None``.
|
||||
|
||||
Reuse the shared ``{choices_class.__name__}`` component (registered by the
|
||||
non-null :class:`IntegerChoicesField` usages) and apply the nullability
|
||||
locally via the ``allOf`` wrapper. Baking ``nullable`` into the component
|
||||
itself would collide with those non-null usages; an inline enum dict would
|
||||
instead make drf-spectacular extract a redundant ``…Enum`` component. This
|
||||
mirrors the shape drf-spectacular emits natively for a nullable ``$ref``.
|
||||
"""
|
||||
return {
|
||||
"allOf": [{"$ref": f"#/components/schemas/{choices_class.__name__}"}],
|
||||
"nullable": True,
|
||||
}
|
||||
|
||||
|
||||
class AbilitiesModelSerializer(serializers.ModelSerializer):
|
||||
"""
|
||||
A ModelSerializer that takes an additional `exclude` argument that
|
||||
@@ -342,6 +359,8 @@ class MailboxSerializer(AbilitiesModelSerializer):
|
||||
"""Serialize mailboxes."""
|
||||
|
||||
email = serializers.SerializerMethodField(read_only=True)
|
||||
name = serializers.SerializerMethodField(read_only=True)
|
||||
domain_id = serializers.UUIDField(read_only=True)
|
||||
role = serializers.SerializerMethodField(read_only=True)
|
||||
count_unread_threads = serializers.SerializerMethodField(read_only=True)
|
||||
count_threads = serializers.SerializerMethodField(read_only=True)
|
||||
@@ -355,6 +374,8 @@ class MailboxSerializer(AbilitiesModelSerializer):
|
||||
fields = [
|
||||
"id",
|
||||
"email",
|
||||
"name",
|
||||
"domain_id",
|
||||
"is_identity",
|
||||
"is_shared",
|
||||
"role",
|
||||
@@ -370,7 +391,13 @@ class MailboxSerializer(AbilitiesModelSerializer):
|
||||
"""Return the email of the mailbox."""
|
||||
return str(instance)
|
||||
|
||||
@extend_schema_field(IntegerChoicesField(choices_class=models.MailboxRoleChoices))
|
||||
def get_name(self, instance) -> str | None:
|
||||
"""Return the display name of the mailbox (its contact name)."""
|
||||
if instance.contact:
|
||||
return instance.contact.name
|
||||
return None
|
||||
|
||||
@extend_schema_field(nullable_choices_schema(models.MailboxRoleChoices))
|
||||
def get_role(self, instance):
|
||||
"""Return the allowed actions of the logged-in user on the instance."""
|
||||
# Use the annotated user_role field
|
||||
@@ -497,6 +524,35 @@ class MailboxSerializer(AbilitiesModelSerializer):
|
||||
return super().get_abilities(instance)
|
||||
|
||||
|
||||
class MailboxNameUpdateSerializer(serializers.Serializer):
|
||||
"""Validate and apply a mailbox display-name update (its contact name)."""
|
||||
|
||||
name = serializers.CharField(max_length=255)
|
||||
|
||||
def validate_name(self, value):
|
||||
"""Strip surrounding whitespace and reject whitespace-only names, which
|
||||
would otherwise be stored (and sent in the ``From`` header) verbatim."""
|
||||
value = value.strip()
|
||||
if not value:
|
||||
raise serializers.ValidationError("Name cannot be blank.")
|
||||
return value
|
||||
|
||||
def create(self, validated_data):
|
||||
"""Do not allow creating instances from this serializer."""
|
||||
raise RuntimeError(f"{self.__class__.__name__} does not support create method")
|
||||
|
||||
def update(self, instance, validated_data):
|
||||
"""Persist the new display name through the mailbox helper.
|
||||
|
||||
The PATCH is partial: an absent ``name`` is a no-op, so callers may send
|
||||
an empty body without error. This matches the optional ``name`` in the
|
||||
generated OpenAPI request schema.
|
||||
"""
|
||||
if "name" in validated_data:
|
||||
instance.set_display_name(validated_data["name"])
|
||||
return instance
|
||||
|
||||
|
||||
class MailboxLightSerializer(serializers.ModelSerializer):
|
||||
"""Serializer for mailbox details in thread access."""
|
||||
|
||||
@@ -851,9 +907,7 @@ class ThreadSerializer(serializers.ModelSerializer):
|
||||
cached = instance.messages.order_by("created_at")
|
||||
return [str(message.id) for message in cached]
|
||||
|
||||
@extend_schema_field(
|
||||
IntegerChoicesField(choices_class=models.ThreadAccessRoleChoices)
|
||||
)
|
||||
@extend_schema_field(nullable_choices_schema(models.ThreadAccessRoleChoices))
|
||||
def get_user_role(self, instance):
|
||||
"""Get current user's role for this thread, scoped to the context mailbox.
|
||||
|
||||
@@ -1116,10 +1170,11 @@ class MessageSerializer(serializers.ModelSerializer):
|
||||
stripped_attachments.append(
|
||||
{
|
||||
"blobId": f"msg_{instance.id}_{index}",
|
||||
"name": attachment["name"],
|
||||
"name": attachment.get("name") or "unnamed",
|
||||
"size": attachment["size"],
|
||||
"type": attachment["type"],
|
||||
"cid": attachment.get("cid"),
|
||||
"sha256": attachment.get("sha256"),
|
||||
}
|
||||
)
|
||||
return stripped_attachments
|
||||
@@ -1404,6 +1459,21 @@ class MailDomainAdminSerializer(AbilitiesModelSerializer):
|
||||
"""Return the abilities for the mail domain."""
|
||||
return super().get_abilities(instance)
|
||||
|
||||
@extend_schema_field(
|
||||
{
|
||||
"type": "array",
|
||||
"nullable": True,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"target": {"type": "string"},
|
||||
"type": {"type": "string"},
|
||||
"value": {"type": "string"},
|
||||
},
|
||||
"required": ["target", "type", "value"],
|
||||
},
|
||||
}
|
||||
)
|
||||
def get_expected_dns_records(self, instance):
|
||||
"""Return the expected DNS records for the mail domain, only in detail views."""
|
||||
|
||||
@@ -1512,7 +1582,9 @@ class MailboxAdminSerializer(serializers.ModelSerializer):
|
||||
many=True, read_only=True
|
||||
) # accesses is the related_name
|
||||
can_reset_password = serializers.BooleanField(read_only=True)
|
||||
contact = ContactSerializer(read_only=True)
|
||||
# ``Mailbox.contact`` is ``SET_NULL, null=True`` — an alias mailbox (or one
|
||||
# whose contact was deleted) has none, so the nested field must be nullable.
|
||||
contact = ContactSerializer(read_only=True, allow_null=True)
|
||||
alias_of = serializers.PrimaryKeyRelatedField(
|
||||
required=False, allow_null=True, queryset=models.Mailbox.objects.none()
|
||||
)
|
||||
@@ -1606,17 +1678,13 @@ class MailboxAdminSerializer(serializers.ModelSerializer):
|
||||
"Domain is required in serializer context."
|
||||
)
|
||||
|
||||
domain = self.context.get("domain")
|
||||
metadata = self.context.get("metadata", {})
|
||||
if metadata.get("type") == "personal" and not domain.identity_sync:
|
||||
raise serializers.ValidationError(
|
||||
{
|
||||
"identity_sync": (
|
||||
"Personal mailboxes cannot be created when "
|
||||
"identity synchronization is disabled."
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
# Personal mailboxes can be created even when identity synchronization is
|
||||
# disabled: this lets admins pre-create mailboxes for users who connect
|
||||
# through a third-party (non-synced) identity provider. No password is
|
||||
# provisioned in that case (see Mailbox.can_reset_password), but the
|
||||
# mailbox can receive emails straight away.
|
||||
|
||||
if metadata.get("type") == "personal":
|
||||
local_part = attrs.get("local_part", "")
|
||||
@@ -1713,11 +1781,10 @@ class MailboxAdminSerializer(serializers.ModelSerializer):
|
||||
|
||||
if instance.is_identity is True:
|
||||
user_updated_fields = {}
|
||||
contact_updated_fields = {}
|
||||
display_name = metadata.get("full_name")
|
||||
|
||||
if full_name := metadata.get("full_name"):
|
||||
user_updated_fields["full_name"] = full_name
|
||||
contact_updated_fields["name"] = full_name
|
||||
if display_name:
|
||||
user_updated_fields["full_name"] = display_name
|
||||
if custom_attributes := metadata.get("custom_attributes"):
|
||||
user_updated_fields["custom_attributes"] = custom_attributes
|
||||
|
||||
@@ -1732,21 +1799,12 @@ class MailboxAdminSerializer(serializers.ModelSerializer):
|
||||
owner.save(update_fields=list(user_updated_fields.keys()))
|
||||
updated = True
|
||||
|
||||
if contact_updated_fields:
|
||||
contact = models.Contact.objects.filter(pk=instance.contact_id)
|
||||
contact.update(**contact_updated_fields)
|
||||
updated = True
|
||||
|
||||
else:
|
||||
contact_updated_fields = {}
|
||||
display_name = metadata.get("name")
|
||||
|
||||
if name := metadata.get("name"):
|
||||
contact_updated_fields["name"] = name
|
||||
|
||||
if contact_updated_fields:
|
||||
contact = models.Contact.objects.filter(pk=instance.contact_id)
|
||||
contact.update(**contact_updated_fields)
|
||||
updated = True
|
||||
if display_name:
|
||||
instance.set_display_name(display_name)
|
||||
updated = True
|
||||
|
||||
if updated:
|
||||
instance.refresh_from_db()
|
||||
@@ -1965,6 +2023,10 @@ class ChannelSerializer(CreateOnlyFieldsMixin, serializers.ModelSerializer):
|
||||
# generators write directly to ``encrypted_settings`` instead.
|
||||
RESERVED_SETTINGS_KEYS = {
|
||||
enums.ChannelTypes.API_KEY: ["api_key_hashes"],
|
||||
# CalDAV credentials must live in ``encrypted_settings``, never in
|
||||
# the plaintext ``settings`` JSONField — a DB read would otherwise
|
||||
# surface every user's CalDAV password.
|
||||
enums.ChannelTypes.CALDAV: ["username", "password"],
|
||||
}
|
||||
|
||||
def create(self, validated_data):
|
||||
@@ -2334,7 +2396,7 @@ class MessageTemplateSerializer(serializers.ModelSerializer):
|
||||
attrs.pop("text_body")
|
||||
attrs.pop("raw_body")
|
||||
else:
|
||||
_html, images = extract_base64_images_from_html(attrs["html_body"])
|
||||
_html, images = extract_inline_images_html(attrs["html_body"])
|
||||
total_image_size = 0
|
||||
for image in images:
|
||||
total_image_size += image["size"]
|
||||
@@ -2553,3 +2615,51 @@ class ProvisioningMailDomainSerializer(serializers.Serializer):
|
||||
|
||||
def update(self, instance, validated_data):
|
||||
"""This serializer is only used to validate the data, not to create or update."""
|
||||
|
||||
|
||||
class ThreadBulkDeleteRequestSerializer(serializers.Serializer):
|
||||
"""Payload for the bulk-delete endpoint: a scope and the threads/messages
|
||||
to permanently delete."""
|
||||
|
||||
# Scopes accepted by the bulk-delete endpoint, mapping each to the queryset
|
||||
# filter selecting the messages whose rows get permanently removed. Only
|
||||
# "draft" is exposed for now: trashed deletion is intentionally not offered
|
||||
# until the product behavior for trashed messages is decided.
|
||||
BULK_DELETE_SCOPE_FILTERS = {"draft": {"is_draft": True}}
|
||||
BULK_DELETE_SCOPES = list(BULK_DELETE_SCOPE_FILTERS)
|
||||
|
||||
scope = serializers.ChoiceField(
|
||||
choices=BULK_DELETE_SCOPES,
|
||||
help_text=(
|
||||
"Which messages to permanently delete. Only 'draft' "
|
||||
"(draft messages) is supported."
|
||||
),
|
||||
)
|
||||
thread_ids = serializers.ListField(
|
||||
child=serializers.UUIDField(),
|
||||
required=False,
|
||||
allow_empty=True,
|
||||
default=list,
|
||||
help_text="Threads whose scope-matching messages should be deleted.",
|
||||
)
|
||||
message_ids = serializers.ListField(
|
||||
child=serializers.UUIDField(),
|
||||
required=False,
|
||||
allow_empty=True,
|
||||
default=list,
|
||||
help_text="Specific messages to delete (still scope-filtered).",
|
||||
)
|
||||
|
||||
def validate(self, attrs):
|
||||
"""Require at least one target list to be non-empty."""
|
||||
if not attrs["thread_ids"] and not attrs["message_ids"]:
|
||||
raise serializers.ValidationError(
|
||||
"Provide at least one of thread_ids or message_ids."
|
||||
)
|
||||
return attrs
|
||||
|
||||
def create(self, validated_data):
|
||||
"""This serializer is only used to validate the data, not to create or update."""
|
||||
|
||||
def update(self, instance, validated_data):
|
||||
"""This serializer is only used to validate the data, not to create or update."""
|
||||
|
||||
@@ -3,15 +3,21 @@
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ValidationError as DjangoValidationError
|
||||
from django.http import HttpResponse
|
||||
from django.utils.decorators import method_decorator
|
||||
from django.utils.http import content_disposition_header
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
|
||||
import magic
|
||||
from drf_spectacular.types import OpenApiTypes
|
||||
from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema
|
||||
from rest_framework import status
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.exceptions import NotFound
|
||||
from rest_framework.exceptions import (
|
||||
APIException,
|
||||
NotFound,
|
||||
ParseError,
|
||||
PermissionDenied,
|
||||
)
|
||||
from rest_framework.parsers import MultiPartParser
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.viewsets import ViewSet
|
||||
@@ -20,6 +26,12 @@ from core import enums, models
|
||||
from core.api import permissions, utils
|
||||
from core.services.blob_gc import upload_and_reserve_blob
|
||||
|
||||
# Number of leading bytes inspected by python-magic on the preview endpoint.
|
||||
# Every format we allowlist is identified by a signature in its first few hundred
|
||||
# bytes, so 2 KiB is a comfortable margin — not a tight bound. We cap the slice
|
||||
# only to avoid handing magic the whole payload before deciding to refuse.
|
||||
_PREVIEW_MAGIC_SNIFF_BYTES = 2048
|
||||
|
||||
# Define logger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -90,7 +102,6 @@ class BlobViewSet(ViewSet):
|
||||
},
|
||||
tags=["blob"],
|
||||
)
|
||||
@method_decorator(csrf_exempt)
|
||||
@action(detail=False, methods=["post"], url_path="upload/(?P<mailbox_id>[^/.]+)")
|
||||
def upload(self, request, mailbox_id=None):
|
||||
"""
|
||||
@@ -167,6 +178,58 @@ class BlobViewSet(ViewSet):
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
def _resolve_blob_source(self, pk, user):
|
||||
"""Resolve a blob to its bytes and metadata.
|
||||
|
||||
`msg_*` IDs are served from the parsed message attachment cache
|
||||
|
||||
Returns:
|
||||
A dict with keys `content` (bytes), `declared_type` (str),
|
||||
`filename` (str), `size` (int).
|
||||
|
||||
Raises:
|
||||
ParseError: malformed `msg_*` ID.
|
||||
NotFound: `msg_*` ID points at a missing attachment.
|
||||
PermissionDenied: blob doesn't exist or user has no access
|
||||
"""
|
||||
if pk.startswith("msg_"):
|
||||
try:
|
||||
attachment = utils.get_attachment_from_blob_id(pk, user)
|
||||
except ValueError as e:
|
||||
raise ParseError("Invalid blob ID") from e
|
||||
except models.Blob.DoesNotExist as e:
|
||||
raise NotFound("Blob not found") from e
|
||||
return {
|
||||
"content": attachment["content"],
|
||||
"declared_type": attachment["type"],
|
||||
"filename": attachment["name"],
|
||||
"size": attachment["size"],
|
||||
}
|
||||
|
||||
try:
|
||||
blob = models.Blob.objects.get(id=pk)
|
||||
except DjangoValidationError as e:
|
||||
# Non-UUID ``pk`` reaches the ORM as a ValidationError; surface
|
||||
# it as a 400 instead of falling through to the generic 500.
|
||||
raise ParseError("Invalid blob ID") from e
|
||||
except models.Blob.DoesNotExist as e:
|
||||
raise PermissionDenied(
|
||||
"You do not have permission to access this blob"
|
||||
) from e
|
||||
|
||||
if not models.Blob.objects.user_can_access(user, blob.id):
|
||||
raise PermissionDenied("You do not have permission to access this blob")
|
||||
attachment_row = models.Attachment.objects.filter(blob=blob).first()
|
||||
|
||||
return {
|
||||
"content": blob.get_content(),
|
||||
"declared_type": blob.content_type,
|
||||
"filename": (
|
||||
attachment_row.name if attachment_row else f"blob-{blob.id}.bin"
|
||||
),
|
||||
"size": blob.size,
|
||||
}
|
||||
|
||||
@action(detail=True, methods=["get"])
|
||||
def download(self, request, pk=None):
|
||||
"""
|
||||
@@ -176,71 +239,23 @@ class BlobViewSet(ViewSet):
|
||||
by checking if the user has access to any mailbox that owns this blob.
|
||||
"""
|
||||
try:
|
||||
# Blob IDs in the form msg_[message_id]_[attachment_number] are looked up
|
||||
# directly in the message's attachments.
|
||||
if pk.startswith("msg_"):
|
||||
try:
|
||||
attachment = utils.get_attachment_from_blob_id(pk, request.user)
|
||||
except ValueError as e:
|
||||
return Response(
|
||||
status=status.HTTP_400_BAD_REQUEST, data={"error": str(e)}
|
||||
)
|
||||
except models.Blob.DoesNotExist as e:
|
||||
return Response(
|
||||
status=status.HTTP_404_NOT_FOUND, data={"error": str(e)}
|
||||
)
|
||||
|
||||
# Create response with decompressed content
|
||||
response = HttpResponse(
|
||||
attachment["content"], content_type=attachment["type"]
|
||||
)
|
||||
|
||||
# Add appropriate headers for download
|
||||
response["Content-Disposition"] = content_disposition_header(
|
||||
True, attachment["name"]
|
||||
)
|
||||
response["Content-Length"] = attachment["size"]
|
||||
# Enable browser caching for 30 days (inline images benefit from this)
|
||||
response["Cache-Control"] = "private, max-age=2592000"
|
||||
|
||||
else:
|
||||
# Get the blob
|
||||
blob = models.Blob.objects.get(id=pk)
|
||||
|
||||
# Authz: walk the reference graph (Attachment, Message,
|
||||
# MessageTemplate) plus any active upload reservation.
|
||||
# See ``BlobManager.user_can_access`` for the union.
|
||||
if not models.Blob.objects.user_can_access(request.user, blob.id):
|
||||
return Response(
|
||||
{"error": "You do not have permission to download this blob"},
|
||||
status=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
|
||||
# Get the first attachment name to use as filename (if available)
|
||||
attachment = models.Attachment.objects.filter(blob=blob).first()
|
||||
filename = attachment.name if attachment else f"blob-{blob.id}.bin"
|
||||
|
||||
# Create response with decompressed content
|
||||
response = HttpResponse(
|
||||
blob.get_content(), content_type=blob.content_type
|
||||
)
|
||||
|
||||
# Add appropriate headers for download
|
||||
response["Content-Disposition"] = content_disposition_header(
|
||||
True, filename
|
||||
)
|
||||
response["Content-Length"] = blob.size
|
||||
# Enable browser caching for 30 days (inline images benefit from this)
|
||||
response["Cache-Control"] = "private, max-age=2592000"
|
||||
source = self._resolve_blob_source(pk, request.user)
|
||||
|
||||
response = HttpResponse(
|
||||
source["content"], content_type=source["declared_type"]
|
||||
)
|
||||
response["Content-Disposition"] = content_disposition_header(
|
||||
True, source["filename"]
|
||||
)
|
||||
response["Content-Length"] = source["size"]
|
||||
# Enable browser caching for 30 days (inline images benefit from this)
|
||||
response["Cache-Control"] = "private, max-age=2592000"
|
||||
return response
|
||||
|
||||
except models.Blob.DoesNotExist:
|
||||
# Same error to hide blob existence
|
||||
return Response(
|
||||
{"error": "You do not have permission to download this blob"},
|
||||
status=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
except APIException:
|
||||
# Let DRF convert ParseError / NotFound / PermissionDenied raised by
|
||||
# ``_resolve_blob_source`` into the proper Response.
|
||||
raise
|
||||
# pylint: disable=broad-exception-caught
|
||||
except Exception as e:
|
||||
logger.exception("Error downloading file: %s", str(e))
|
||||
@@ -248,3 +263,125 @@ class BlobViewSet(ViewSet):
|
||||
{"error": "Error downloading file"},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
@extend_schema(
|
||||
responses={
|
||||
(200, "application/octet-stream"): OpenApiResponse(
|
||||
description=(
|
||||
"Inline preview of the blob. The Content-Type is the MIME "
|
||||
"type detected server-side and is guaranteed to belong to "
|
||||
"``PREVIEWABLE_MIME_TYPES``."
|
||||
),
|
||||
response=OpenApiTypes.BINARY,
|
||||
),
|
||||
400: OpenApiResponse(description="Invalid blob ID"),
|
||||
403: OpenApiResponse(
|
||||
description="Forbidden - User does not have permission to preview this blob"
|
||||
),
|
||||
404: OpenApiResponse(description="Blob not found"),
|
||||
415: OpenApiResponse(
|
||||
description=(
|
||||
"Unsupported media type for inline preview. The detected "
|
||||
"MIME is not in ``PREVIEWABLE_MIME_TYPES`` or does not "
|
||||
"match the declared Content-Type. The response body "
|
||||
"includes a ``code`` field set to either ``suspicious`` "
|
||||
"(declared type was previewable but bytes disagree) or "
|
||||
"``unsupported`` (type is plainly not previewable)."
|
||||
),
|
||||
),
|
||||
500: OpenApiResponse(description="Internal server error"),
|
||||
},
|
||||
tags=["blob"],
|
||||
)
|
||||
@action(detail=True, methods=["get"])
|
||||
def preview(self, request, pk=None):
|
||||
"""
|
||||
Serve a blob inline for the FilePreview viewer.
|
||||
|
||||
Sibling of ``download`` with the same authorization model but two
|
||||
extra guarantees:
|
||||
|
||||
- the response Content-Type is the MIME type detected from the bytes
|
||||
(via ``python-magic``), not the value declared at upload time;
|
||||
- the detected MIME must belong to ``PREVIEWABLE_MIME_TYPES``,
|
||||
otherwise the endpoint refuses with 415.
|
||||
|
||||
Returning 415 (rather than 200 with the raw payload) is the security
|
||||
contract that lets the frontend render the response inline: any byte
|
||||
we send back has been re-classified server-side as one of the safe
|
||||
previewable types.
|
||||
"""
|
||||
try:
|
||||
source = self._resolve_blob_source(pk, request.user)
|
||||
content = source["content"]
|
||||
declared_type = source["declared_type"]
|
||||
|
||||
# Normalize the declared Content-Type (e.g. image/PNG; charset=binary)
|
||||
declared_media_type = declared_type.partition(";")[0].strip().lower()
|
||||
detected_type = magic.from_buffer(
|
||||
content[:_PREVIEW_MAGIC_SNIFF_BYTES], mime=True
|
||||
).lower()
|
||||
|
||||
# A preview is served only when the detected bytes are an
|
||||
# allowlisted type AND match the declared Content-Type. Anything
|
||||
# else is refused — the browser must never render bytes the
|
||||
# uploader lied about. The blob can still be downloaded via
|
||||
# /download/.
|
||||
if (
|
||||
detected_type not in enums.PREVIEWABLE_MIME_TYPES
|
||||
or detected_type != declared_media_type
|
||||
):
|
||||
# When the *declared* type was itself previewable, the bytes
|
||||
# don't back that claim — flag the attachment as suspicious.
|
||||
# Otherwise it's plainly not previewable.
|
||||
suspicious = declared_media_type in enums.PREVIEWABLE_MIME_TYPES
|
||||
code = (
|
||||
enums.PreviewRefusalCode.SUSPICIOUS
|
||||
if suspicious
|
||||
else enums.PreviewRefusalCode.UNSUPPORTED
|
||||
)
|
||||
logger.log(
|
||||
logging.WARNING if suspicious else logging.INFO,
|
||||
"Refused preview for blob %s: declared %s, detected %s (%s)",
|
||||
pk,
|
||||
declared_type,
|
||||
detected_type,
|
||||
code.value,
|
||||
)
|
||||
return Response(
|
||||
{
|
||||
"error": "File type not supported for preview",
|
||||
"code": code.value,
|
||||
},
|
||||
status=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
)
|
||||
|
||||
response = HttpResponse(content, content_type=detected_type)
|
||||
response["Content-Disposition"] = content_disposition_header(
|
||||
False, source["filename"]
|
||||
)
|
||||
response["Content-Length"] = source["size"]
|
||||
response["Cache-Control"] = "private, max-age=2592000"
|
||||
# Defense in depth on top of the global SECURE_CONTENT_TYPE_NOSNIFF.
|
||||
response["X-Content-Type-Options"] = "nosniff"
|
||||
response["Referrer-Policy"] = "no-referrer"
|
||||
# Strict CSP: the response is expected to be loaded only via
|
||||
# <img>, <video>, <audio> or fetch() (PDF.js) — never as a
|
||||
# top-level document with scripts.
|
||||
response["Content-Security-Policy"] = (
|
||||
"default-src 'none'; img-src 'self' blob: data:; "
|
||||
"media-src 'self' blob:; sandbox"
|
||||
)
|
||||
return response
|
||||
|
||||
except APIException:
|
||||
# Let DRF convert ParseError / NotFound / PermissionDenied raised by
|
||||
# ``_resolve_blob_source`` into the proper Response.
|
||||
raise
|
||||
# pylint: disable=broad-exception-caught
|
||||
except Exception as e:
|
||||
logger.exception("Error previewing file: %s", str(e))
|
||||
return Response(
|
||||
{"error": "Error previewing file"},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,484 @@
|
||||
"""API ViewSet for calendar operations (RSVP, conflict detection, calendar listing)."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from django.conf import settings
|
||||
from django.shortcuts import get_object_or_404
|
||||
from django.utils.functional import cached_property
|
||||
|
||||
from drf_spectacular.utils import (
|
||||
OpenApiResponse,
|
||||
extend_schema,
|
||||
inline_serializer,
|
||||
)
|
||||
from rest_framework import serializers as drf_serializers
|
||||
from rest_framework import status
|
||||
from rest_framework.exceptions import NotFound
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.throttling import ScopedRateThrottle
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from core import enums, models
|
||||
from core.api.permissions import HasAccessToMailbox, HasWriteAccessToMailbox
|
||||
from core.api.viewsets.task import register_task_owner
|
||||
from core.services.calendar.service import CalDAVError, CalDAVService
|
||||
from core.services.calendar.tasks import calendar_add_event_task, calendar_rsvp_task
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Shared OpenAPI error-response schema for the calendar endpoints. Inline
|
||||
# here so each endpoint's @extend_schema can reference it without dragging
|
||||
# a one-off serializer class through drf_spectacular.
|
||||
_ERROR_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {"detail": {"type": "string"}},
|
||||
}
|
||||
|
||||
|
||||
class CalDAVChannelMixin:
|
||||
"""Mixin to get the CalDAV channel or deployment-default config for a mailbox."""
|
||||
|
||||
@cached_property
|
||||
def mailbox(self):
|
||||
"""The Mailbox referenced in the URL."""
|
||||
return get_object_or_404(models.Mailbox, id=self.kwargs["mailbox_id"])
|
||||
|
||||
@cached_property
|
||||
def caldav_channel(self):
|
||||
"""The CalDAV channel for the mailbox, if any."""
|
||||
return models.Channel.objects.filter(
|
||||
mailbox=self.mailbox, type=enums.ChannelTypes.CALDAV
|
||||
).first()
|
||||
|
||||
def get_caldav_service(self):
|
||||
"""Get a CalDAVService for this mailbox.
|
||||
|
||||
Priority: per-mailbox Channel (user-configured, pointing at any
|
||||
CalDAV provider) > deployment-default config (``CALDAV_DEFAULT_*``
|
||||
env vars). For the default path, the *requesting user's OIDC
|
||||
identity email* is sent as the Basic Auth username — not the
|
||||
mailbox email — because the CalDAV server (e.g. suitenumerique
|
||||
/calendars) keys principals on the OIDC ``email`` claim, and the
|
||||
two can diverge (a Mailbox's ``local_part@domain.name`` is not
|
||||
always the human's primary identity address). Using the mailbox
|
||||
email here returns 403 Unknown User for those cases and silently
|
||||
hides the calendar UI; using the OIDC email routes to the
|
||||
principal the calendar provider provisioned on first login.
|
||||
See ``CalDAVService.from_instance_config`` for the trust model.
|
||||
Returns None if neither config path is available.
|
||||
"""
|
||||
return CalDAVService.from_channel_or_instance(
|
||||
self.caldav_channel, self.request.user.email
|
||||
)
|
||||
|
||||
def require_caldav_service(self):
|
||||
"""Get the CalDAVService or raise 404."""
|
||||
service = self.get_caldav_service()
|
||||
if not service:
|
||||
raise NotFound("No CalDAV calendar is configured for this mailbox.")
|
||||
return service
|
||||
|
||||
|
||||
@extend_schema(tags=["calendar"])
|
||||
class CalendarRsvpView(CalDAVChannelMixin, APIView):
|
||||
"""Submit an RSVP response to a calendar event."""
|
||||
|
||||
# Writing an RSVP on behalf of a mailbox is a CalDAV write that produces
|
||||
# an outbound iTIP REPLY — VIEWER-only access must not be able to do it.
|
||||
permission_classes = [HasWriteAccessToMailbox]
|
||||
|
||||
@extend_schema(
|
||||
request=inline_serializer(
|
||||
name="CalendarRsvpRequest",
|
||||
fields={
|
||||
"ics_data": drf_serializers.CharField(
|
||||
help_text="Raw ICS content of the event"
|
||||
),
|
||||
"response": drf_serializers.ChoiceField(
|
||||
choices=["ACCEPTED", "DECLINED", "TENTATIVE"],
|
||||
help_text="RSVP response",
|
||||
),
|
||||
"calendar_id": drf_serializers.CharField(
|
||||
required=False,
|
||||
allow_null=True,
|
||||
help_text="Optional specific calendar URL",
|
||||
),
|
||||
},
|
||||
),
|
||||
responses={
|
||||
200: inline_serializer(
|
||||
name="CalendarRsvpResponse",
|
||||
fields={
|
||||
"task_id": drf_serializers.CharField(),
|
||||
},
|
||||
),
|
||||
400: OpenApiResponse(
|
||||
response=_ERROR_SCHEMA,
|
||||
description="Missing or invalid ics_data / response.",
|
||||
),
|
||||
503: OpenApiResponse(
|
||||
response=_ERROR_SCHEMA,
|
||||
description="Task broker unavailable; the RSVP could not be enqueued.",
|
||||
),
|
||||
},
|
||||
)
|
||||
def post(self, request, mailbox_id): # pylint: disable=unused-argument
|
||||
"""Submit an RSVP response via a background CalDAV task."""
|
||||
ics_data = request.data.get("ics_data")
|
||||
response_type = request.data.get("response")
|
||||
calendar_id = request.data.get("calendar_id")
|
||||
|
||||
if not ics_data or not response_type:
|
||||
return Response(
|
||||
{"detail": "ics_data and response are required."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
if response_type not in ("ACCEPTED", "DECLINED", "TENTATIVE"):
|
||||
return Response(
|
||||
{"detail": "response must be ACCEPTED, DECLINED, or TENTATIVE."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
self.require_caldav_service()
|
||||
channel = self.caldav_channel
|
||||
mailbox_email = str(self.mailbox)
|
||||
|
||||
try:
|
||||
task = calendar_rsvp_task.delay(
|
||||
channel_id=str(channel.id) if channel else None,
|
||||
user_email=request.user.email,
|
||||
ics_data=ics_data,
|
||||
response=response_type,
|
||||
attendee_email=mailbox_email,
|
||||
calendar_id=calendar_id,
|
||||
)
|
||||
register_task_owner(task.id, request.user.id)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.exception("Failed to enqueue calendar_rsvp_task: %s", e)
|
||||
return Response(
|
||||
{"detail": "Could not schedule the RSVP task."},
|
||||
status=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
|
||||
return Response({"task_id": task.id}, status=status.HTTP_200_OK)
|
||||
|
||||
|
||||
@extend_schema(tags=["calendar"])
|
||||
class CalendarAddEventView(CalDAVChannelMixin, APIView):
|
||||
"""Add an event to a CalDAV calendar."""
|
||||
|
||||
# Writing an event into the mailbox's calendar must not be allowed for
|
||||
# VIEWER-only access.
|
||||
permission_classes = [HasWriteAccessToMailbox]
|
||||
|
||||
@extend_schema(
|
||||
request=inline_serializer(
|
||||
name="CalendarAddEventRequest",
|
||||
fields={
|
||||
"ics_data": drf_serializers.CharField(
|
||||
help_text="Raw ICS content of the event"
|
||||
),
|
||||
"calendar_id": drf_serializers.CharField(
|
||||
required=False,
|
||||
allow_null=True,
|
||||
help_text="Optional specific calendar URL",
|
||||
),
|
||||
},
|
||||
),
|
||||
responses={
|
||||
200: inline_serializer(
|
||||
name="CalendarAddEventResponse",
|
||||
fields={
|
||||
"task_id": drf_serializers.CharField(),
|
||||
},
|
||||
),
|
||||
400: OpenApiResponse(
|
||||
response=_ERROR_SCHEMA,
|
||||
description="Missing ics_data.",
|
||||
),
|
||||
503: OpenApiResponse(
|
||||
response=_ERROR_SCHEMA,
|
||||
description="Task broker unavailable; the add-event could not be enqueued.",
|
||||
),
|
||||
},
|
||||
)
|
||||
def post(self, request, mailbox_id): # pylint: disable=unused-argument
|
||||
"""Add an event to the mailbox's CalDAV calendar via a background task."""
|
||||
ics_data = request.data.get("ics_data")
|
||||
calendar_id = request.data.get("calendar_id")
|
||||
|
||||
if not ics_data:
|
||||
return Response(
|
||||
{"detail": "ics_data is required."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
self.require_caldav_service()
|
||||
channel = self.caldav_channel
|
||||
|
||||
try:
|
||||
task = calendar_add_event_task.delay(
|
||||
channel_id=str(channel.id) if channel else None,
|
||||
user_email=request.user.email,
|
||||
ics_data=ics_data,
|
||||
calendar_id=calendar_id,
|
||||
)
|
||||
register_task_owner(task.id, request.user.id)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.exception("Failed to enqueue calendar_add_event_task: %s", e)
|
||||
return Response(
|
||||
{"detail": "Could not schedule the add-event task."},
|
||||
status=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
|
||||
return Response({"task_id": task.id}, status=status.HTTP_200_OK)
|
||||
|
||||
|
||||
@extend_schema(tags=["calendar"])
|
||||
class CalendarConflictsView(CalDAVChannelMixin, APIView):
|
||||
"""Check for conflicting events in a given time range.
|
||||
|
||||
Note: CalDAV calls are intentionally blocking (synchronous) here because
|
||||
the user is waiting for the result before interacting with the UI.
|
||||
|
||||
Throttled per user under the ``caldav_conflicts`` scope. Each call
|
||||
PROPFINDs the home set and REPORTs every calendar in it, so a tight
|
||||
polling loop both stresses the CalDAV server and ties up request
|
||||
workers; a 30/min cap is generous for legitimate UI use (one call
|
||||
per opened invite) and bounds the cost of a runaway script.
|
||||
"""
|
||||
|
||||
permission_classes = [HasAccessToMailbox]
|
||||
throttle_classes = [ScopedRateThrottle]
|
||||
throttle_scope = "caldav_conflicts"
|
||||
|
||||
@extend_schema(
|
||||
request=inline_serializer(
|
||||
name="CalendarConflictsRequest",
|
||||
fields={
|
||||
"start": drf_serializers.DateTimeField(
|
||||
help_text="Start of the time range (ISO 8601)"
|
||||
),
|
||||
"end": drf_serializers.DateTimeField(
|
||||
help_text="End of the time range (ISO 8601)"
|
||||
),
|
||||
"exclude_uid": drf_serializers.CharField(
|
||||
required=False,
|
||||
allow_null=True,
|
||||
allow_blank=True,
|
||||
help_text=(
|
||||
"Optional UID of an event to exclude from conflicts "
|
||||
"(avoids flagging prior imports of the same invite)."
|
||||
),
|
||||
),
|
||||
},
|
||||
),
|
||||
responses={
|
||||
200: inline_serializer(
|
||||
name="CalendarConflictsResponse",
|
||||
fields={
|
||||
"conflicts": drf_serializers.ListField(
|
||||
child=drf_serializers.DictField()
|
||||
),
|
||||
"existing_partstats": drf_serializers.DictField(
|
||||
child=drf_serializers.CharField(),
|
||||
help_text=(
|
||||
"PARTSTAT per attendee identity (calendar owner "
|
||||
"email, lowercased) on the prior copy of "
|
||||
"``exclude_uid``. Lets the UI pre-select the right "
|
||||
"prior RSVP for the *selected* calendar when a "
|
||||
"mailbox can act through several attendee-owned "
|
||||
"calendars."
|
||||
),
|
||||
),
|
||||
},
|
||||
),
|
||||
400: OpenApiResponse(
|
||||
response=_ERROR_SCHEMA,
|
||||
description="Missing or invalid start/end.",
|
||||
),
|
||||
502: OpenApiResponse(
|
||||
response=_ERROR_SCHEMA,
|
||||
description="CalDAV server error while checking conflicts.",
|
||||
),
|
||||
},
|
||||
)
|
||||
def post(self, request, mailbox_id): # pylint: disable=unused-argument
|
||||
"""Return a list of events overlapping the requested time range."""
|
||||
start = request.data.get("start")
|
||||
end = request.data.get("end")
|
||||
exclude_uid = request.data.get("exclude_uid") or None
|
||||
|
||||
if not start or not end:
|
||||
return Response(
|
||||
{"detail": "start and end are required."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
try:
|
||||
if isinstance(start, str):
|
||||
start = datetime.fromisoformat(start)
|
||||
if isinstance(end, str):
|
||||
end = datetime.fromisoformat(end)
|
||||
except (ValueError, TypeError):
|
||||
return Response(
|
||||
{"detail": "start and end must be valid ISO 8601 datetimes."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
# The CalDAV time-range filter (`_format_utc`) expects aware
|
||||
# datetimes — silently treating naive input as UTC would lie
|
||||
# about the wall-clock value sent to the server.
|
||||
if start.tzinfo is None or end.tzinfo is None:
|
||||
return Response(
|
||||
{"detail": "start and end must include timezone info."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
if end <= start:
|
||||
return Response(
|
||||
{"detail": "end must be after start."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
service = self.require_caldav_service()
|
||||
|
||||
try:
|
||||
result = service.check_conflicts(
|
||||
start=start,
|
||||
end=end,
|
||||
exclude_uid=exclude_uid,
|
||||
attendee_email=str(self.mailbox),
|
||||
)
|
||||
except CalDAVError as e:
|
||||
# Upstream CalDAV failure (network, 4xx/5xx from server,
|
||||
# SSRF guard tripped, etc.) — 502 is the right shape.
|
||||
logger.warning("CalDAV upstream failed during conflicts: %s", e)
|
||||
return Response(
|
||||
{"detail": "CalDAV server returned an error while checking conflicts."},
|
||||
status=status.HTTP_502_BAD_GATEWAY,
|
||||
)
|
||||
# Anything else is an unexpected programming error — let Django's
|
||||
# default handler return 500 so it doesn't get mis-labelled as
|
||||
# an upstream issue.
|
||||
|
||||
return Response(result, status=status.HTTP_200_OK)
|
||||
|
||||
|
||||
@extend_schema(tags=["calendar"])
|
||||
class CalendarListView(CalDAVChannelMixin, APIView):
|
||||
"""List available calendars on the CalDAV server.
|
||||
|
||||
Note: CalDAV calls are intentionally blocking (synchronous) here because
|
||||
the user is waiting for the result before interacting with the UI.
|
||||
"""
|
||||
|
||||
permission_classes = [HasAccessToMailbox]
|
||||
|
||||
@extend_schema(
|
||||
responses={
|
||||
200: inline_serializer(
|
||||
name="CalendarListResponse",
|
||||
fields={
|
||||
"calendars": drf_serializers.ListField(
|
||||
child=drf_serializers.DictField()
|
||||
),
|
||||
"web_url": drf_serializers.CharField(
|
||||
allow_null=True,
|
||||
help_text=("Public URL of the calendar web UI, if configured."),
|
||||
),
|
||||
"configured": drf_serializers.BooleanField(
|
||||
help_text=(
|
||||
"True when a CalDAV service is configured for this "
|
||||
"mailbox (per-mailbox channel or deployment default). "
|
||||
"False means the integration is disabled."
|
||||
),
|
||||
),
|
||||
},
|
||||
),
|
||||
403: OpenApiResponse(
|
||||
response=_ERROR_SCHEMA,
|
||||
description=(
|
||||
"Per-mailbox CalDAV channel denied access (upstream 403). "
|
||||
"Not returned for the deployment-default config, where a "
|
||||
"403 is treated as an empty calendar list."
|
||||
),
|
||||
),
|
||||
502: OpenApiResponse(
|
||||
response=_ERROR_SCHEMA,
|
||||
description="CalDAV server error while listing calendars.",
|
||||
),
|
||||
},
|
||||
)
|
||||
def get(self, request, mailbox_id): # pylint: disable=unused-argument
|
||||
"""Return the list of calendars available for the mailbox."""
|
||||
web_url = settings.CALDAV_DEFAULT_WEB_URL
|
||||
service = self.get_caldav_service()
|
||||
if not service:
|
||||
return Response(
|
||||
{"calendars": [], "web_url": web_url, "configured": False},
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
try:
|
||||
# Only list calendars the user can write to — the UI uses this
|
||||
# to pick a destination for RSVP/add-event, and read-only
|
||||
# calendars would fail at PUT time.
|
||||
calendars = service.list_calendars(writable_only=True)
|
||||
except CalDAVError as e:
|
||||
# 403 on the *instance-level* path means "we authenticated the
|
||||
# service credential but this OIDC identity is not yet a
|
||||
# principal upstream" — calendars (and similar providers)
|
||||
# provision a principal on first login, so this is the
|
||||
# "user has no calendars *yet*" state, not "integration
|
||||
# disabled". Surface it as configured=True with an empty
|
||||
# list so the UI shows the create-a-calendar CTA.
|
||||
#
|
||||
# This rationale only holds for the deployment-default config:
|
||||
# a per-mailbox Channel uses the user's own credentials against
|
||||
# a CalDAV provider of their choice, so a 403 there is a genuine
|
||||
# ACL/auth failure that must surface — not be hidden behind a
|
||||
# spurious empty list.
|
||||
if e.status_code == 403 and self.caldav_channel is None:
|
||||
logger.info(
|
||||
"CalDAV reports user %s has no calendar account yet (HTTP 403); "
|
||||
"treating as empty calendar list.",
|
||||
request.user.id,
|
||||
)
|
||||
return Response(
|
||||
{"calendars": [], "web_url": web_url, "configured": True},
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
if e.status_code == 403:
|
||||
# Log only the channel id + status — the CalDAVError message
|
||||
# embeds the (user-supplied) upstream URL, which must not
|
||||
# reach the logs.
|
||||
logger.warning(
|
||||
"CalDAV channel %s denied access (HTTP 403) during list_calendars.",
|
||||
self.caldav_channel.id,
|
||||
)
|
||||
return Response(
|
||||
{"detail": "CalDAV server denied access while listing calendars."},
|
||||
status=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
# Same redaction: never log ``e`` — its message embeds the
|
||||
# upstream URL (and, for network errors, the raw requests
|
||||
# exception). Channel id + HTTP status are enough to triage.
|
||||
logger.warning(
|
||||
"CalDAV upstream failed during list_calendars "
|
||||
"(channel=%s, HTTP status=%s).",
|
||||
self.caldav_channel.id if self.caldav_channel else None,
|
||||
e.status_code,
|
||||
)
|
||||
return Response(
|
||||
{"detail": "CalDAV server returned an error while listing calendars."},
|
||||
status=status.HTTP_502_BAD_GATEWAY,
|
||||
)
|
||||
# Anything else is unexpected — let Django return 500.
|
||||
|
||||
return Response(
|
||||
{"calendars": calendars, "web_url": web_url, "configured": True},
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
@@ -60,13 +60,23 @@ class ConfigView(drf.views.APIView):
|
||||
"type": "string",
|
||||
"readOnly": True,
|
||||
},
|
||||
"preview_url": {
|
||||
"type": "string",
|
||||
"readOnly": True,
|
||||
},
|
||||
"app_name": {
|
||||
"type": "string",
|
||||
"readOnly": True,
|
||||
},
|
||||
},
|
||||
"readOnly": True,
|
||||
"required": ["sdk_url", "api_url", "file_url", "app_name"],
|
||||
"required": [
|
||||
"sdk_url",
|
||||
"api_url",
|
||||
"file_url",
|
||||
"preview_url",
|
||||
"app_name",
|
||||
],
|
||||
},
|
||||
"SCHEMA_CUSTOM_ATTRIBUTES_USER": {
|
||||
"type": "object",
|
||||
@@ -218,6 +228,9 @@ class ConfigView(drf.views.APIView):
|
||||
"sdk_url": f"{base_url}{settings.DRIVE_CONFIG.get('sdk_url')}",
|
||||
"api_url": f"{base_url}{settings.DRIVE_CONFIG.get('api_url')}",
|
||||
"file_url": f"{base_url}{settings.DRIVE_CONFIG.get('file_url')}",
|
||||
"preview_url": (
|
||||
f"{base_url}{settings.DRIVE_CONFIG.get('preview_url')}"
|
||||
),
|
||||
"app_name": settings.DRIVE_CONFIG.get("app_name"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import json
|
||||
import logging
|
||||
|
||||
from django.core.exceptions import ValidationError as DjangoValidationError
|
||||
from django.db import transaction
|
||||
|
||||
import rest_framework as drf
|
||||
@@ -190,8 +191,34 @@ class DraftMessageView(APIView):
|
||||
Return updated draft message
|
||||
"""
|
||||
|
||||
permission_classes = [permissions.IsAllowedToCreateMessage]
|
||||
mailbox = None
|
||||
permission_classes = [permissions.IsAuthenticated]
|
||||
|
||||
@staticmethod
|
||||
def _resolve_editable_sender_mailbox(user, sender_id):
|
||||
"""Return the ``sender_id`` mailbox the ``user`` may draft/send from.
|
||||
|
||||
Resolves and authorizes in one query: the mailbox is looked up scoped
|
||||
to a ``MAILBOX_ROLES_CAN_EDIT`` access for ``user``. Raises 403 when the
|
||||
mailbox is missing OR the user lacks an editor role on it — without
|
||||
distinguishing the two, so the endpoint never reveals which mailboxes
|
||||
exist. (This is the authorization that previously lived in the
|
||||
single-use, request-shape-coupled ``IsAllowedToCreateMessage``
|
||||
permission; create_draft still enforces parent-thread access for
|
||||
replies, and the PUT draft lookup scopes to an editable draft thread.)
|
||||
"""
|
||||
try:
|
||||
mailbox = models.Mailbox.objects.filter(
|
||||
id=sender_id,
|
||||
accesses__user=user,
|
||||
accesses__role__in=enums.MAILBOX_ROLES_CAN_EDIT,
|
||||
).first()
|
||||
except (DjangoValidationError, ValueError, TypeError):
|
||||
mailbox = None # malformed senderId — treat as no access
|
||||
if mailbox is None:
|
||||
raise drf.exceptions.PermissionDenied(
|
||||
"You do not have permission to send as this mailbox."
|
||||
)
|
||||
return mailbox
|
||||
|
||||
@transaction.atomic
|
||||
def post(self, request):
|
||||
@@ -203,13 +230,10 @@ class DraftMessageView(APIView):
|
||||
|
||||
subject = request.data.get("subject")
|
||||
|
||||
# Get mailbox (permission class validates access)
|
||||
try:
|
||||
sender_mailbox = models.Mailbox.objects.get(id=sender_id)
|
||||
except models.Mailbox.DoesNotExist as exc:
|
||||
raise drf.exceptions.NotFound(
|
||||
f"Mailbox with senderId {sender_id} not found."
|
||||
) from exc
|
||||
# Resolve + authorize the sender mailbox (user must hold an editor-or-
|
||||
# above role on it). create_draft separately enforces access to the
|
||||
# parent thread for replies.
|
||||
sender_mailbox = self._resolve_editable_sender_mailbox(request.user, sender_id)
|
||||
|
||||
# Create draft
|
||||
message = create_draft(
|
||||
@@ -249,13 +273,10 @@ class DraftMessageView(APIView):
|
||||
"senderId is required in request body for update."
|
||||
)
|
||||
|
||||
# Get mailbox
|
||||
try:
|
||||
sender_mailbox = models.Mailbox.objects.get(id=sender_id)
|
||||
except models.Mailbox.DoesNotExist as exc:
|
||||
raise drf.exceptions.NotFound(
|
||||
f"Mailbox with senderId {sender_id} not found."
|
||||
) from exc
|
||||
# Resolve + authorize the sender mailbox (editor-or-above role). The
|
||||
# draft lookup below additionally scopes to a draft whose thread this
|
||||
# mailbox can edit (404 otherwise).
|
||||
sender_mailbox = self._resolve_editable_sender_mailbox(request.user, sender_id)
|
||||
|
||||
# Get the draft message
|
||||
try:
|
||||
|
||||
@@ -8,6 +8,7 @@ from django.conf import settings
|
||||
|
||||
import jwt
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from jmap_email import parse_email
|
||||
from rest_framework import status, viewsets
|
||||
from rest_framework.authentication import BaseAuthentication
|
||||
from rest_framework.decorators import action
|
||||
@@ -17,15 +18,32 @@ from rest_framework.response import Response
|
||||
|
||||
from core import models
|
||||
from core.mda.inbound import check_local_recipients, deliver_inbound_message
|
||||
from core.mda.rfc5322 import EmailParseError, parse_email_message, remove_mime_headers
|
||||
from core.mda.raw_mime import remove_mime_headers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MTAJWTAuthentication(BaseAuthentication):
|
||||
"""
|
||||
Custom authentication for MTA endpoints using JWT tokens with email hash validation.
|
||||
Returns None or (user, auth)
|
||||
"""Authenticate the MTA-to-MDA channel via an HS256 JWT.
|
||||
|
||||
Trust model: the whole channel rests on the shared ``MDA_API_SECRET``.
|
||||
Only the MTA-in service knows it, so a valid HMAC signature *is* the proof
|
||||
of identity — there is no per-request identity beyond "signed by the
|
||||
secret". Consequently we do NOT attempt replay protection (a ``jti`` nonce
|
||||
store, etc.): anyone able to forge the signature already holds the secret
|
||||
and could mint fresh tokens at will, and anyone who cannot is stopped by
|
||||
the signature check. Keeping the secret out of source (it has no default —
|
||||
see ``settings.MDA_API_SECRET``) and the transport on TLS is what actually
|
||||
secures this path.
|
||||
|
||||
On top of the signature we keep two cheap, narrow guards:
|
||||
- ``exp``: bounds a leaked token's useful lifetime. The issuer sizes the
|
||||
claim to cover its full retry window (see mta-in ``mda_api_call``).
|
||||
- ``body_hash``: binds the token to its exact request body, so a captured
|
||||
token can't be repurposed for a *different* body within that window.
|
||||
Enforced even for an empty body (the bodyless ``/check`` path).
|
||||
|
||||
Returns None or (user, auth).
|
||||
"""
|
||||
|
||||
def authenticate(self, request):
|
||||
@@ -40,20 +58,26 @@ class MTAJWTAuthentication(BaseAuthentication):
|
||||
settings.MDA_API_SECRET,
|
||||
algorithms=["HS256"],
|
||||
options={
|
||||
"require": ["exp"],
|
||||
# exp bounds the lifetime; body_hash binds the token to its
|
||||
# payload. Both are mandatory.
|
||||
"require": ["exp", "body_hash"],
|
||||
"verify_exp": True,
|
||||
"verify_signature": True,
|
||||
},
|
||||
)
|
||||
|
||||
if not payload.get("exp"):
|
||||
raise jwt.InvalidTokenError("Missing expiration time")
|
||||
|
||||
# Validate email hash if there's a body
|
||||
if request.body:
|
||||
body_hash = hashlib.sha256(request.body).hexdigest()
|
||||
if not secrets.compare_digest(body_hash, payload["body_hash"]):
|
||||
raise jwt.InvalidTokenError("Invalid email hash")
|
||||
# Bind the token to its payload. Always enforced — including for
|
||||
# an empty body (sha256 of b"") — so the bodyless /check endpoint
|
||||
# can't be driven with a token minted for a different request.
|
||||
claimed_hash = payload["body_hash"]
|
||||
# ``compare_digest`` raises TypeError on mismatched types (e.g. a
|
||||
# numeric ``body_hash`` claim), which would surface as a 500 rather
|
||||
# than an auth failure. Reject a non-string claim up front.
|
||||
if not isinstance(claimed_hash, str):
|
||||
raise jwt.InvalidTokenError("Invalid email hash")
|
||||
body_hash = hashlib.sha256(request.body or b"").hexdigest()
|
||||
if not secrets.compare_digest(body_hash, claimed_hash):
|
||||
raise jwt.InvalidTokenError("Invalid email hash")
|
||||
|
||||
service_account = models.User()
|
||||
return (service_account, payload)
|
||||
@@ -183,10 +207,10 @@ class InboundMTAViewSet(viewsets.GenericViewSet):
|
||||
).encode("utf-8") + raw_data
|
||||
|
||||
# Parse the email message once
|
||||
try:
|
||||
parsed_email = parse_email_message(raw_data)
|
||||
except EmailParseError as e:
|
||||
logger.error("Failed to parse inbound email: %s", str(e))
|
||||
parsed_email = parse_email(raw_data)
|
||||
if parsed_email is None:
|
||||
# Sender-supplied malformed input; not an internal error.
|
||||
logger.warning("Failed to parse inbound email (returning 400)")
|
||||
# Consider saving the raw email for debugging
|
||||
return Response(
|
||||
{"status": "error", "detail": "Failed to parse email"},
|
||||
|
||||
@@ -4,25 +4,66 @@ import logging
|
||||
from html import escape as html_escape
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.core.validators import validate_email
|
||||
from django.utils import timezone
|
||||
from django.conf import settings
|
||||
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from jmap_email import compose_email, parse_address
|
||||
from rest_framework import status, viewsets
|
||||
from rest_framework.authentication import BaseAuthentication
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.exceptions import AuthenticationFailed
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.throttling import SimpleRateThrottle
|
||||
|
||||
from core import models
|
||||
from core.api.permissions import IsAuthenticated
|
||||
from core.mda.inbound import deliver_inbound_message
|
||||
from core.mda.rfc5322 import compose_email
|
||||
from core.mda.utils import current_sent_at
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WidgetChannelThrottle(SimpleRateThrottle):
|
||||
"""Per-channel rate limit for the public widget deliver endpoint.
|
||||
|
||||
The channel id is the literal value embedded in the public HTML snippet,
|
||||
so it offers no secrecy — anyone who scrapes it can POST. Keying the
|
||||
throttle on the channel (not the source IP) caps the total inbound volume
|
||||
a single widget can push into its mailbox regardless of how many IPs the
|
||||
caller rotates through, bounding mailbox/blob/Contact/Thread growth and
|
||||
per-message AI labeling cost.
|
||||
"""
|
||||
|
||||
scope = "widget_inbound_channel"
|
||||
|
||||
def get_cache_key(self, request, view):
|
||||
auth = getattr(request, "auth", None)
|
||||
channel = auth.get("channel") if isinstance(auth, dict) else None
|
||||
if channel is None:
|
||||
return None # Unauthenticated — auth layer will reject it.
|
||||
return self.cache_format % {"scope": self.scope, "ident": str(channel.id)}
|
||||
|
||||
|
||||
class WidgetIPThrottle(SimpleRateThrottle):
|
||||
"""Per-IP burst limit, layered under the per-channel cap above.
|
||||
|
||||
Stops a single source from saturating a channel's quota and gives a
|
||||
cheaper first line of defense against floods from one host.
|
||||
"""
|
||||
|
||||
scope = "widget_inbound_ip"
|
||||
|
||||
def get_cache_key(self, request, view):
|
||||
# Key on REMOTE_ADDR, not DRF's get_ident(): get_ident() prefers the
|
||||
# raw X-Forwarded-For header (client-spoofable, and only trustworthy
|
||||
# when NUM_PROXIES is configured — this project does not use it).
|
||||
# Instead REMOTE_ADDR is normalized to the real client IP by
|
||||
# XForwardedForMiddleware when USE_X_FORWARDED_FOR is enabled, which is
|
||||
# the IP the rest of this view already trusts.
|
||||
ident = request.META.get("REMOTE_ADDR")
|
||||
return self.cache_format % {"scope": self.scope, "ident": ident}
|
||||
|
||||
|
||||
class WidgetAuthentication(BaseAuthentication):
|
||||
"""
|
||||
Custom authentication for widget endpoints using channel_id header
|
||||
@@ -54,6 +95,17 @@ class InboundWidgetViewSet(viewsets.GenericViewSet):
|
||||
permission_classes = [IsAuthenticated]
|
||||
authentication_classes = [WidgetAuthentication]
|
||||
|
||||
def get_throttles(self):
|
||||
"""Rate-limit only the public ``deliver`` endpoint.
|
||||
|
||||
``config`` is a cheap idempotent read fetched on widget load and is
|
||||
left unthrottled; ``deliver`` is the write path an attacker could
|
||||
abuse, so it carries both the per-channel and per-IP throttles.
|
||||
"""
|
||||
if getattr(self, "action", None) == "deliver":
|
||||
return [WidgetChannelThrottle(), WidgetIPThrottle()]
|
||||
return super().get_throttles()
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
@action(
|
||||
detail=False,
|
||||
@@ -81,8 +133,6 @@ class InboundWidgetViewSet(viewsets.GenericViewSet):
|
||||
def deliver(self, request):
|
||||
"""Handle incoming widget message."""
|
||||
|
||||
# TODO: throttle
|
||||
|
||||
data = request.data
|
||||
auth_data = request.auth
|
||||
channel = auth_data["channel"]
|
||||
@@ -95,13 +145,26 @@ class InboundWidgetViewSet(viewsets.GenericViewSet):
|
||||
{"detail": "Missing email"}, status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
|
||||
# Validate the sender email format with django's email validator
|
||||
try:
|
||||
validate_email(sender_email)
|
||||
except ValidationError:
|
||||
# Cap the body so a caller can't fill blob storage with one giant
|
||||
# message. ``message_text`` is the only unbounded field; it is expanded
|
||||
# into both the text and HTML parts of the stored MIME, so bounding it
|
||||
# bounds the resulting blob. Mirrors the MAX_INCOMING_EMAIL_SIZE limit
|
||||
# the MTA path already enforces.
|
||||
if len(message_text.encode("utf-8")) > settings.MAX_INCOMING_EMAIL_SIZE:
|
||||
return Response(
|
||||
{"detail": "Message too large"},
|
||||
status=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
)
|
||||
|
||||
# Validate through the same parser the rest of the pipeline
|
||||
# uses. ``parse_address`` is strict by default and returns
|
||||
# ``("", "")`` on garbage input.
|
||||
_, normalised_sender = parse_address(sender_email)
|
||||
if not normalised_sender:
|
||||
return Response(
|
||||
{"detail": "Invalid email format"}, status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
sender_email = normalised_sender
|
||||
|
||||
if not message_text:
|
||||
return Response(
|
||||
@@ -157,13 +220,13 @@ class InboundWidgetViewSet(viewsets.GenericViewSet):
|
||||
# Sanitize subject to prevent header injection (strip newlines/carriage returns)
|
||||
subject = subject.replace("\r", "").replace("\n", "")
|
||||
|
||||
# Build a JMAP-like structured format that we could have got from parse_email_message()
|
||||
# Build a JMAP-like structured format that we could have got from parse_email()
|
||||
|
||||
parsed_email = {
|
||||
"subject": subject,
|
||||
"from": {"email": sender_email},
|
||||
"from": [{"email": sender_email}],
|
||||
"to": [{"name": target_name, "email": target_email}],
|
||||
"date": timezone.now(),
|
||||
"sentAt": current_sent_at(),
|
||||
"htmlBody": [{"content": html_escape(message_text).replace("\n", "<br/>")}],
|
||||
"textBody": [{"content": message_text}],
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ class MailboxViewSet(
|
||||
def get_queryset(self):
|
||||
"""Restrict results to the current user's mailboxes."""
|
||||
user = self.request.user
|
||||
# For regular users, annotate with their actual role
|
||||
|
||||
return (
|
||||
models.Mailbox.objects.filter(accesses__user=user)
|
||||
.prefetch_related("accesses__user", "domain")
|
||||
@@ -38,6 +38,36 @@ class MailboxViewSet(
|
||||
.order_by("-created_at")
|
||||
)
|
||||
|
||||
def get_permissions(self):
|
||||
"""Require mailbox-admin rights to edit; reading stays open to any
|
||||
member of the mailbox."""
|
||||
if self.action == "partial_update":
|
||||
return [
|
||||
permissions.IsAuthenticated(),
|
||||
permissions.IsMailboxAdminObject(),
|
||||
]
|
||||
return super().get_permissions()
|
||||
|
||||
@extend_schema(
|
||||
tags=["mailboxes"],
|
||||
request=serializers.MailboxNameUpdateSerializer,
|
||||
responses=serializers.MailboxSerializer,
|
||||
)
|
||||
def partial_update(self, request, *args, **kwargs):
|
||||
"""Rename a mailbox (its display contact name). Mailbox admins only.
|
||||
|
||||
``partial=True`` keeps true PATCH semantics: omitting ``name`` is a no-op
|
||||
rather than a 400, so the runtime matches the optional request schema.
|
||||
"""
|
||||
mailbox = self.get_object()
|
||||
serializer = serializers.MailboxNameUpdateSerializer(
|
||||
instance=mailbox, data=request.data, partial=True
|
||||
)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
serializer.save()
|
||||
output = self.get_serializer(mailbox)
|
||||
return Response(output.data)
|
||||
|
||||
@extend_schema(
|
||||
tags=["mailboxes"],
|
||||
parameters=[
|
||||
|
||||
@@ -52,9 +52,9 @@ class MailboxAccessViewSet(
|
||||
Return MailboxAccess instances for the specific Mailbox from the URL.
|
||||
Permissions should have already verified the user can access this mailbox.
|
||||
"""
|
||||
mailbox = self.get_mailbox_object() # Ensures mailbox exists and handles 404
|
||||
mailbox = self.get_mailbox_object()
|
||||
return mailbox.accesses.select_related("user", "mailbox__domain").order_by(
|
||||
"-created_at"
|
||||
"created_at"
|
||||
)
|
||||
|
||||
def get_serializer_context(self):
|
||||
|
||||
@@ -9,7 +9,9 @@ from drf_spectacular.utils import (
|
||||
extend_schema,
|
||||
)
|
||||
from rest_framework import mixins, viewsets
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.generics import get_object_or_404
|
||||
from rest_framework.response import Response
|
||||
|
||||
from core.api import permissions
|
||||
from core.api.serializers import (
|
||||
@@ -19,6 +21,7 @@ from core.api.serializers import (
|
||||
from core.api.viewsets.mixins import MessageTemplateResponseMixin
|
||||
from core.models import (
|
||||
Mailbox,
|
||||
Message,
|
||||
MessageTemplate,
|
||||
MessageTemplateTypeChoices,
|
||||
)
|
||||
@@ -55,7 +58,7 @@ class MailboxMessageTemplateViewSet(
|
||||
|
||||
def get_permissions(self):
|
||||
"""Get permissions for the viewset."""
|
||||
if self.action in ["list", "retrieve"]:
|
||||
if self.action in ["list", "retrieve", "render"]:
|
||||
return [permissions.HasAccessToMailbox()]
|
||||
return super().get_permissions()
|
||||
|
||||
@@ -66,7 +69,7 @@ class MailboxMessageTemplateViewSet(
|
||||
|
||||
def get_queryset(self):
|
||||
"""Get message templates for a mailbox the user has access to."""
|
||||
if self.action == "retrieve":
|
||||
if self.action in ("retrieve", "render"):
|
||||
queryset = MessageTemplate.objects.filter(
|
||||
Q(mailbox=self.mailbox) | Q(maildomain=self.mailbox.domain)
|
||||
)
|
||||
@@ -112,6 +115,68 @@ class MailboxMessageTemplateViewSet(
|
||||
"""Retrieve a message template."""
|
||||
return super().retrieve(request, *args, **kwargs)
|
||||
|
||||
@extend_schema(
|
||||
summary="Render a message template",
|
||||
description=(
|
||||
"Render the template's html and text bodies with placeholders "
|
||||
"resolved from the mailbox and the authenticated user "
|
||||
"(name, user_name, custom attributes). When a draft message_id is "
|
||||
"provided, message-level placeholders (recipient_name) are also "
|
||||
"resolved. Unresolved placeholders keep their {placeholder} token."
|
||||
),
|
||||
parameters=[
|
||||
OpenApiParameter(
|
||||
name="message_id",
|
||||
type=OpenApiTypes.UUID,
|
||||
location=OpenApiParameter.QUERY,
|
||||
required=False,
|
||||
description=(
|
||||
"Optional draft id used to resolve message-level "
|
||||
"placeholders. Ignored unless it references a draft "
|
||||
"owned by this mailbox."
|
||||
),
|
||||
),
|
||||
],
|
||||
responses={
|
||||
200: {
|
||||
"type": "object",
|
||||
"description": "The rendered template bodies.",
|
||||
"properties": {
|
||||
"html_body": {"type": "string"},
|
||||
"text_body": {"type": "string"},
|
||||
},
|
||||
"required": ["html_body", "text_body"],
|
||||
"example": {
|
||||
"html_body": "<p>John Doe</p>",
|
||||
"text_body": "John Doe",
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
@action(detail=True, methods=["get"])
|
||||
def render(self, request, *args, **kwargs):
|
||||
"""Render a template with placeholders resolved for the current context.
|
||||
|
||||
The mailbox comes from the URL. A draft (message_id) is only honored
|
||||
when it belongs to this mailbox, so the endpoint cannot be used to
|
||||
probe recipients of drafts the user does not own.
|
||||
"""
|
||||
template = self.get_object()
|
||||
|
||||
message = None
|
||||
message_id = request.query_params.get("message_id")
|
||||
if message_id:
|
||||
message = Message.objects.filter(
|
||||
id=message_id,
|
||||
is_draft=True,
|
||||
sender__mailbox=self.mailbox,
|
||||
).first()
|
||||
|
||||
rendered = template.render_template(
|
||||
mailbox=self.mailbox, user=request.user, message=message
|
||||
)
|
||||
return Response(rendered)
|
||||
|
||||
|
||||
class AvailableMailboxMessageTemplateViewSet(
|
||||
mixins.ListModelMixin, viewsets.GenericViewSet
|
||||
|
||||
@@ -11,6 +11,10 @@ from rest_framework.views import APIView
|
||||
|
||||
from core import enums, models
|
||||
|
||||
# Built-in placeholders, always available. They carry no label: the frontend
|
||||
# localizes them client-side from its "placeholders" i18next namespace.
|
||||
BUILTIN_PLACEHOLDER_FIELDS = ("name", "recipient_name", "user_name")
|
||||
|
||||
|
||||
@extend_schema(tags=["placeholders"])
|
||||
class PlaceholderView(APIView):
|
||||
@@ -32,40 +36,65 @@ class PlaceholderView(APIView):
|
||||
responses={
|
||||
200: {
|
||||
"type": "object",
|
||||
"description": "Field slugs mapped to their verbose labels",
|
||||
"description": (
|
||||
"Field slugs mapped to their label metadata. Built-in "
|
||||
"fields have an empty object and are localized client-side. "
|
||||
"Custom attribute fields expose their schema title and "
|
||||
"optional per-language translations."
|
||||
),
|
||||
"additionalProperties": {
|
||||
"type": "string",
|
||||
"description": "Verbose label for the field",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {
|
||||
"type": "string",
|
||||
"description": "Default label (custom fields only).",
|
||||
},
|
||||
"i18n": {
|
||||
"type": "object",
|
||||
"additionalProperties": {"type": "string"},
|
||||
"description": (
|
||||
"Label translations by language code, from the "
|
||||
"schema 'x-i18n' entry (custom fields only)."
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
"example": {
|
||||
"name": "Name",
|
||||
"job_title": "Job title",
|
||||
"is_elected": "Is elected",
|
||||
"name": {},
|
||||
"recipient_name": {},
|
||||
"job_title": {
|
||||
"title": "Job title",
|
||||
"i18n": {"en": "Job title", "fr": "Fonction"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
def get(self, request):
|
||||
"""Get the structure of available fields."""
|
||||
current_language = settings.LANGUAGE_CODE.split("-")[0]
|
||||
fields = {
|
||||
"name": "Name",
|
||||
"recipient_name": "Recipient name",
|
||||
}
|
||||
# Add user custom attributes fields from schema
|
||||
"""Get the structure of available fields.
|
||||
|
||||
Built-in fields are returned as empty objects and localized
|
||||
client-side. Custom attribute fields carry their schema title and,
|
||||
when defined, the ``x-i18n`` title translations so the frontend can
|
||||
pick the right language.
|
||||
"""
|
||||
fields = {field_name: {} for field_name in BUILTIN_PLACEHOLDER_FIELDS}
|
||||
|
||||
# Add user custom attributes fields from schema. Only string fields are
|
||||
# exposed: a placeholder is substituted as text, so non-string types
|
||||
# (e.g. boolean, integer) are not meaningful here.
|
||||
schema = settings.SCHEMA_CUSTOM_ATTRIBUTES_USER
|
||||
schema_properties = schema.get("properties", {})
|
||||
for field_name, field_schema in schema_properties.items():
|
||||
# Check if there's internationalization
|
||||
i18n_data = field_schema.get("x-i18n", {})
|
||||
if "title" in i18n_data:
|
||||
label = i18n_data["title"].get(
|
||||
current_language, i18n_data["title"].get("en", field_name)
|
||||
)
|
||||
else:
|
||||
# No internationalization, use schema title
|
||||
label = field_schema.get("title", field_name)
|
||||
fields[field_name] = label
|
||||
if field_schema.get("type") != "string":
|
||||
continue
|
||||
field = {"title": field_schema.get("title", field_name)}
|
||||
x_i18n = field_schema.get("x-i18n")
|
||||
if isinstance(x_i18n, dict):
|
||||
i18n_titles = x_i18n.get("title")
|
||||
if isinstance(i18n_titles, dict) and i18n_titles:
|
||||
field["i18n"] = i18n_titles
|
||||
fields[field_name] = field
|
||||
return Response(fields)
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
"""API ViewSet for sending messages."""
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
from django.db import transaction
|
||||
|
||||
from drf_spectacular.utils import (
|
||||
OpenApiExample,
|
||||
@@ -13,7 +16,7 @@ from rest_framework import status
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from core import models
|
||||
from core import enums, models
|
||||
from core.api.viewsets.task import register_task_owner
|
||||
from core.mda.outbound import prepare_outbound_message
|
||||
from core.mda.outbound_tasks import send_message_task
|
||||
@@ -30,8 +33,7 @@ logger = logging.getLogger(__name__)
|
||||
200: inline_serializer(
|
||||
name="SendMessageResponse",
|
||||
fields={
|
||||
"message": serializers.MessageSerializer(),
|
||||
"task_id": drf_serializers.CharField(help_text="Task ID for tracking"),
|
||||
"task_id": drf_serializers.UUIDField(help_text="Task ID for tracking"),
|
||||
},
|
||||
),
|
||||
400: OpenApiExample(
|
||||
@@ -42,8 +44,8 @@ logger = logging.getLogger(__name__)
|
||||
"Permission Error",
|
||||
value={"detail": "You do not have permission to send this message."},
|
||||
),
|
||||
503: OpenApiExample(
|
||||
"Service Unavailable",
|
||||
500: OpenApiExample(
|
||||
"Prepare Failure",
|
||||
value={"detail": "Failed to prepare message for sending."},
|
||||
),
|
||||
},
|
||||
@@ -63,6 +65,12 @@ logger = logging.getLogger(__name__)
|
||||
"textBody": "Hello, world!",
|
||||
"htmlBody": "<p>Hello, world!</p>",
|
||||
},
|
||||
request_only=True,
|
||||
),
|
||||
OpenApiExample(
|
||||
"Send Draft Result",
|
||||
value={"task_id": "123e4567-e89b-12d3-a456-426614174000"},
|
||||
response_only=True,
|
||||
),
|
||||
],
|
||||
)
|
||||
@@ -107,29 +115,60 @@ class SendMessageView(APIView):
|
||||
|
||||
self.check_object_permissions(request, message)
|
||||
|
||||
prepared = prepare_outbound_message(
|
||||
mailbox_sender,
|
||||
message,
|
||||
request.data.get("textBody"),
|
||||
request.data.get("htmlBody"),
|
||||
request.user,
|
||||
)
|
||||
if not prepared:
|
||||
raise drf_exceptions.APIException(
|
||||
"Failed to prepare message for sending.",
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
# The sender mailbox itself must be authorised to send on this thread.
|
||||
# ``IsAllowedToAccess`` only proves the user can SEND through *some*
|
||||
# mailbox holding EDITOR access to the thread — not necessarily
|
||||
# ``mailbox_sender``. Re-check against the specific ``senderId`` so a
|
||||
# VIEWER on the sender mailbox cannot send as it by piggy-backing on a
|
||||
# SENDER role they hold on a different mailbox sharing the thread.
|
||||
can_send_as_sender = models.ThreadAccess.objects.filter(
|
||||
thread=message.thread,
|
||||
mailbox=mailbox_sender,
|
||||
role=enums.ThreadAccessRoleChoices.EDITOR,
|
||||
mailbox__accesses__user=request.user,
|
||||
mailbox__accesses__role__in=enums.MAILBOX_ROLES_CAN_SEND,
|
||||
).exists()
|
||||
if not can_send_as_sender:
|
||||
raise drf_exceptions.PermissionDenied(
|
||||
"You do not have permission to send as this mailbox."
|
||||
)
|
||||
|
||||
# Launch async task for sending the message
|
||||
task = send_message_task.delay(str(message.id), must_archive=must_archive)
|
||||
register_task_owner(task.id, request.user.id)
|
||||
# Pre-generate the Celery task id so we can return it to the caller
|
||||
# while still deferring the actual dispatch to ``transaction.on_commit``
|
||||
# below — the broker must never receive a delivery task for a message
|
||||
# whose finalized state is still uncommitted (or rolled back).
|
||||
task_id = str(uuid.uuid4())
|
||||
|
||||
# --- Finalize ---
|
||||
# Message state should be updated by prepare_outbound_message/send_message
|
||||
# Refresh from DB to get final state (e.g., is_draft=False)
|
||||
message.refresh_from_db()
|
||||
with transaction.atomic():
|
||||
prepared = prepare_outbound_message(
|
||||
mailbox_sender,
|
||||
message,
|
||||
request.data.get("textBody"),
|
||||
request.data.get("htmlBody"),
|
||||
request.user,
|
||||
)
|
||||
if not prepared:
|
||||
raise drf_exceptions.APIException(
|
||||
"Failed to prepare message for sending.",
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
# Update thread stats after un-drafting
|
||||
message.thread.update_stats()
|
||||
register_task_owner(task_id, request.user.id)
|
||||
|
||||
return Response({"task_id": task.id}, status=status.HTTP_200_OK)
|
||||
# Dispatch only once the message's finalized state is durable.
|
||||
transaction.on_commit(
|
||||
lambda: send_message_task.apply_async(
|
||||
args=[str(message.id)],
|
||||
kwargs={"must_archive": must_archive},
|
||||
task_id=task_id,
|
||||
)
|
||||
)
|
||||
|
||||
# --- Finalize ---
|
||||
# Message state was updated by prepare_outbound_message (e.g.
|
||||
# is_draft=False); refresh and update thread stats in the same
|
||||
# transaction so the un-drafting and stats commit atomically.
|
||||
message.refresh_from_db()
|
||||
message.thread.update_stats()
|
||||
|
||||
return Response({"task_id": task_id}, status=status.HTTP_200_OK)
|
||||
|
||||
@@ -10,8 +10,10 @@ creation) and dispatches SMTP delivery asynchronously via Celery.
|
||||
import logging
|
||||
|
||||
from django.core.exceptions import ValidationError as DjangoValidationError
|
||||
from django.db import transaction
|
||||
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from jmap_email import find_headers, parse_email
|
||||
from rest_framework import status
|
||||
from rest_framework.exceptions import PermissionDenied
|
||||
from rest_framework.response import Response
|
||||
@@ -24,7 +26,6 @@ from core.enums import MAILBOX_ROLES_CAN_SEND, ChannelApiKeyScope
|
||||
from core.mda.inbound_create import _create_message_from_inbound
|
||||
from core.mda.outbound import prepare_outbound_message
|
||||
from core.mda.outbound_tasks import send_message_task
|
||||
from core.mda.rfc5322 import EmailParseError, parse_email_message
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -103,72 +104,94 @@ class SubmitRawEmailView(APIView):
|
||||
)
|
||||
|
||||
# Parse to validate structure
|
||||
try:
|
||||
parsed = parse_email_message(raw_mime)
|
||||
except EmailParseError:
|
||||
parsed = parse_email(raw_mime)
|
||||
if parsed is None:
|
||||
return Response(
|
||||
{"detail": "Failed to parse email message."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
# Validate sender matches the mailbox
|
||||
sender_email = (parsed.get("from") or {}).get("email", "")
|
||||
# Validate sender matches the mailbox. A multi-address From
|
||||
# would let the caller pair their authorised mailbox with an
|
||||
# unrelated identity that the receiver may display instead —
|
||||
# the From header must collapse to exactly one entry, the
|
||||
# acting mailbox.
|
||||
from_list = parsed.get("from") or []
|
||||
mailbox_email = str(mailbox)
|
||||
if sender_email.lower() != mailbox_email.lower():
|
||||
if (
|
||||
len(from_list) != 1
|
||||
or (from_list[0].get("email") or "").lower() != mailbox_email.lower()
|
||||
):
|
||||
return Response(
|
||||
{
|
||||
"detail": (
|
||||
f"From header '{sender_email}' does not match"
|
||||
f" mailbox '{mailbox_email}'."
|
||||
)
|
||||
},
|
||||
{"detail": f"From header does not match mailbox '{mailbox_email}'."},
|
||||
status=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
|
||||
# Create thread, contacts, message, and recipients from the parsed email.
|
||||
# is_outbound=True skips blob creation (handled by prepare_outbound_message
|
||||
# with DKIM) and AI features.
|
||||
message = _create_message_from_inbound(
|
||||
recipient_email=mailbox_email,
|
||||
parsed_email=parsed,
|
||||
raw_data=raw_mime,
|
||||
mailbox=mailbox,
|
||||
is_outbound=True,
|
||||
)
|
||||
if not message:
|
||||
# Bcc must travel in the X-Rcpt-To envelope, never in the MIME: a Bcc
|
||||
# header would be signed and delivered visibly to every recipient
|
||||
# (RFC 5322 §3.6.3). Reject rather than silently rewrite caller bytes.
|
||||
if find_headers(parsed, "bcc"):
|
||||
return Response(
|
||||
{"detail": "Failed to create message."},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
{
|
||||
"detail": (
|
||||
"Bcc headers are not allowed; pass blind recipients "
|
||||
"via the X-Rcpt-To envelope."
|
||||
)
|
||||
},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
# Add envelope-only recipients as BCC. _create_message_from_inbound
|
||||
# creates MessageRecipient rows from the MIME To/Cc/Bcc headers, but
|
||||
# true BCC recipients appear only in the envelope (X-Rcpt-To), never
|
||||
# in the MIME headers — that's how BCC works in SMTP.
|
||||
mime_recipients = {
|
||||
e.lower()
|
||||
for e in message.recipients.values_list("contact__email", flat=True)
|
||||
}
|
||||
for addr in recipient_emails:
|
||||
if addr.lower() not in mime_recipients:
|
||||
try:
|
||||
contact, _ = models.Contact.objects.get_or_create(
|
||||
email=addr,
|
||||
mailbox=mailbox,
|
||||
defaults={"name": addr.split("@")[0]},
|
||||
)
|
||||
models.MessageRecipient.objects.get_or_create(
|
||||
message=message,
|
||||
contact=contact,
|
||||
type=models.MessageRecipientTypeChoices.BCC,
|
||||
)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Failed to add BCC recipient (masked)")
|
||||
# Create the message, sign it, and arm the SMTP dispatch atomically.
|
||||
# The whole thing rolls back on any failure (no orphan draft), and the
|
||||
# Celery task is dispatched via ``transaction.on_commit`` so the broker
|
||||
# never receives a delivery task for a message that is still uncommitted
|
||||
# or whose transaction later rolls back.
|
||||
with transaction.atomic():
|
||||
# Create thread, contacts, message, and recipients from the parsed
|
||||
# email. is_outbound=True skips blob creation (handled by
|
||||
# prepare_outbound_message with DKIM) and AI features.
|
||||
message = _create_message_from_inbound(
|
||||
recipient_email=mailbox_email,
|
||||
parsed_email=parsed,
|
||||
raw_data=raw_mime,
|
||||
mailbox=mailbox,
|
||||
is_outbound=True,
|
||||
)
|
||||
if not message:
|
||||
# Roll back so any partial writes from the failed creation
|
||||
# don't commit — returning from inside the atomic block would
|
||||
# otherwise commit them (mirrors the prepare-failure path below).
|
||||
transaction.set_rollback(True)
|
||||
return Response(
|
||||
{"detail": "Failed to create message."},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
# Synchronous: validate recipients, throttle, DKIM sign, create blob.
|
||||
# This is a one-shot API — clean up on any failure so no orphan
|
||||
# draft remains.
|
||||
try:
|
||||
# Add envelope-only recipients as BCC. _create_message_from_inbound
|
||||
# creates MessageRecipient rows from the MIME To/Cc/Bcc headers, but
|
||||
# true BCC recipients appear only in the envelope (X-Rcpt-To), never
|
||||
# in the MIME headers — that's how BCC works in SMTP.
|
||||
mime_recipients = {
|
||||
e.lower()
|
||||
for e in message.recipients.values_list("contact__email", flat=True)
|
||||
}
|
||||
for addr in recipient_emails:
|
||||
if addr.lower() not in mime_recipients:
|
||||
try:
|
||||
contact, _ = models.Contact.objects.get_or_create(
|
||||
email=addr,
|
||||
mailbox=mailbox,
|
||||
defaults={"name": addr.split("@")[0]},
|
||||
)
|
||||
models.MessageRecipient.objects.get_or_create(
|
||||
message=message,
|
||||
contact=contact,
|
||||
type=models.MessageRecipientTypeChoices.BCC,
|
||||
)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Failed to add BCC recipient (masked)")
|
||||
|
||||
# Synchronous: validate recipients, throttle, DKIM sign, create blob.
|
||||
prepared = prepare_outbound_message(
|
||||
mailbox,
|
||||
message,
|
||||
@@ -176,19 +199,18 @@ class SubmitRawEmailView(APIView):
|
||||
"",
|
||||
raw_mime=raw_mime,
|
||||
)
|
||||
except Exception:
|
||||
message.delete()
|
||||
raise
|
||||
if not prepared:
|
||||
# Roll back so no orphan draft survives the failed prepare —
|
||||
# returning from inside the atomic block would otherwise commit
|
||||
# the partially-built message.
|
||||
transaction.set_rollback(True)
|
||||
return Response(
|
||||
{"detail": "Failed to prepare message for sending."},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
if not prepared:
|
||||
message.delete()
|
||||
return Response(
|
||||
{"detail": "Failed to prepare message for sending."},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
# Dispatch async SMTP delivery
|
||||
send_message_task.delay(str(message.id))
|
||||
# Dispatch async SMTP delivery once the message is durably committed.
|
||||
transaction.on_commit(lambda: send_message_task.delay(str(message.id)))
|
||||
|
||||
return Response(
|
||||
{"message_id": str(message.id), "status": "accepted"},
|
||||
|
||||
@@ -21,8 +21,8 @@ from rest_framework.response import Response
|
||||
|
||||
from core import enums, models
|
||||
from core.ai.thread_summarizer import summarize_thread
|
||||
from core.mda.utils import thread_snippet
|
||||
from core.services.search import search_threads
|
||||
from core.utils import extract_snippet
|
||||
|
||||
from .. import permissions, serializers
|
||||
|
||||
@@ -46,7 +46,7 @@ class ThreadViewSet(
|
||||
`refresh_summary` mutates thread state (writes `thread.summary`)
|
||||
so it must also gate on full edit rights, not just authentication.
|
||||
"""
|
||||
if self.action in ("destroy", "split", "refresh_summary"):
|
||||
if self.action in ("destroy", "split", "refresh_summary", "bulk_delete"):
|
||||
return [permissions.HasThreadEditAccess()]
|
||||
return super().get_permissions()
|
||||
|
||||
@@ -702,10 +702,25 @@ class ThreadViewSet(
|
||||
page = int(self.paginator.get_page_number(request, self))
|
||||
page_size = int(self.paginator.get_page_size(request))
|
||||
|
||||
# Scope the search to the user's mailboxes. With an explicit
|
||||
# mailbox_id we already verified access above; without one, fall
|
||||
# back to every mailbox the user can access — never the whole
|
||||
# cluster. An unscoped query leaks hit-totals and content-existence
|
||||
# across mailboxes even though the bodies are access-filtered below.
|
||||
if mailbox_id:
|
||||
search_mailbox_ids = [mailbox_id]
|
||||
else:
|
||||
search_mailbox_ids = [
|
||||
str(mid)
|
||||
for mid in models.MailboxAccess.objects.filter(
|
||||
user=request.user
|
||||
).values_list("mailbox_id", flat=True)
|
||||
]
|
||||
|
||||
# Get search results from OpenSearch
|
||||
results = search_threads(
|
||||
query=search_query,
|
||||
mailbox_ids=[mailbox_id] if mailbox_id else None,
|
||||
mailbox_ids=search_mailbox_ids,
|
||||
filters=es_filters,
|
||||
from_offset=(page - 1) * page_size,
|
||||
size=page_size,
|
||||
@@ -912,7 +927,7 @@ class ThreadViewSet(
|
||||
|
||||
with transaction.atomic():
|
||||
new_subject = split_message.subject or old_thread.subject
|
||||
snippet = extract_snippet(
|
||||
snippet = thread_snippet(
|
||||
split_message.get_parsed_data(),
|
||||
fallback=new_subject or "",
|
||||
)
|
||||
@@ -985,7 +1000,7 @@ class ThreadViewSet(
|
||||
# Recalculate old thread snippet from its most recent remaining message
|
||||
last_remaining = old_thread.messages.order_by("-created_at").first()
|
||||
if last_remaining:
|
||||
old_thread.snippet = extract_snippet(
|
||||
old_thread.snippet = thread_snippet(
|
||||
last_remaining.get_parsed_data(),
|
||||
fallback=old_thread.subject or "",
|
||||
)
|
||||
@@ -1009,83 +1024,106 @@ class ThreadViewSet(
|
||||
)
|
||||
return drf.response.Response(serializer.data, status=status.HTTP_201_CREATED)
|
||||
|
||||
# @extend_schema(
|
||||
# tags=["threads"],
|
||||
# request=inline_serializer(
|
||||
# name="ThreadBulkDeleteRequest",
|
||||
# fields={
|
||||
# "thread_ids": drf_serializers.ListField(
|
||||
# child=drf_serializers.UUIDField(),
|
||||
# required=True,
|
||||
# help_text="List of thread IDs to delete",
|
||||
# ),
|
||||
# },
|
||||
# ),
|
||||
# responses={
|
||||
# 200: OpenApiExample(
|
||||
# "Success Response",
|
||||
# value={"detail": "Successfully deleted 5 threads", "deleted_count": 5},
|
||||
# ),
|
||||
# 400: OpenApiExample(
|
||||
# "Validation Error", value={"detail": "thread_ids must be provided"}
|
||||
# ),
|
||||
# },
|
||||
# description="Delete multiple threads at once by providing a list of thread IDs.",
|
||||
# )
|
||||
# @drf.decorators.action(
|
||||
# detail=False,
|
||||
# methods=["post"],
|
||||
# url_path="bulk-delete",
|
||||
# url_name="bulk-delete",
|
||||
# )
|
||||
# def bulk_delete(self, request):
|
||||
# """Delete multiple threads at once."""
|
||||
# thread_ids = request.data.get("thread_ids", [])
|
||||
@extend_schema(
|
||||
tags=["threads"],
|
||||
request=serializers.ThreadBulkDeleteRequestSerializer,
|
||||
responses={
|
||||
200: OpenApiResponse(
|
||||
response={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"deleted_count": {"type": "integer"},
|
||||
},
|
||||
"required": ["success", "deleted_count"],
|
||||
},
|
||||
description="Messages permanently deleted.",
|
||||
),
|
||||
400: OpenApiResponse(
|
||||
response={
|
||||
"type": "object",
|
||||
"properties": {"detail": {"type": "string"}},
|
||||
},
|
||||
description="Missing or invalid parameters.",
|
||||
),
|
||||
401: OpenApiResponse(
|
||||
description=(
|
||||
"Authentication credentials were not provided or are invalid."
|
||||
),
|
||||
),
|
||||
403: OpenApiResponse(
|
||||
description=(
|
||||
"You do not have permission to delete drafts in one or more "
|
||||
"targeted threads."
|
||||
),
|
||||
),
|
||||
},
|
||||
description=(
|
||||
"Permanently delete (hard-delete) draft messages within the given "
|
||||
"accessible and editable threads. A thread emptied by the deletion is "
|
||||
"removed; otherwise its stats are recomputed."
|
||||
),
|
||||
)
|
||||
@drf.decorators.action(
|
||||
detail=False,
|
||||
methods=["post"],
|
||||
url_path="bulk-delete",
|
||||
url_name="bulk-delete",
|
||||
)
|
||||
def bulk_delete(self, request):
|
||||
"""Permanently delete draft messages in bulk.
|
||||
|
||||
# if not thread_ids:
|
||||
# return drf.response.Response(
|
||||
# {"detail": "thread_ids must be provided"},
|
||||
# status=drf.status.HTTP_400_BAD_REQUEST,
|
||||
# )
|
||||
Unlike the soft-delete flag endpoint, this removes the message rows for
|
||||
good. It targets only the messages matching ``scope`` so reply-draft
|
||||
threads keep their other messages.
|
||||
"""
|
||||
serializer = serializers.ThreadBulkDeleteRequestSerializer(data=request.data)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
scope = serializer.validated_data["scope"]
|
||||
thread_ids = serializer.validated_data["thread_ids"]
|
||||
message_ids = serializer.validated_data["message_ids"]
|
||||
|
||||
# # Get threads the user has access to
|
||||
# # Check if user has delete permission for each thread
|
||||
# threads_to_delete = []
|
||||
# forbidden_threads = []
|
||||
# The scope is constrained to a known key by the serializer, so the
|
||||
# lookup is guaranteed to resolve.
|
||||
scope_filter = (
|
||||
serializers.ThreadBulkDeleteRequestSerializer.BULK_DELETE_SCOPE_FILTERS[
|
||||
scope
|
||||
]
|
||||
)
|
||||
|
||||
# for thread_id in thread_ids:
|
||||
# try:
|
||||
# thread = models.Thread.objects.get(id=thread_id)
|
||||
# # Check if user has permission to delete this thread
|
||||
# try:
|
||||
# self.check_object_permissions(self.request, thread)
|
||||
# except drf.exceptions.PermissionDenied:
|
||||
# forbidden_threads.append(thread_id)
|
||||
# else:
|
||||
# threads_to_delete.append(thread_id)
|
||||
# except models.Thread.DoesNotExist:
|
||||
# # Skip threads that don't exist
|
||||
# pass
|
||||
# Per-thread authorization: restrict to threads the user can fully edit
|
||||
# (EDITOR ThreadAccess + CAN_EDIT MailboxAccess), exactly like the flag
|
||||
# endpoint. The view-level HasThreadEditAccess only gates authentication
|
||||
# for this detail=False action, so the real check is this queryset scope.
|
||||
accessible_thread_ids = models.ThreadAccess.objects.editable_by(
|
||||
request.user
|
||||
).values_list("thread_id", flat=True)
|
||||
|
||||
# if forbidden_threads and not threads_to_delete:
|
||||
# # If all requested threads are forbidden, return 403
|
||||
# return drf.response.Response(
|
||||
# {"detail": "You don't have permission to delete these threads"},
|
||||
# status=drf.status.HTTP_403_FORBIDDEN,
|
||||
# )
|
||||
with transaction.atomic():
|
||||
messages_to_delete = models.Message.objects.filter(
|
||||
thread_id__in=accessible_thread_ids,
|
||||
**scope_filter,
|
||||
)
|
||||
if thread_ids:
|
||||
messages_to_delete = messages_to_delete.filter(thread_id__in=thread_ids)
|
||||
if message_ids:
|
||||
messages_to_delete = messages_to_delete.filter(id__in=message_ids)
|
||||
|
||||
# # Update thread_ids to only include those with proper permissions
|
||||
# accessible_threads = self.get_queryset().filter(id__in=threads_to_delete)
|
||||
affected_thread_ids = set(
|
||||
messages_to_delete.values_list("thread_id", flat=True)
|
||||
)
|
||||
# Count before deletion: the cascade total returned by delete() also
|
||||
# includes related rows (recipients, attachments), not just messages.
|
||||
deleted_count = messages_to_delete.count()
|
||||
messages_to_delete.delete()
|
||||
|
||||
# # Count before deletion
|
||||
# count = accessible_threads.count()
|
||||
# An emptied thread is removed; a thread that still has messages has
|
||||
# its denormalized stats (has_draft, has_trashed, ...) recomputed so
|
||||
# it drops out of the corresponding folder filter.
|
||||
for thread in models.Thread.objects.filter(pk__in=affected_thread_ids):
|
||||
if thread.messages.exists():
|
||||
thread.update_stats()
|
||||
else:
|
||||
thread.delete()
|
||||
|
||||
# # Delete the threads
|
||||
# accessible_threads.delete()
|
||||
|
||||
# return drf.response.Response(
|
||||
# {
|
||||
# "detail": f"Successfully deleted {count} threads",
|
||||
# "deleted_count": count,
|
||||
# }
|
||||
# )
|
||||
return drf.response.Response({"success": True, "deleted_count": deleted_count})
|
||||
|
||||
@@ -41,9 +41,9 @@ class UserViewSet(viewsets.GenericViewSet):
|
||||
permission_classes = [permissions.IsAuthenticated & permissions.IsSelf]
|
||||
|
||||
elif self.action == "list":
|
||||
permission_classes = [
|
||||
permissions.IsSuperUser | permissions.IsMailDomainAdmin
|
||||
]
|
||||
# Fine-grained authorization (super user, domain admin or mailbox
|
||||
# admin of the requested maildomain) is enforced in ``list``.
|
||||
permission_classes = [permissions.IsAuthenticated]
|
||||
else:
|
||||
return super().get_permissions()
|
||||
|
||||
@@ -77,17 +77,25 @@ class UserViewSet(viewsets.GenericViewSet):
|
||||
maildomain_pk = request.query_params.get("maildomain_pk")
|
||||
is_superuser = request.user.is_superuser
|
||||
|
||||
# If not superuser, a maildomain_pk is required and the user must be an admin of that maildomain
|
||||
# If not superuser, a maildomain_pk is required and the user must be an
|
||||
# admin of that maildomain, or an admin of at least one mailbox within
|
||||
# it (so mailbox admins can search users to grant them access).
|
||||
if not is_superuser:
|
||||
if not maildomain_pk:
|
||||
raise drf.exceptions.PermissionDenied(
|
||||
"You do not have permission to perform this action."
|
||||
)
|
||||
if not models.MailDomainAccess.objects.filter(
|
||||
is_domain_admin = models.MailDomainAccess.objects.filter(
|
||||
user=request.user,
|
||||
maildomain_id=maildomain_pk,
|
||||
role=models.MailDomainAccessRoleChoices.ADMIN,
|
||||
).exists():
|
||||
).exists()
|
||||
is_mailbox_admin = models.MailboxAccess.objects.filter(
|
||||
user=request.user,
|
||||
mailbox__domain_id=maildomain_pk,
|
||||
role=models.MailboxRoleChoices.ADMIN,
|
||||
).exists()
|
||||
if not (is_domain_admin or is_mailbox_admin):
|
||||
raise drf.exceptions.PermissionDenied(
|
||||
"You do not have administrative rights for this mail domain."
|
||||
)
|
||||
|
||||
@@ -218,6 +218,7 @@ class ChannelTypes(StrEnum):
|
||||
WIDGET = "widget"
|
||||
API_KEY = "api_key"
|
||||
WEBHOOK = "webhook"
|
||||
CALDAV = "caldav"
|
||||
|
||||
|
||||
class WebhookEvents(StrEnum):
|
||||
@@ -360,3 +361,36 @@ BLACKLISTED_PROXY_IMAGE_MIME_TYPES = [
|
||||
"image/cgm",
|
||||
"image/x-cut",
|
||||
]
|
||||
|
||||
# Attachment types that can be rendered inline by the FilePreview viewer.
|
||||
# Anything outside this set is refused with 415 by the /blob/{id}/preview/
|
||||
# endpoint. SVG and HEIC are intentionally excluded — SVG can carry
|
||||
# JavaScript, HEIC is not natively rendered by browsers.
|
||||
PREVIEWABLE_MIME_TYPES = frozenset(
|
||||
{
|
||||
"image/png",
|
||||
"image/jpeg",
|
||||
"image/gif",
|
||||
"image/webp",
|
||||
"application/pdf",
|
||||
"video/mp4",
|
||||
"video/webm",
|
||||
"audio/mpeg",
|
||||
"audio/ogg",
|
||||
"audio/wav",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class PreviewRefusalCode(StrEnum):
|
||||
"""Machine-readable reason returned with a 415 from /blob/{id}/preview/.
|
||||
|
||||
Lets the frontend react without re-deriving the allowlist client-side:
|
||||
|
||||
- ``SUSPICIOUS``: the declared (previewable) Content-Type doesn't match the
|
||||
detected bytes, so the UI should warn the user.
|
||||
- ``UNSUPPORTED``: the file type simply can't be previewed.
|
||||
"""
|
||||
|
||||
SUSPICIOUS = "suspicious"
|
||||
UNSUPPORTED = "unsupported"
|
||||
|
||||
@@ -18,14 +18,14 @@ import base64
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.core.validators import validate_email
|
||||
|
||||
from jmap_email import compose_email, parse_address
|
||||
|
||||
from core import models
|
||||
from core.mda.outbound import send_outbound_email
|
||||
from core.mda.rfc5322 import compose_email
|
||||
from core.mda.signing import sign_message_dkim
|
||||
from core.mda.utils import current_sent_at
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -73,17 +73,20 @@ class Command(BaseCommand):
|
||||
from_email = options.get("from_email")
|
||||
dry_run = options.get("dry_run", False)
|
||||
|
||||
# Validate email addresses
|
||||
try:
|
||||
validate_email(to_email)
|
||||
except ValidationError as e:
|
||||
raise CommandError(f"Invalid recipient email address: {to_email}") from e
|
||||
# Validate email addresses through the same parser the rest of
|
||||
# the inbound / outbound pipeline uses. ``parse_address`` is
|
||||
# strict by default: ``("", "")`` on anything that isn't a real
|
||||
# addr-spec.
|
||||
_, parsed_to = parse_address(to_email)
|
||||
if not parsed_to:
|
||||
raise CommandError(f"Invalid recipient email address: {to_email}")
|
||||
to_email = parsed_to
|
||||
|
||||
if from_email:
|
||||
try:
|
||||
validate_email(from_email)
|
||||
except ValidationError as e:
|
||||
raise CommandError(f"Invalid sender email address: {from_email}") from e
|
||||
_, parsed_from = parse_address(from_email)
|
||||
if not parsed_from:
|
||||
raise CommandError(f"Invalid sender email address: {from_email}")
|
||||
from_email = parsed_from
|
||||
|
||||
# Get sender mailbox or use minimal setup
|
||||
sender_mailbox = None
|
||||
@@ -97,10 +100,12 @@ class Command(BaseCommand):
|
||||
)
|
||||
maildomain_custom_settings = sender_mailbox.domain.custom_settings or {}
|
||||
except models.Mailbox.DoesNotExist:
|
||||
# Use minimal setup without mailbox
|
||||
# Use minimal setup without mailbox. Log domain only —
|
||||
# the full address is PII and the local part doesn't help
|
||||
# diagnose the missing-mailbox case.
|
||||
logger.warning(
|
||||
"Mailbox with email '%s' not found, sending without DKIM",
|
||||
from_email,
|
||||
"Mailbox not found in domain '%s', sending without DKIM",
|
||||
from_email.split("@", 1)[-1],
|
||||
)
|
||||
else:
|
||||
# Use minimal setup without mailbox
|
||||
@@ -111,8 +116,14 @@ class Command(BaseCommand):
|
||||
sender_mailbox.contact.name if sender_mailbox else None
|
||||
) or from_email.split("@")[0]
|
||||
|
||||
logger.info("Sending email from %s to %s", from_email, to_email)
|
||||
logger.info("Subject: %s", subject)
|
||||
# Domain-only in logs to avoid PII leakage; the full address is
|
||||
# in the recipient model and the MIME envelope for forensics.
|
||||
logger.info(
|
||||
"Sending email from <%s> to <%s>",
|
||||
from_email.split("@", 1)[-1],
|
||||
to_email.split("@", 1)[-1],
|
||||
)
|
||||
logger.info("Subject length: %d", len(subject or ""))
|
||||
|
||||
# Generate MIME ID
|
||||
mime_id = (
|
||||
@@ -120,15 +131,15 @@ class Command(BaseCommand):
|
||||
)
|
||||
mime_id = f"{mime_id}@_lst.{from_email.split('@')[1]}"
|
||||
|
||||
# Generate MIME content
|
||||
mime_data = {
|
||||
"from": [{"name": from_name, "email": from_email}],
|
||||
"to": [{"name": to_email.split("@")[0], "email": to_email}],
|
||||
"cc": [],
|
||||
"subject": subject,
|
||||
"sentAt": current_sent_at(),
|
||||
"textBody": [{"content": body}],
|
||||
"htmlBody": [],
|
||||
"message_id": mime_id,
|
||||
"messageId": [mime_id],
|
||||
}
|
||||
|
||||
# Compose the email
|
||||
|
||||
@@ -8,33 +8,41 @@ from django.conf import settings
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from jmap_email import (
|
||||
JmapEmail,
|
||||
find_header,
|
||||
find_headers,
|
||||
first_address_email,
|
||||
has_header,
|
||||
)
|
||||
|
||||
from core import models
|
||||
from core.enums import (
|
||||
MessageRecipientTypeChoices,
|
||||
MessageTemplateTypeChoices,
|
||||
)
|
||||
from core.mda.outbound import compose_and_sign_mime
|
||||
from core.mda.rfc5322.composer import make_reply_subject
|
||||
from core.mda.replies import reply_subject
|
||||
from core.services.throttle import ThrottleLimitExceeded, ThrottleManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Headers that indicate an automatic message (loop prevention)
|
||||
_PRECEDENCE_VALUES = {"bulk", "list", "junk"}
|
||||
_LOOP_HEADERS = {
|
||||
"x-auto-response-suppress",
|
||||
"x-autoreply",
|
||||
"x-autorespond",
|
||||
"x-loop",
|
||||
"list-id",
|
||||
"list-unsubscribe",
|
||||
"list-post",
|
||||
"list-help",
|
||||
"list-subscribe",
|
||||
"list-owner",
|
||||
"list-archive",
|
||||
"feedback-id",
|
||||
}
|
||||
_LOOP_HEADERS = (
|
||||
"X-Auto-Response-Suppress",
|
||||
"X-Autoreply",
|
||||
"X-Autorespond",
|
||||
"X-Loop",
|
||||
"List-Id",
|
||||
"List-Unsubscribe",
|
||||
"List-Post",
|
||||
"List-Help",
|
||||
"List-Subscribe",
|
||||
"List-Owner",
|
||||
"List-Archive",
|
||||
"Feedback-ID",
|
||||
)
|
||||
|
||||
# Addresses that should never receive autoreplies (RFC 3834 / RFC 5230)
|
||||
_NOREPLY_PATTERNS = re.compile(
|
||||
@@ -51,71 +59,57 @@ def _is_noreply_address(email: str) -> bool:
|
||||
return bool(_NOREPLY_PATTERNS.search(email))
|
||||
|
||||
|
||||
def _is_recipient_explicit(mailbox_email: str, parsed_email_headers: dict) -> bool:
|
||||
def _is_recipient_explicit(mailbox_email: str, parsed_email: JmapEmail) -> bool:
|
||||
"""Check that the mailbox address appears in To or Cc.
|
||||
|
||||
Per RFC 5230 Section 4.5, a vacation responder MUST NOT respond to a
|
||||
message unless the recipient's address is explicitly listed. We only
|
||||
check To and Cc because BCC headers are stripped before delivery — if
|
||||
the mailbox was BCC'd its address won't appear in the received headers,
|
||||
which is exactly the behaviour we want (no autoreply for BCC'd copies).
|
||||
Per RFC 5230 §4.5, a vacation responder MUST NOT respond to a
|
||||
message unless the recipient's address is explicitly listed. We
|
||||
only check To and Cc because BCC headers are stripped before
|
||||
delivery — if the mailbox was BCC'd its address won't appear in
|
||||
the received headers, which is exactly the behaviour we want
|
||||
(no autoreply for BCC'd copies).
|
||||
"""
|
||||
target = mailbox_email.lower()
|
||||
for field in ("to", "cc"):
|
||||
recipients = parsed_email_headers.get(field) or []
|
||||
for recipient in recipients:
|
||||
if isinstance(recipient, dict):
|
||||
if recipient.get("email", "").lower() == target:
|
||||
return True
|
||||
elif isinstance(recipient, str):
|
||||
if recipient.lower() == target:
|
||||
return True
|
||||
for entry in parsed_email.get(field) or []:
|
||||
if isinstance(entry, dict) and (entry.get("email") or "").lower() == target:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_auto_reply_message(headers: dict) -> bool:
|
||||
def _is_auto_reply_message(parsed_email: JmapEmail) -> bool:
|
||||
"""Detect whether the inbound message is itself an automatic reply.
|
||||
|
||||
Checks Auto-Submitted, Precedence, List-Id, X-Auto-Response-Suppress,
|
||||
X-Autoreply, X-Autorespond headers.
|
||||
X-Autoreply, X-Autorespond, and Return-Path bounce indicators.
|
||||
"""
|
||||
if not headers:
|
||||
return False
|
||||
|
||||
# Normalize header keys to lowercase for comparison
|
||||
lower_headers = {k.lower(): v for k, v in headers.items()}
|
||||
|
||||
# Return-Path: empty or <> means bounce (RFC 3834)
|
||||
if "return-path" in lower_headers:
|
||||
return_path = lower_headers["return-path"].strip()
|
||||
if return_path in ("", "<>"):
|
||||
# Return-Path empty or <> means bounce (RFC 3834). Walk every
|
||||
# occurrence so a benign duplicate can't mask a bounce indicator.
|
||||
for return_path in find_headers(parsed_email, "Return-Path"):
|
||||
if return_path.strip() in ("", "<>"):
|
||||
return True
|
||||
|
||||
# Auto-Submitted: anything other than "no" means auto-generated.
|
||||
# RFC 3834 allows parameters after ";" (e.g. "auto-replied; owner-email=...").
|
||||
auto_submitted = lower_headers.get("auto-submitted", "").strip().lower()
|
||||
if auto_submitted:
|
||||
# Strip parameters: "auto-replied; foo=bar" -> "auto-replied"
|
||||
auto_submitted_value = auto_submitted.split(";", 1)[0].strip()
|
||||
if auto_submitted_value and auto_submitted_value != "no":
|
||||
return True
|
||||
|
||||
# Precedence: bulk, list, junk
|
||||
precedence = lower_headers.get("precedence", "").strip().lower()
|
||||
if precedence in _PRECEDENCE_VALUES:
|
||||
# Auto-Submitted (max=1 per RFC 3834 §5). Parameters after ``;``
|
||||
# (e.g. ``auto-replied; owner-email=...``) are stripped before
|
||||
# comparison; anything other than ``no`` counts.
|
||||
auto_submitted = find_header(parsed_email, "Auto-Submitted").strip().lower()
|
||||
if auto_submitted and auto_submitted.split(";", 1)[0].strip() not in ("", "no"):
|
||||
return True
|
||||
|
||||
# Presence of any loop header
|
||||
for header_name in _LOOP_HEADERS:
|
||||
if lower_headers.get(header_name):
|
||||
# Precedence: bulk / list / junk. Repeatable per RFC 5322
|
||||
# (optional-field).
|
||||
for precedence in find_headers(parsed_email, "Precedence"):
|
||||
if precedence.strip().lower() in _PRECEDENCE_VALUES:
|
||||
return True
|
||||
|
||||
return False
|
||||
# Presence of any loop indicator header is enough (list-id,
|
||||
# list-unsubscribe, x-loop, …).
|
||||
return any(has_header(parsed_email, name) for name in _LOOP_HEADERS)
|
||||
|
||||
|
||||
def should_send_autoreply(
|
||||
mailbox: models.Mailbox,
|
||||
parsed_email_headers: dict,
|
||||
parsed_email: JmapEmail,
|
||||
is_spam: bool = False,
|
||||
) -> Optional[models.MessageTemplate]:
|
||||
"""Determine whether we should send an autoreply and return the template.
|
||||
@@ -127,16 +121,12 @@ def should_send_autoreply(
|
||||
if is_spam:
|
||||
return None
|
||||
|
||||
headers = parsed_email_headers.get("headers", {})
|
||||
|
||||
# 2. Skip auto-generated messages (loop prevention)
|
||||
if _is_auto_reply_message(headers):
|
||||
if _is_auto_reply_message(parsed_email):
|
||||
return None
|
||||
|
||||
# 3. Self-reply prevention: skip if sender == mailbox email
|
||||
sender_info = parsed_email_headers.get("from", {})
|
||||
sender_email = sender_info.get("email", "").lower() if sender_info else ""
|
||||
|
||||
sender_email = first_address_email(parsed_email.get("from")).lower()
|
||||
if not sender_email:
|
||||
return None
|
||||
|
||||
@@ -150,7 +140,7 @@ def should_send_autoreply(
|
||||
|
||||
# 3c. RFC 5230 §4.5: only reply if mailbox address appears in To/Cc.
|
||||
# Prevents autoreplies to BCC'd copies and mailing-list expansions.
|
||||
if not _is_recipient_explicit(mailbox_email, parsed_email_headers):
|
||||
if not _is_recipient_explicit(mailbox_email, parsed_email):
|
||||
return None
|
||||
|
||||
# 4. Find active autoreply template for this mailbox
|
||||
@@ -216,7 +206,7 @@ def send_autoreply_for_message(
|
||||
)
|
||||
|
||||
# 2. Build subject with Re: prefix
|
||||
reply_subject = make_reply_subject(inbound_message.subject or "")[:255]
|
||||
subject = reply_subject(inbound_message.subject or "")[:255]
|
||||
|
||||
# 3-7: Create records and compose MIME atomically so a failure in
|
||||
# compose_and_sign_mime does not leave orphan Message/Recipient rows.
|
||||
@@ -225,7 +215,7 @@ def send_autoreply_for_message(
|
||||
message = models.Message.objects.create(
|
||||
thread=thread,
|
||||
sender=mailbox_contact,
|
||||
subject=reply_subject,
|
||||
subject=subject,
|
||||
parent=inbound_message,
|
||||
sent_at=timezone.now(),
|
||||
is_draft=False,
|
||||
@@ -285,7 +275,7 @@ def send_autoreply_for_message(
|
||||
|
||||
def try_send_autoreply(
|
||||
mailbox: models.Mailbox,
|
||||
parsed_email: dict,
|
||||
parsed_email: JmapEmail,
|
||||
message: models.Message,
|
||||
is_spam: bool = False,
|
||||
):
|
||||
@@ -295,16 +285,7 @@ def try_send_autoreply(
|
||||
Exceptions are logged but never propagated.
|
||||
"""
|
||||
try:
|
||||
parsed_headers = {
|
||||
"from": parsed_email.get("from", {}),
|
||||
"to": parsed_email.get("to", []),
|
||||
"cc": parsed_email.get("cc", []),
|
||||
"subject": parsed_email.get("subject", ""),
|
||||
"messageId": parsed_email.get("messageId")
|
||||
or parsed_email.get("message_id"),
|
||||
"headers": parsed_email.get("headers", {}),
|
||||
}
|
||||
template = should_send_autoreply(mailbox, parsed_headers, is_spam=is_spam)
|
||||
template = should_send_autoreply(mailbox, parsed_email, is_spam=is_spam)
|
||||
if template:
|
||||
send_autoreply_for_message(template, mailbox, message)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
|
||||
@@ -3,12 +3,13 @@
|
||||
# pylint: disable=broad-exception-caught
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db.utils import Error as DjangoDbError
|
||||
|
||||
from jmap_email import JmapEmail, first_msgid
|
||||
|
||||
from core import models
|
||||
from core.mda.inbound_tasks import process_inbound_message_task
|
||||
from core.services.importer.labels import (
|
||||
@@ -131,13 +132,13 @@ def count_external_recipients(message) -> int:
|
||||
|
||||
def deliver_inbound_message(
|
||||
recipient_email: str,
|
||||
parsed_email: Dict[str, Any],
|
||||
parsed_email: JmapEmail,
|
||||
raw_data: bytes,
|
||||
is_import: bool = False,
|
||||
is_import_sender: bool = False,
|
||||
imap_labels: Optional[List[str]] = None,
|
||||
imap_flags: Optional[List[str]] = None,
|
||||
channel: Optional[models.Channel] = None,
|
||||
imap_labels: list[str] | None = None,
|
||||
imap_flags: list[str] | None = None,
|
||||
channel: models.Channel | None = None,
|
||||
skip_inbound_queue: bool = False,
|
||||
) -> bool: # Return True on success, False on failure
|
||||
"""Deliver a parsed inbound email message.
|
||||
@@ -160,12 +161,8 @@ def deliver_inbound_message(
|
||||
return False
|
||||
|
||||
# --- 2. Check for Duplicate Message --- #
|
||||
mime_id = parsed_email.get("messageId", parsed_email.get("message_id"))
|
||||
mime_id = first_msgid(parsed_email.get("messageId"))
|
||||
if mime_id:
|
||||
# Remove angle brackets if present
|
||||
if mime_id.startswith("<") and mime_id.endswith(">"):
|
||||
mime_id = mime_id[1:-1]
|
||||
|
||||
# Check if a message with this MIME ID already exists in this mailbox
|
||||
existing_message = models.Message.objects.filter(
|
||||
mime_id=mime_id, thread__accesses__mailbox=mailbox
|
||||
|
||||
@@ -14,8 +14,13 @@ Rules applied for every backend:
|
||||
- If DMARC is absent or passes, DKIM alone decides.
|
||||
|
||||
The backend is picked by ``SPAM_CONFIG["inbound_auth"]``:
|
||||
- ``"native"``: verify DKIM locally (crypto + DNS). DMARC is not yet
|
||||
implemented for native, so only the DKIM rule applies.
|
||||
- ``"native"``: verify DKIM locally (crypto + DNS) AND require the signing
|
||||
``d=`` domain to match the From: domain (strict alignment). Raw DKIM only
|
||||
proves *some* domain signed the message; without alignment an attacker who
|
||||
controls any DKIM-enabled domain could sign a message bearing a forged
|
||||
From:. Full DMARC policy lookup is not implemented for native, so an
|
||||
unaligned-but-cryptographically-valid signature collapses to ``"none"``
|
||||
(we can't call it forgery without the From domain's published policy).
|
||||
- ``"rspamd"``: read DKIM / DMARC symbols from the rspamd /checkv2 result
|
||||
(reused from the spam check, or fetched on demand by the caller).
|
||||
- ``"authentication-results"``: parse ``dkim=`` / ``dmarc=`` entries from the
|
||||
@@ -32,9 +37,12 @@ an explicit DMARC fail.
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any
|
||||
|
||||
from jmap_email import JmapEmail, first_address_email
|
||||
|
||||
from core.mda.signing import verify_message_dkim
|
||||
from core.mda.utils import headers_blocks
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -46,7 +54,7 @@ _NONE = "none" # explicitly no signature / policy
|
||||
|
||||
# Rspamd symbol names -> outcome, per check type.
|
||||
# https://rspamd.com/doc/modules/dkim.html / dmarc
|
||||
_RSPAMD_SYMBOLS: Dict[str, Dict[str, str]] = {
|
||||
_RSPAMD_SYMBOLS: dict[str, dict[str, str]] = {
|
||||
"dkim": {
|
||||
"R_DKIM_ALLOW": _PASS,
|
||||
"R_DKIM_REJECT": _FAIL,
|
||||
@@ -97,16 +105,14 @@ def _scrub_ar_value(value: str) -> str:
|
||||
return _AR_QUOTED_STRING_RE.sub(" ", value)
|
||||
|
||||
|
||||
def _rspamd_outcome(
|
||||
check: str, rspamd_result: Optional[Dict[str, Any]]
|
||||
) -> Optional[str]:
|
||||
def _rspamd_outcome(check: str, rspamd_result: dict[str, Any] | None) -> str | None:
|
||||
if not rspamd_result:
|
||||
return None
|
||||
symbols = rspamd_result.get("symbols") or {}
|
||||
if not isinstance(symbols, dict):
|
||||
return None
|
||||
mapping = _RSPAMD_SYMBOLS.get(check, {})
|
||||
outcome: Optional[str] = None
|
||||
outcome: str | None = None
|
||||
for symbol, result in mapping.items():
|
||||
if symbol not in symbols:
|
||||
continue
|
||||
@@ -121,16 +127,16 @@ def _rspamd_outcome(
|
||||
|
||||
|
||||
def _authentication_results_values(
|
||||
parsed_email: Dict[str, Any], trusted_relays: int
|
||||
) -> List[str]:
|
||||
parsed_email: JmapEmail, trusted_relays: int
|
||||
) -> list[str]:
|
||||
"""Collect Authentication-Results header values from trusted header blocks.
|
||||
|
||||
Block 0 is what we (or our MTA) prepended; blocks 1..N are upstream relays
|
||||
(most recent first). Anything past ``trusted_relays`` is ignored.
|
||||
"""
|
||||
blocks = parsed_email.get("headers_blocks") or []
|
||||
blocks = headers_blocks(parsed_email)
|
||||
blocks_to_check = trusted_relays + 1
|
||||
values: List[str] = []
|
||||
values: list[str] = []
|
||||
for block in blocks[:blocks_to_check]:
|
||||
ar = block.get("authentication-results")
|
||||
if not ar:
|
||||
@@ -142,11 +148,11 @@ def _authentication_results_values(
|
||||
return values
|
||||
|
||||
|
||||
def _ar_outcome(check: str, ar_values: List[str]) -> Optional[str]:
|
||||
def _ar_outcome(check: str, ar_values: list[str]) -> str | None:
|
||||
if not ar_values:
|
||||
return None
|
||||
found = False
|
||||
outcome: Optional[str] = None
|
||||
outcome: str | None = None
|
||||
for value in ar_values:
|
||||
scrubbed = _scrub_ar_value(value)
|
||||
for match in _AR_METHOD_RE.finditer(scrubbed):
|
||||
@@ -163,19 +169,57 @@ def _ar_outcome(check: str, ar_values: List[str]) -> Optional[str]:
|
||||
return outcome if found else None
|
||||
|
||||
|
||||
def _native_dkim_outcome(raw_data: bytes) -> Optional[str]:
|
||||
def _from_header_domain(parsed_email: JmapEmail) -> str | None:
|
||||
"""Return the lowercased domain of the RFC5322 From address, or ``None``."""
|
||||
from_email = first_address_email(parsed_email.get("from"))
|
||||
if not from_email:
|
||||
return None
|
||||
domain = from_email.strip().rstrip(".").lower().rpartition("@")[2]
|
||||
return domain or None
|
||||
|
||||
|
||||
def _native_dkim_outcome(raw_data: bytes, parsed_email: JmapEmail) -> str | None:
|
||||
"""Verify DKIM locally and require From/DKIM identifier alignment.
|
||||
|
||||
A valid DKIM signature only proves that *some* domain signed the message,
|
||||
so we additionally require the signing domain (``d=``) to match the From:
|
||||
domain — strict alignment, an exact case-insensitive match. Without it an
|
||||
attacker who owns any DKIM-enabled domain could sign a message carrying a
|
||||
forged From: and have it shown as verified.
|
||||
|
||||
Native mode never returns ``_FAIL``: it does no DMARC policy lookup, and a
|
||||
bare DKIM verify can't tell a *missing* signature from an *invalid* one, so
|
||||
it has no grounds to assert an explicit failure. Every non-pass outcome —
|
||||
no/invalid signature, or a valid signature whose ``d=`` doesn't align with
|
||||
From — collapses to ``_NONE`` ("unverified"). The unaligned case also logs
|
||||
the mismatch, since a *valid* signature not matching From is the spoofing
|
||||
signature.
|
||||
"""
|
||||
try:
|
||||
return _PASS if verify_message_dkim(raw_data) else _FAIL
|
||||
signing_domain = verify_message_dkim(raw_data)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Native DKIM verification errored: %s", e)
|
||||
return None
|
||||
if not signing_domain:
|
||||
# No signature, or one that didn't validate — a bare verify can't tell
|
||||
# them apart, so this is "can't verify", not an explicit failure.
|
||||
return _NONE
|
||||
from_domain = _from_header_domain(parsed_email)
|
||||
if from_domain and signing_domain == from_domain:
|
||||
return _PASS
|
||||
logger.info(
|
||||
"Native DKIM signature not aligned with From: d=%s from=%s -> unverified",
|
||||
signing_domain,
|
||||
from_domain,
|
||||
)
|
||||
return _NONE
|
||||
|
||||
|
||||
VERDICT_UNVERIFIED = "none"
|
||||
VERDICT_FORGED = "fail"
|
||||
|
||||
|
||||
def get_inbound_auth_mode(spam_config: Dict[str, Any]) -> str:
|
||||
def get_inbound_auth_mode(spam_config: dict[str, Any]) -> str:
|
||||
"""Return the normalized ``inbound_auth`` mode from a spam config.
|
||||
|
||||
Empty or missing values become an empty string. Callers can treat the
|
||||
@@ -187,10 +231,10 @@ def get_inbound_auth_mode(spam_config: Dict[str, Any]) -> str:
|
||||
|
||||
def check_inbound_authentication(
|
||||
raw_data: bytes,
|
||||
parsed_email: Dict[str, Any],
|
||||
spam_config: Dict[str, Any],
|
||||
rspamd_result: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[str]:
|
||||
parsed_email: JmapEmail,
|
||||
spam_config: dict[str, Any],
|
||||
rspamd_result: dict[str, Any] | None = None,
|
||||
) -> str | None:
|
||||
"""Return the ``X-StMsg-Sender-Auth`` verdict for this message.
|
||||
|
||||
See module docstring for the rule set and supported backends.
|
||||
@@ -200,13 +244,13 @@ def check_inbound_authentication(
|
||||
return None
|
||||
|
||||
if mode == "native":
|
||||
dkim = _native_dkim_outcome(raw_data)
|
||||
dmarc: Optional[str] = None
|
||||
dkim = _native_dkim_outcome(raw_data, parsed_email)
|
||||
dmarc: str | None = None
|
||||
elif mode == "rspamd":
|
||||
dkim = _rspamd_outcome("dkim", rspamd_result)
|
||||
dmarc = _rspamd_outcome("dmarc", rspamd_result)
|
||||
elif mode == "authentication-results":
|
||||
trusted_relays = int(spam_config.get("trusted_relays", 1))
|
||||
trusted_relays = int(spam_config.get("trusted_relays", 0))
|
||||
ar_values = _authentication_results_values(parsed_email, trusted_relays)
|
||||
dkim = _ar_outcome("dkim", ar_values)
|
||||
dmarc = _ar_outcome("dmarc", ar_values)
|
||||
|
||||
@@ -4,13 +4,22 @@
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
import uuid
|
||||
from contextlib import contextmanager, nullcontext
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db import transaction
|
||||
from django.db import connection, transaction
|
||||
from django.db.utils import Error as DjangoDbError
|
||||
from django.utils import timezone
|
||||
|
||||
from jmap_email import (
|
||||
JmapEmail,
|
||||
first_address_email,
|
||||
first_address_name,
|
||||
first_msgid,
|
||||
sent_at_to_datetime,
|
||||
)
|
||||
|
||||
from core import enums, models
|
||||
from core.ai.call_label import assign_label_to_thread
|
||||
from core.ai.thread_summarizer import summarize_thread
|
||||
@@ -19,76 +28,98 @@ from core.ai.utils import (
|
||||
is_ai_summary_enabled,
|
||||
is_auto_labels_enabled,
|
||||
)
|
||||
from core.mda.utils import thread_snippet
|
||||
from core.services.importer.labels import (
|
||||
compute_labels_and_flags,
|
||||
)
|
||||
from core.utils import extract_snippet
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Helper function to extract Message-IDs
|
||||
MESSAGE_ID_RE = re.compile(r"<([^<>]+)>")
|
||||
|
||||
TOKEN_THRESHOLD_FOR_SUMMARY = 200 # Minimum token count to trigger summarization
|
||||
MINIMUM_MESSAGES_FOR_SUMMARY = 3 # Minimum number of messages to trigger summarization
|
||||
|
||||
# Advisory-lock namespace for inbound delivery. Distinct ``classid`` from the
|
||||
# blob-cohort locks (see core.services.tiered_storage) so the two never
|
||||
# collide in Postgres' single global advisory-lock keyspace.
|
||||
_ADVISORY_LOCK_CLASSID_INBOUND = 0x696E626E # 'inbn' in ASCII
|
||||
|
||||
|
||||
@contextmanager
|
||||
def inbound_mailbox_lock(mailbox_id: uuid.UUID):
|
||||
"""Serialize inbound message creation for one mailbox.
|
||||
|
||||
Dedup (does a message with this ``mime_id`` already exist?) and
|
||||
thread-bucketing (does a thread already exist for this ``In-Reply-To`` /
|
||||
``References``?) are both read-then-decide: two concurrent inbound
|
||||
deliveries to the same mailbox could each read "nothing there yet" and
|
||||
both create a row, yielding duplicate Messages or two parallel Threads
|
||||
with no reconcile path. Holding a per-mailbox Postgres advisory lock for
|
||||
the duration of the find-or-create makes that critical section
|
||||
cluster-wide serial.
|
||||
|
||||
Must be called inside ``transaction.atomic()`` — ``pg_advisory_xact_lock``
|
||||
binds the lock to the current transaction and releases it on
|
||||
commit/rollback. The lock is held only across the (DB-only) find-or-create
|
||||
work; slow steps (spam scoring, AI summary/labels) run outside it.
|
||||
"""
|
||||
# First 4 bytes of the mailbox UUID as a signed int32 — the two-arg
|
||||
# pg_advisory_xact_lock(classid, objid) form takes two int4s. A 2^-32
|
||||
# false-share collision is operationally invisible given the short,
|
||||
# DB-only critical section it guards.
|
||||
objid = int.from_bytes(mailbox_id.bytes[:4], byteorder="big", signed=True)
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT pg_advisory_xact_lock(%s, %s)",
|
||||
[_ADVISORY_LOCK_CLASSID_INBOUND, objid],
|
||||
)
|
||||
yield
|
||||
|
||||
|
||||
def _canonicalize_subject(subject: str | None) -> str:
|
||||
"""Strip leading ``Re:`` / ``Fwd:`` (and i18n variants) for thread match."""
|
||||
return re.sub(
|
||||
r"^((re|fwd|fw|rep|tr|rép)\s*:\s+)+",
|
||||
"",
|
||||
(subject or "").lower(),
|
||||
flags=re.IGNORECASE,
|
||||
).strip()
|
||||
|
||||
|
||||
def find_thread_for_inbound_message(
|
||||
parsed_email: Dict[str, Any], mailbox: models.Mailbox
|
||||
) -> Optional[models.Thread]:
|
||||
parsed_email: JmapEmail, mailbox: models.Mailbox
|
||||
) -> models.Thread | None:
|
||||
"""Attempt to find an existing thread for an inbound message.
|
||||
|
||||
Follows JMAP spec recommendations:
|
||||
https://www.ietf.org/rfc/rfc8621.html#section-3
|
||||
"""
|
||||
in_reply_to = first_msgid(parsed_email.get("inReplyTo"))
|
||||
references = parsed_email.get("references") or []
|
||||
|
||||
def find_message_ids(txt):
|
||||
# Extract all unique message IDs from a header string
|
||||
return set(MESSAGE_ID_RE.findall(txt or ""))
|
||||
|
||||
def canonicalize_subject(subject):
|
||||
return re.sub(
|
||||
r"^((re|fwd|fw|rep|tr|rép)\s*:\s+)+",
|
||||
"",
|
||||
subject.lower(),
|
||||
flags=re.IGNORECASE,
|
||||
).strip()
|
||||
|
||||
# --- Logic --- #
|
||||
in_reply_to_ids = (
|
||||
{parsed_email.get("in_reply_to")} if parsed_email.get("in_reply_to") else set()
|
||||
)
|
||||
references_ids = find_message_ids(parsed_email.get("headers", {}).get("references"))
|
||||
all_referenced_ids = in_reply_to_ids.union(references_ids)
|
||||
|
||||
# logger.info("All referenced IDs: %s %s", all_referenced_ids, parsed_email)
|
||||
all_referenced_ids = set(references)
|
||||
if in_reply_to:
|
||||
all_referenced_ids.add(in_reply_to)
|
||||
|
||||
if not all_referenced_ids:
|
||||
return None # No headers to match on
|
||||
|
||||
# Prepare a list of IDs without angle brackets for DB query
|
||||
db_query_ids = list(all_referenced_ids)
|
||||
|
||||
# Find potential parent messages in the target mailbox based on references
|
||||
potential_parents = list(
|
||||
models.Message.objects.filter(
|
||||
# Query only for the bracketless IDs
|
||||
mime_id__in=db_query_ids,
|
||||
mime_id__in=list(all_referenced_ids),
|
||||
thread__accesses__mailbox=mailbox,
|
||||
)
|
||||
.select_related("thread")
|
||||
.order_by("-created_at") # Prefer newer matches if multiple found
|
||||
)
|
||||
|
||||
# logger.info("Potential parents: %s", potential_parents)
|
||||
|
||||
if len(potential_parents) == 0:
|
||||
return None # No matching messages found by ID in this mailbox
|
||||
|
||||
# Strategy 1: Match by reference AND canonical subject
|
||||
incoming_subject_canonical = canonicalize_subject(parsed_email.get("subject"))
|
||||
incoming_subject_canonical = _canonicalize_subject(parsed_email.get("subject"))
|
||||
for parent in potential_parents:
|
||||
parent_subject_canonical = canonicalize_subject(parent.subject)
|
||||
parent_subject_canonical = _canonicalize_subject(parent.subject)
|
||||
if incoming_subject_canonical == parent_subject_canonical:
|
||||
return parent.thread # Found a match!
|
||||
|
||||
@@ -98,16 +129,16 @@ def find_thread_for_inbound_message(
|
||||
|
||||
|
||||
def find_thread_for_import(
|
||||
parsed_email: Dict[str, Any], mailbox: models.Mailbox
|
||||
) -> Optional[models.Thread]:
|
||||
parsed_email: JmapEmail, mailbox: models.Mailbox
|
||||
) -> models.Thread | None:
|
||||
"""
|
||||
During import, try to find an existing thread that contains messages
|
||||
with the same subject or referenced message IDs.
|
||||
"""
|
||||
|
||||
subject = parsed_email.get("subject", "")
|
||||
in_reply_to = parsed_email.get("in_reply_to")
|
||||
references = parsed_email.get("headers", {}).get("references", "")
|
||||
in_reply_to = first_msgid(parsed_email.get("inReplyTo"))
|
||||
references = parsed_email.get("references") or []
|
||||
|
||||
# First try to find a thread by message IDs
|
||||
thread = _find_thread_by_message_ids(in_reply_to, references, mailbox)
|
||||
@@ -115,12 +146,7 @@ def find_thread_for_import(
|
||||
# If no thread found by message IDs, try by subject
|
||||
if not thread and subject:
|
||||
# Look for threads with similar subjects
|
||||
canonical_subject = re.sub(
|
||||
r"^((re|fwd|fw|rep|tr|rép)\s*:\s+)+",
|
||||
"",
|
||||
subject.lower(),
|
||||
flags=re.IGNORECASE,
|
||||
).strip()
|
||||
canonical_subject = _canonicalize_subject(subject)
|
||||
thread = models.Thread.objects.filter(
|
||||
subject__iregex=rf"^(re|fwd|fw|rep|tr|rép)\s*:\s*{re.escape(canonical_subject)}$",
|
||||
accesses__mailbox=mailbox,
|
||||
@@ -129,12 +155,10 @@ def find_thread_for_import(
|
||||
return thread
|
||||
|
||||
|
||||
def _create_thread(
|
||||
parsed_email: Dict[str, Any], mailbox: models.Mailbox
|
||||
) -> models.Thread:
|
||||
def _create_thread(parsed_email: JmapEmail, mailbox: models.Mailbox) -> models.Thread:
|
||||
"""Create a new thread."""
|
||||
|
||||
snippet = extract_snippet(
|
||||
snippet = thread_snippet(
|
||||
parsed_email,
|
||||
fallback=parsed_email.get("subject") or "(No snippet available)",
|
||||
)
|
||||
@@ -159,40 +183,36 @@ def _create_thread(
|
||||
|
||||
|
||||
def _find_thread_by_message_ids(
|
||||
in_reply_to: str, references: str, mailbox: models.Mailbox
|
||||
) -> Optional[models.Thread]:
|
||||
"""Find thread by message IDs (in_reply_to and references)."""
|
||||
# First try to find a thread by message IDs
|
||||
in_reply_to: str, references: list[str], mailbox: models.Mailbox
|
||||
) -> models.Thread | None:
|
||||
"""Find thread by message IDs (``inReplyTo`` and ``references``)."""
|
||||
if in_reply_to or references:
|
||||
thread = models.Thread.objects.filter(
|
||||
messages__mime_id__in=[in_reply_to] if in_reply_to else [],
|
||||
accesses__mailbox=mailbox,
|
||||
).first()
|
||||
if not thread and references:
|
||||
# Extract message IDs from references
|
||||
ref_ids = MESSAGE_ID_RE.findall(references)
|
||||
if ref_ids:
|
||||
thread = models.Thread.objects.filter(
|
||||
messages__mime_id__in=ref_ids,
|
||||
accesses__mailbox=mailbox,
|
||||
).first()
|
||||
thread = models.Thread.objects.filter(
|
||||
messages__mime_id__in=references,
|
||||
accesses__mailbox=mailbox,
|
||||
).first()
|
||||
return thread
|
||||
return None
|
||||
|
||||
|
||||
def _create_message_from_inbound( # pylint: disable=too-many-arguments
|
||||
recipient_email: str,
|
||||
parsed_email: Dict[str, Any],
|
||||
parsed_email: JmapEmail,
|
||||
raw_data: bytes,
|
||||
mailbox: models.Mailbox,
|
||||
is_import: bool = False,
|
||||
is_import_sender: bool = False,
|
||||
imap_labels: Optional[List[str]] = None,
|
||||
imap_flags: Optional[List[str]] = None,
|
||||
channel: Optional[models.Channel] = None,
|
||||
imap_labels: list[str] | None = None,
|
||||
imap_flags: list[str] | None = None,
|
||||
channel: models.Channel | None = None,
|
||||
is_spam: bool = False,
|
||||
is_outbound: bool = False,
|
||||
) -> Optional[models.Message]:
|
||||
) -> models.Message | None:
|
||||
"""Create a message and thread from parsed email data.
|
||||
|
||||
Used for inbound delivery, imports, and outbound submission.
|
||||
@@ -210,223 +230,266 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments
|
||||
# pylint: disable=too-many-locals,too-many-branches,too-many-statements
|
||||
message_flags = {}
|
||||
|
||||
# --- 3. Find or Create Thread --- #
|
||||
try:
|
||||
thread = None
|
||||
if is_import:
|
||||
thread = find_thread_for_import(parsed_email, mailbox)
|
||||
mime_id = first_msgid(parsed_email.get("messageId")) or None
|
||||
|
||||
# If no thread found or not an import, use normal thread finding logic
|
||||
if not thread:
|
||||
thread = find_thread_for_inbound_message(parsed_email, mailbox)
|
||||
|
||||
if not thread:
|
||||
thread = _create_thread(parsed_email, mailbox)
|
||||
|
||||
except (DjangoDbError, ValidationError) as e:
|
||||
logger.error("Failed to find or create thread for %s: %s", recipient_email, e)
|
||||
return None # Indicate failure
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Unexpected error finding/creating thread for %s: %s",
|
||||
recipient_email,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
|
||||
if is_import:
|
||||
# get labels from parsed_email
|
||||
labels, message_flags = compute_labels_and_flags(
|
||||
parsed_email, imap_labels, imap_flags
|
||||
)
|
||||
for label in labels:
|
||||
try:
|
||||
label_obj, _ = models.Label.objects.get_or_create(
|
||||
name=label, mailbox=mailbox
|
||||
)
|
||||
thread.labels.add(label_obj)
|
||||
except Exception as e:
|
||||
logger.exception("Error creating label %s: %s", label, e)
|
||||
continue
|
||||
|
||||
# Apply labels from channel settings (e.g., widget channel tags)
|
||||
if channel and channel.settings:
|
||||
channel_tags = channel.settings.get("tags", [])
|
||||
for tag_id in channel_tags:
|
||||
try:
|
||||
label_obj = models.Label.objects.get(id=tag_id, mailbox=mailbox)
|
||||
thread.labels.add(label_obj)
|
||||
except models.Label.DoesNotExist:
|
||||
logger.warning(
|
||||
"Label %s not found for channel %s, skipping", tag_id, channel.id
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error adding label %s from channel: %s", tag_id, e)
|
||||
|
||||
# --- 4. Get or Create Sender Contact --- #
|
||||
sender_info = parsed_email.get("from", {})
|
||||
sender_email = sender_info.get("email")
|
||||
sender_name = sender_info.get("name")
|
||||
|
||||
if not sender_email:
|
||||
logger.warning(
|
||||
"Inbound message for %s missing 'From' email, using fallback.",
|
||||
recipient_email,
|
||||
)
|
||||
sender_email = f"unknown-sender@{mailbox.domain.name}" # Use recipient's domain
|
||||
sender_name = sender_name or "Unknown Sender"
|
||||
|
||||
try:
|
||||
# Validate sender_email format before saving
|
||||
models.Contact(email=sender_email).full_clean(
|
||||
exclude=["mailbox", "name"]
|
||||
) # Validate email format
|
||||
|
||||
sender_contact, created = models.Contact.objects.get_or_create(
|
||||
email=sender_email,
|
||||
mailbox=mailbox, # Associate contact with the recipient mailbox
|
||||
defaults={
|
||||
"name": sender_name or sender_email.split("@")[0],
|
||||
"email": sender_email, # Ensure correct casing is saved
|
||||
},
|
||||
)
|
||||
if created:
|
||||
logger.info(
|
||||
"Created contact for sender %s in mailbox %s", sender_email, mailbox.id
|
||||
)
|
||||
|
||||
except ValidationError as e:
|
||||
logger.error(
|
||||
"Validation error for sender contact %s in mailbox %s: %s. Using fallback.",
|
||||
sender_email,
|
||||
mailbox.id,
|
||||
e,
|
||||
)
|
||||
# Fallback: Use a generic placeholder contact if validation fails
|
||||
sender_email = f"invalid-sender@{mailbox.domain.name}"
|
||||
sender_name = "Invalid Sender Address"
|
||||
sender_contact, _ = models.Contact.objects.get_or_create(
|
||||
email=sender_email,
|
||||
mailbox=mailbox,
|
||||
defaults={"name": sender_name, "email": sender_email},
|
||||
)
|
||||
except DjangoDbError as e:
|
||||
logger.error(
|
||||
"DB error getting/creating sender contact %s in mailbox %s: %s",
|
||||
sender_email,
|
||||
mailbox.id,
|
||||
e,
|
||||
)
|
||||
return None # Indicate failure
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Unexpected error with sender contact %s in mailbox %s: %s",
|
||||
sender_email,
|
||||
mailbox.id,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
|
||||
# --- 5. Create Message --- #
|
||||
try:
|
||||
# Can we get a parent message for reference?
|
||||
# TODO: validate this doesn't create security issues
|
||||
parent_message = None
|
||||
if parsed_email.get("in_reply_to"):
|
||||
parent_message = models.Message.objects.filter(
|
||||
mime_id=parsed_email.get("in_reply_to"), thread=thread
|
||||
# Dedup, thread-bucketing and the message INSERT form one read-then-write
|
||||
# critical section. Serialize it per mailbox under a Postgres advisory lock
|
||||
# (held only across this DB-only work) so concurrent inbound deliveries to
|
||||
# the same mailbox cannot create duplicate Messages or split a conversation
|
||||
# into two parallel Threads. Imports are a single-writer backfill path and
|
||||
# skip the lock to avoid serializing bulk loads.
|
||||
lock_ctx = nullcontext() if is_import else inbound_mailbox_lock(mailbox.id)
|
||||
with transaction.atomic(), lock_ctx:
|
||||
# Recheck for an already-stored copy now that we hold the lock.
|
||||
# deliver_inbound_message dedups before queueing, but the async
|
||||
# processing path and concurrent deliveries can still reach here twice
|
||||
# for the same Message-ID; this makes creation idempotent per
|
||||
# (mailbox, mime_id).
|
||||
if not is_outbound and mime_id:
|
||||
existing_message = models.Message.objects.filter(
|
||||
mime_id=mime_id, thread__accesses__mailbox=mailbox
|
||||
).first()
|
||||
if existing_message:
|
||||
logger.info(
|
||||
"Duplicate inbound message %s (MIME ID: %s) in mailbox %s; "
|
||||
"skipping create",
|
||||
existing_message.id,
|
||||
mime_id,
|
||||
mailbox.id,
|
||||
)
|
||||
return existing_message
|
||||
|
||||
# Truncate subject to 255 characters if it exceeds max_length
|
||||
subject = parsed_email.get("subject")
|
||||
if subject and len(subject) > 255:
|
||||
subject = subject[:255]
|
||||
# --- 3. Find or Create Thread --- #
|
||||
try:
|
||||
thread = None
|
||||
if is_import:
|
||||
thread = find_thread_for_import(parsed_email, mailbox)
|
||||
|
||||
is_sender = is_outbound or (is_import and is_import_sender)
|
||||
# If no thread found or not an import, use normal thread finding logic
|
||||
if not thread:
|
||||
thread = find_thread_for_inbound_message(parsed_email, mailbox)
|
||||
|
||||
# The Blob INSERT and the Message INSERT must commit together
|
||||
# so the GC sweep never sees the Blob row without its
|
||||
# referencing FK on ``Message.blob``. Outbound messages have
|
||||
# no blob yet — ``prepare_outbound_message`` adds it later.
|
||||
with transaction.atomic():
|
||||
blob = None
|
||||
if not is_outbound:
|
||||
blob = models.Blob.objects.create_blob(
|
||||
content=raw_data,
|
||||
content_type="message/rfc822",
|
||||
if not thread:
|
||||
thread = _create_thread(parsed_email, mailbox)
|
||||
|
||||
except (DjangoDbError, ValidationError) as e:
|
||||
logger.error(
|
||||
"Failed to find or create thread for %s: %s", recipient_email, e
|
||||
)
|
||||
# Returning from inside the atomic block would commit any partial
|
||||
# writes (e.g. a thread without its message); roll back instead.
|
||||
transaction.set_rollback(True)
|
||||
return None # Indicate failure
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Unexpected error finding/creating thread for %s: %s",
|
||||
recipient_email,
|
||||
e,
|
||||
)
|
||||
transaction.set_rollback(True)
|
||||
return None
|
||||
|
||||
if is_import:
|
||||
# get labels from parsed_email
|
||||
labels, message_flags = compute_labels_and_flags(
|
||||
parsed_email, imap_labels, imap_flags
|
||||
)
|
||||
for label in labels:
|
||||
try:
|
||||
label_obj, _ = models.Label.objects.get_or_create(
|
||||
name=label, mailbox=mailbox
|
||||
)
|
||||
thread.labels.add(label_obj)
|
||||
except Exception as e:
|
||||
logger.exception("Error creating label %s: %s", label, e)
|
||||
continue
|
||||
|
||||
# Apply labels from channel settings (e.g., widget channel tags)
|
||||
if channel and channel.settings:
|
||||
channel_tags = channel.settings.get("tags", [])
|
||||
for tag_id in channel_tags:
|
||||
try:
|
||||
label_obj = models.Label.objects.get(id=tag_id, mailbox=mailbox)
|
||||
thread.labels.add(label_obj)
|
||||
except models.Label.DoesNotExist:
|
||||
logger.warning(
|
||||
"Label %s not found for channel %s, skipping",
|
||||
tag_id,
|
||||
channel.id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Error adding label %s from channel: %s", tag_id, e
|
||||
)
|
||||
|
||||
# --- 4. Get or Create Sender Contact --- #
|
||||
sender_email = first_address_email(parsed_email.get("from"))
|
||||
sender_name = first_address_name(parsed_email.get("from"))
|
||||
|
||||
if not sender_email:
|
||||
logger.warning(
|
||||
"Inbound message for %s missing 'From' email, using fallback.",
|
||||
recipient_email,
|
||||
)
|
||||
sender_email = (
|
||||
f"unknown-sender@{mailbox.domain.name}" # Use recipient's domain
|
||||
)
|
||||
sender_name = sender_name or "Unknown Sender"
|
||||
|
||||
try:
|
||||
# Validate sender_email format before saving
|
||||
models.Contact(email=sender_email).full_clean(
|
||||
exclude=["mailbox", "name"]
|
||||
) # Validate email format
|
||||
|
||||
sender_contact, created = models.Contact.objects.get_or_create(
|
||||
email=sender_email,
|
||||
mailbox=mailbox, # Associate contact with the recipient mailbox
|
||||
defaults={
|
||||
"name": sender_name or sender_email.split("@")[0],
|
||||
"email": sender_email, # Ensure correct casing is saved
|
||||
},
|
||||
)
|
||||
if created:
|
||||
logger.info(
|
||||
"Created contact for sender %s in mailbox %s",
|
||||
sender_email,
|
||||
mailbox.id,
|
||||
)
|
||||
|
||||
message = models.Message.objects.create(
|
||||
thread=thread,
|
||||
sender=sender_contact,
|
||||
subject=subject,
|
||||
blob=blob,
|
||||
mime_id=parsed_email.get("messageId", parsed_email.get("message_id"))
|
||||
or None,
|
||||
parent=parent_message,
|
||||
sent_at=(
|
||||
None
|
||||
if is_outbound
|
||||
else (parsed_email.get("date") or timezone.now())
|
||||
),
|
||||
is_draft=is_outbound, # Outbound: draft until prepare_outbound_message finalizes
|
||||
is_sender=is_sender,
|
||||
is_trashed=False,
|
||||
is_spam=is_spam,
|
||||
has_attachments=len(parsed_email.get("attachments", [])) > 0,
|
||||
channel=channel,
|
||||
except ValidationError as e:
|
||||
logger.error(
|
||||
"Validation error for sender contact %s in mailbox %s: %s. Using fallback.",
|
||||
sender_email,
|
||||
mailbox.id,
|
||||
e,
|
||||
)
|
||||
if is_import:
|
||||
# We need to set the created_at field to the date of the message
|
||||
# because the inbound message is not created at the same time as the message is received
|
||||
message.created_at = parsed_email.get("date") or timezone.now()
|
||||
# Extract flags handled via ThreadAccess (not Message fields)
|
||||
import_is_unread = message_flags.pop("is_unread", True)
|
||||
import_is_starred = message_flags.pop("_starred", False)
|
||||
# Fallback: Use a generic placeholder contact if validation fails
|
||||
sender_email = f"invalid-sender@{mailbox.domain.name}"
|
||||
sender_name = "Invalid Sender Address"
|
||||
sender_contact, _ = models.Contact.objects.get_or_create(
|
||||
email=sender_email,
|
||||
mailbox=mailbox,
|
||||
defaults={"name": sender_name, "email": sender_email},
|
||||
)
|
||||
except DjangoDbError as e:
|
||||
logger.error(
|
||||
"DB error getting/creating sender contact %s in mailbox %s: %s",
|
||||
sender_email,
|
||||
mailbox.id,
|
||||
e,
|
||||
)
|
||||
transaction.set_rollback(True)
|
||||
return None # Indicate failure
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Unexpected error with sender contact %s in mailbox %s: %s",
|
||||
sender_email,
|
||||
mailbox.id,
|
||||
e,
|
||||
)
|
||||
transaction.set_rollback(True)
|
||||
return None
|
||||
|
||||
for flag, value in message_flags.items():
|
||||
if hasattr(message, flag):
|
||||
setattr(message, flag, value)
|
||||
message.save(
|
||||
update_fields=[
|
||||
"created_at",
|
||||
*message_flags.keys(),
|
||||
]
|
||||
)
|
||||
# Update ThreadAccess for read/starred state
|
||||
access = models.ThreadAccess.objects.filter(
|
||||
thread=thread, mailbox=mailbox
|
||||
).first()
|
||||
if access:
|
||||
update_fields = []
|
||||
# Sent messages are always considered read by the sender
|
||||
if (is_sender or not import_is_unread) and (
|
||||
access.read_at is None or message.created_at > access.read_at
|
||||
):
|
||||
# --- 5. Create Message --- #
|
||||
try:
|
||||
# Can we get a parent message for reference?
|
||||
# TODO: validate this doesn't create security issues
|
||||
parent_message = None
|
||||
parent_msg_id = first_msgid(parsed_email.get("inReplyTo"))
|
||||
if parent_msg_id:
|
||||
parent_message = models.Message.objects.filter(
|
||||
mime_id=parent_msg_id, thread=thread
|
||||
).first()
|
||||
|
||||
# Truncate subject to 255 characters if it exceeds max_length
|
||||
subject = parsed_email.get("subject")
|
||||
if subject and len(subject) > 255:
|
||||
subject = subject[:255]
|
||||
|
||||
is_sender = is_outbound or (is_import and is_import_sender)
|
||||
sent_at = sent_at_to_datetime(parsed_email.get("sentAt"))
|
||||
|
||||
# The Blob INSERT and the Message INSERT must commit together
|
||||
# so the GC sweep never sees the Blob row without its
|
||||
# referencing FK on ``Message.blob``. Outbound messages have
|
||||
# no blob yet — ``prepare_outbound_message`` adds it later.
|
||||
with transaction.atomic():
|
||||
blob = None
|
||||
if not is_outbound:
|
||||
blob = models.Blob.objects.create_blob(
|
||||
content=raw_data,
|
||||
content_type="message/rfc822",
|
||||
)
|
||||
|
||||
message = models.Message.objects.create(
|
||||
thread=thread,
|
||||
sender=sender_contact,
|
||||
subject=subject,
|
||||
blob=blob,
|
||||
mime_id=first_msgid(parsed_email.get("messageId")) or None,
|
||||
parent=parent_message,
|
||||
sent_at=(None if is_outbound else (sent_at or timezone.now())),
|
||||
is_draft=is_outbound, # Outbound: draft until prepare_outbound_message finalizes
|
||||
is_sender=is_sender,
|
||||
is_trashed=False,
|
||||
is_spam=is_spam,
|
||||
has_attachments=len(parsed_email.get("attachments", [])) > 0,
|
||||
channel=channel,
|
||||
)
|
||||
if is_import:
|
||||
# We need to set the created_at field to the date of the message
|
||||
# because the inbound message is not created at the same time as the message is received
|
||||
message.created_at = sent_at or timezone.now()
|
||||
# Extract flags handled via ThreadAccess (not Message fields)
|
||||
import_is_unread = message_flags.pop("is_unread", True)
|
||||
import_is_starred = message_flags.pop("_starred", False)
|
||||
|
||||
for flag, value in message_flags.items():
|
||||
if hasattr(message, flag):
|
||||
setattr(message, flag, value)
|
||||
message.save(
|
||||
update_fields=[
|
||||
"created_at",
|
||||
*message_flags.keys(),
|
||||
]
|
||||
)
|
||||
# Update ThreadAccess for read/starred state
|
||||
access = models.ThreadAccess.objects.filter(
|
||||
thread=thread, mailbox=mailbox
|
||||
).first()
|
||||
if access:
|
||||
update_fields = []
|
||||
# Sent messages are always considered read by the sender
|
||||
if (is_sender or not import_is_unread) and (
|
||||
access.read_at is None or message.created_at > access.read_at
|
||||
):
|
||||
access.read_at = message.created_at
|
||||
update_fields.append("read_at")
|
||||
if import_is_starred and access.starred_at is None:
|
||||
access.starred_at = message.created_at
|
||||
update_fields.append("starred_at")
|
||||
if update_fields:
|
||||
access.save(update_fields=update_fields)
|
||||
elif is_sender:
|
||||
access = models.ThreadAccess.objects.filter(
|
||||
thread=thread, mailbox=mailbox
|
||||
).first()
|
||||
if access:
|
||||
access.read_at = message.created_at
|
||||
update_fields.append("read_at")
|
||||
if import_is_starred and access.starred_at is None:
|
||||
access.starred_at = message.created_at
|
||||
update_fields.append("starred_at")
|
||||
if update_fields:
|
||||
access.save(update_fields=update_fields)
|
||||
elif is_sender:
|
||||
access = models.ThreadAccess.objects.filter(
|
||||
thread=thread, mailbox=mailbox
|
||||
).first()
|
||||
if access:
|
||||
access.read_at = message.created_at
|
||||
access.save(update_fields=["read_at"])
|
||||
except (DjangoDbError, ValidationError) as e:
|
||||
logger.error("Failed to create message in thread %s: %s", thread.id, e)
|
||||
return None # Indicate failure
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Unexpected error creating message in thread %s: %s",
|
||||
thread.id,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
access.save(update_fields=["read_at"])
|
||||
except (DjangoDbError, ValidationError) as e:
|
||||
logger.error("Failed to create message in thread %s: %s", thread.id, e)
|
||||
transaction.set_rollback(True)
|
||||
return None # Indicate failure
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Unexpected error creating message in thread %s: %s",
|
||||
thread.id,
|
||||
e,
|
||||
)
|
||||
transaction.set_rollback(True)
|
||||
return None
|
||||
|
||||
# --- 6. Create Recipient Contacts and Links --- #
|
||||
# deduplicate recipients
|
||||
@@ -439,7 +502,7 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments
|
||||
recipients = list(
|
||||
{
|
||||
frozenset(recipient.items())
|
||||
for recipient in parsed_email.get(type_name, [])
|
||||
for recipient in (parsed_email.get(type_name) or [])
|
||||
}
|
||||
)
|
||||
recipient_types_to_process.append(
|
||||
@@ -519,7 +582,7 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments
|
||||
try:
|
||||
# Update snippet using the new message's body if possible
|
||||
# (This assumes the subject was used for the initial snippet if body was empty)
|
||||
new_snippet = extract_snippet(
|
||||
new_snippet = thread_snippet(
|
||||
parsed_email,
|
||||
fallback=parsed_email.get("subject", ""),
|
||||
)
|
||||
@@ -572,7 +635,7 @@ def _create_message_from_inbound( # pylint: disable=too-many-arguments
|
||||
|
||||
|
||||
# def _process_attachments(
|
||||
# message: models.Message, attachment_data: List[Dict], mailbox: models.Mailbox
|
||||
# message: models.Message, attachment_data: list[Dict], mailbox: models.Mailbox
|
||||
# ) -> None:
|
||||
# """
|
||||
# Process attachments found during email parsing.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# pylint: disable=unused-argument, broad-exception-raised, broad-exception-caught, too-many-lines
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
@@ -11,6 +11,12 @@ from django.utils import timezone
|
||||
|
||||
import requests
|
||||
from celery.utils.log import get_task_logger
|
||||
from jmap_email import (
|
||||
JmapEmail,
|
||||
first_address_email,
|
||||
has_header,
|
||||
parse_email,
|
||||
)
|
||||
|
||||
from core import models
|
||||
from core.mda.inbound_auth import (
|
||||
@@ -18,14 +24,14 @@ from core.mda.inbound_auth import (
|
||||
get_inbound_auth_mode,
|
||||
)
|
||||
from core.mda.inbound_create import _create_message_from_inbound
|
||||
from core.mda.rfc5322 import parse_email_message
|
||||
from core.mda.utils import headers_blocks
|
||||
|
||||
from messages.celery_app import app as celery_app
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def _is_selfcheck_message(parsed_email: Dict[str, Any], recipient_email: str) -> bool:
|
||||
def _is_selfcheck_message(parsed_email: JmapEmail, recipient_email: str) -> bool:
|
||||
"""Return True when this message is the self-check probe.
|
||||
|
||||
Match is strict on both envelope ends: the From address must equal
|
||||
@@ -37,7 +43,7 @@ def _is_selfcheck_message(parsed_email: Dict[str, Any], recipient_email: str) ->
|
||||
if not selfcheck_from or not selfcheck_to:
|
||||
return False
|
||||
|
||||
from_email = ((parsed_email.get("from") or {}).get("email") or "").strip().lower()
|
||||
from_email = first_address_email(parsed_email.get("from")).strip().lower()
|
||||
if from_email != selfcheck_from:
|
||||
return False
|
||||
|
||||
@@ -45,8 +51,8 @@ def _is_selfcheck_message(parsed_email: Dict[str, Any], recipient_email: str) ->
|
||||
|
||||
|
||||
def _check_spam_with_hardcoded_rules(
|
||||
parsed_email: Dict[str, Any], spam_config: Dict[str, Any]
|
||||
) -> Optional[bool]:
|
||||
parsed_email: JmapEmail, spam_config: dict[str, Any]
|
||||
) -> bool | None:
|
||||
"""Check if a message is spam using hardcoded rules.
|
||||
|
||||
Args:
|
||||
@@ -57,7 +63,6 @@ def _check_spam_with_hardcoded_rules(
|
||||
is_spam: True if the message is spam, False otherwise. None if no rules matched.
|
||||
"""
|
||||
rules = spam_config.get("rules", [])
|
||||
headers = parsed_email.get("headers", {})
|
||||
|
||||
for rule in rules:
|
||||
if rule.get("header_match") or rule.get("header_match_regex"):
|
||||
@@ -73,33 +78,45 @@ def _check_spam_with_hardcoded_rules(
|
||||
key = key.lower().strip()
|
||||
value = value.lower().strip()
|
||||
|
||||
# Get header value(s) - can be a string or list
|
||||
header_value = headers.get(key)
|
||||
if header_value is None:
|
||||
# Existence check first — the actual value is read from
|
||||
# ``headersBlocks`` below to apply the trusted-relays cut.
|
||||
if not has_header(parsed_email, key):
|
||||
continue
|
||||
|
||||
# Use headers_blocks to identify which headers to trust based on trusted_relays config.
|
||||
# Each block ends with a Received header, marking everything above it as trusted.
|
||||
# Block 0: headers before first Received (ours from MTA), ending with first Received
|
||||
# Block 1: headers between first and second Received, ending with second Received (relay 1)
|
||||
# Block 2+: headers after second Received, ending with third Received (relay 2+)
|
||||
headers_blocks = parsed_email.get("headers_blocks", [])
|
||||
# Use ``ext.headersBlocks`` to identify which headers to
|
||||
# trust based on the trusted_relays config. Each block ends
|
||||
# with a Received header, marking everything above it as
|
||||
# trusted.
|
||||
# Block 0: headers before first Received (ours from MTA),
|
||||
# ending with first Received.
|
||||
# Block 1: headers between first and second Received,
|
||||
# ending with second Received (relay 1).
|
||||
# Block 2+: headers after second Received, ending with
|
||||
# third Received (relay 2+).
|
||||
blocks = headers_blocks(parsed_email)
|
||||
|
||||
# Get number of trusted relays (default: 1, meaning we trust block 0 and block 1)
|
||||
trusted_relays = spam_config.get("trusted_relays", 1)
|
||||
# Number of blocks to check: block 0 (before our Received) + trusted_relays blocks
|
||||
# Default trusted_relays = 0: trust only block 0 (the Received our
|
||||
# own MTA prepends, plus the headers above it). A sender can prepend
|
||||
# their own Received lines, which land in block 1+ — trusting those
|
||||
# by default would let them slip a forged header (e.g. an
|
||||
# action="ham" allowlist match) into the trusted slice. Operators
|
||||
# with real upstream relays opt in by setting trusted_relays to the
|
||||
# number of hops they actually control.
|
||||
trusted_relays = spam_config.get("trusted_relays", 0)
|
||||
# block 0 (our Received) + trusted_relays upstream blocks.
|
||||
blocks_to_check = trusted_relays + 1
|
||||
|
||||
# Check only the trusted blocks (slicing beyond list length just returns all blocks)
|
||||
# Blocks are ordered from most recent to oldest, so we want the first match (most recent)
|
||||
# Check only the trusted blocks (slicing beyond list length
|
||||
# just returns all blocks). Blocks are ordered most recent
|
||||
# to oldest, so we want the first match (most recent).
|
||||
found_value = None
|
||||
for block in headers_blocks[:blocks_to_check]:
|
||||
for block in blocks[:blocks_to_check]:
|
||||
if key in block:
|
||||
block_value = block[key]
|
||||
# Values are always lists in headers_blocks, use the first one (most recent in that block)
|
||||
# Values inside a block are always lists; first entry
|
||||
# is the most recent occurrence within that block.
|
||||
if block_value:
|
||||
found_value = block_value[0]
|
||||
# Break after first match since blocks are ordered most recent to oldest
|
||||
break
|
||||
|
||||
if found_value is None:
|
||||
@@ -131,8 +148,8 @@ def _check_spam_with_hardcoded_rules(
|
||||
|
||||
|
||||
def _check_spam_with_rspamd(
|
||||
raw_data: bytes, spam_config: Dict[str, Any]
|
||||
) -> Tuple[bool, Optional[str], Optional[Dict[str, Any]]]:
|
||||
raw_data: bytes, spam_config: dict[str, Any]
|
||||
) -> tuple[bool, str | None, dict[str, Any] | None]:
|
||||
"""Check if a message is spam using rspamd.
|
||||
|
||||
Args:
|
||||
@@ -230,18 +247,9 @@ def process_inbound_message_task(self, inbound_message_id: str):
|
||||
|
||||
# Parse the email from raw_data
|
||||
raw_data_bytes = bytes(inbound_message.raw_data)
|
||||
try:
|
||||
parsed_email = parse_email_message(raw_data_bytes)
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to parse email message: {e}"
|
||||
logger.error(error_msg)
|
||||
inbound_message.error_message = error_msg
|
||||
inbound_message.save(update_fields=["error_message"])
|
||||
# Keep the message for retry
|
||||
return {"success": False, "error": error_msg}
|
||||
|
||||
if not parsed_email:
|
||||
error_msg = "Failed to parse email message (returned None)"
|
||||
parsed_email = parse_email(raw_data_bytes)
|
||||
if parsed_email is None:
|
||||
error_msg = "Failed to parse email message"
|
||||
logger.error(error_msg)
|
||||
inbound_message.error_message = error_msg
|
||||
inbound_message.save(update_fields=["error_message"])
|
||||
@@ -251,7 +259,7 @@ def process_inbound_message_task(self, inbound_message_id: str):
|
||||
# Get spam config from maildomain (includes global settings + domain-specific overrides)
|
||||
spam_config = mailbox.domain.get_spam_config()
|
||||
|
||||
rspamd_result: Optional[Dict[str, Any]] = None
|
||||
rspamd_result: dict[str, Any] | None = None
|
||||
if _is_selfcheck_message(parsed_email, recipient_email):
|
||||
logger.debug(
|
||||
"Bypassing spam checks for selfcheck message %s", inbound_message_id
|
||||
@@ -286,18 +294,18 @@ def process_inbound_message_task(self, inbound_message_id: str):
|
||||
f"X-StMsg-Sender-Auth: {auth_verdict}\r\n".encode("ascii")
|
||||
+ raw_data_bytes
|
||||
)
|
||||
try:
|
||||
parsed_email = parse_email_message(prepended)
|
||||
reparsed = parse_email(prepended)
|
||||
if reparsed is not None:
|
||||
parsed_email = reparsed
|
||||
raw_data_bytes = prepended
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
else:
|
||||
# Keep raw_data_bytes / parsed_email in lockstep: if the
|
||||
# re-parse breaks, store the original bytes so the blob stays
|
||||
# parseable for display (subject/body/recipients). The
|
||||
# sender-auth banner is sacrificed in this rare case.
|
||||
logger.warning(
|
||||
"Failed to re-parse email after prepending auth header, "
|
||||
"dropping the prepend: %s",
|
||||
e,
|
||||
"dropping the prepend"
|
||||
)
|
||||
|
||||
# Create the message using the extracted function
|
||||
|
||||
@@ -1,4 +1,16 @@
|
||||
"""Utility functions for RFC5322 email processing."""
|
||||
"""Extract base64-encoded inline images from editor input.
|
||||
|
||||
When a user pastes a screenshot into the rich-text editor, the
|
||||
underlying HTML carries the image as a ``<img src="data:image/png;
|
||||
base64,…">`` data URL. Sending that verbatim makes the message huge and
|
||||
breaks deduplication across replies. These helpers walk the HTML / text
|
||||
body, replace each data URL with a ``cid:`` reference, and return the
|
||||
decoded binary payload as a list of attachment dicts ready for
|
||||
:func:`jmap_email.compose_email`.
|
||||
|
||||
Strictly a Messages editor-side preprocessor — the JMAP library has no
|
||||
opinion on what callers do with the body string before composing.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
@@ -19,7 +31,7 @@ _MD_BASE64_IMG_RE = re.compile(
|
||||
r"(!\[[^\]]*\]\()data:(image/[a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/\n\r =]+)(\))"
|
||||
)
|
||||
|
||||
# Map common image MIME types to file extensions
|
||||
# Common image MIME types → file extensions for the synthesized filename.
|
||||
_MIME_TO_EXT = {
|
||||
"image/png": "png",
|
||||
"image/jpeg": "jpg",
|
||||
@@ -37,10 +49,10 @@ def _resolve_image(
|
||||
) -> str:
|
||||
"""Return the CID for *content*, reusing an existing one when possible.
|
||||
|
||||
If *known_images* is provided and already contains an entry whose SHA-256
|
||||
digest matches *content*, the existing CID is returned without creating a
|
||||
duplicate. Otherwise a new image dict is appended to *images* (and
|
||||
registered in *known_images* if supplied).
|
||||
If *known_images* is provided and already contains an entry whose
|
||||
SHA-256 digest matches *content*, the existing CID is returned
|
||||
without creating a duplicate. Otherwise a new image dict is appended
|
||||
to *images* (and registered in *known_images* if supplied).
|
||||
"""
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
|
||||
@@ -55,7 +67,9 @@ def _resolve_image(
|
||||
{
|
||||
"cid": cid,
|
||||
"content": content,
|
||||
"content_type": content_type,
|
||||
# JMAP / composer key name so the dict is feedable directly
|
||||
# into ``compose_email(attachments=[…])``.
|
||||
"type": content_type,
|
||||
"name": filename,
|
||||
"size": len(content),
|
||||
}
|
||||
@@ -92,25 +106,27 @@ def _make_replacer(
|
||||
return _replace
|
||||
|
||||
|
||||
def extract_base64_images_from_text(
|
||||
def extract_inline_images_text(
|
||||
text: str,
|
||||
known_images: dict[str, str] | None = None,
|
||||
) -> tuple[str, list[dict]]:
|
||||
"""Extract base64 images from plain text and replace them with CID references.
|
||||
|
||||
Handles both markdown image syntax ``
|
||||
and any residual HTML `<img src="data:image/...;base64,...">` tags.
|
||||
Handles both markdown image syntax ````
|
||||
and any residual HTML ``<img src="data:image/...;base64,...">`` tags.
|
||||
|
||||
Args:
|
||||
text: The plain text string potentially containing base64 images.
|
||||
known_images: Optional dict mapping SHA-256 hex digests to CIDs.
|
||||
When provided, duplicate images are de-duplicated across calls
|
||||
by reusing the same CID.
|
||||
When provided, duplicate images are de-duplicated across
|
||||
calls by reusing the same CID.
|
||||
|
||||
Returns:
|
||||
A tuple of (stripped_text, images) where *images* is a list of dicts
|
||||
with keys `cid`, `content` (bytes), `content_type`, `name`,
|
||||
and `size`.
|
||||
A tuple of (stripped_text, images) where *images* is a list of
|
||||
dicts with keys ``cid``, ``content`` (bytes), ``type``, ``name``,
|
||||
and ``size``. The shape is directly feedable into
|
||||
``compose_email(attachments=[…])`` after stamping each entry
|
||||
with ``disposition="inline"``.
|
||||
"""
|
||||
images: list[dict] = []
|
||||
replace = _make_replacer(images, known_images)
|
||||
@@ -121,84 +137,25 @@ def extract_base64_images_from_text(
|
||||
return stripped_text, images
|
||||
|
||||
|
||||
def remove_mime_headers(
|
||||
raw_email: bytes,
|
||||
*,
|
||||
prefixes: typing.Iterable[str] = (),
|
||||
names: typing.Iterable[str] = (),
|
||||
) -> bytes:
|
||||
"""Remove headers from the head section of a raw MIME message.
|
||||
|
||||
A header is dropped when its name (case-insensitive, ASCII) either equals
|
||||
one of *names* or starts with one of *prefixes*. RFC 5322 §2.2.3 folded
|
||||
continuation lines (lines beginning with SP or HTAB) are dropped along
|
||||
with the header they continue.
|
||||
|
||||
Operates on the head as raw bytes split at the first blank line; the
|
||||
body and the bytes of every retained header are left byte-identical.
|
||||
DKIM body hashing is unaffected, and signed headers we keep are not
|
||||
refolded or re-encoded.
|
||||
|
||||
Returns the input unchanged when nothing matched.
|
||||
"""
|
||||
name_set = {n.lower().encode("ascii") for n in names}
|
||||
prefix_tuple = tuple(p.lower().encode("ascii") for p in prefixes)
|
||||
if not name_set and not prefix_tuple:
|
||||
return raw_email
|
||||
|
||||
split = raw_email.find(b"\r\n\r\n")
|
||||
if split < 0:
|
||||
split = raw_email.find(b"\n\n")
|
||||
if split < 0:
|
||||
head, body = raw_email, b""
|
||||
else:
|
||||
head, body = raw_email[:split], raw_email[split:]
|
||||
|
||||
out: list[bytes] = []
|
||||
dropping = False
|
||||
for line in head.splitlines(keepends=True):
|
||||
if line[:1] in (b" ", b"\t"):
|
||||
if not dropping:
|
||||
out.append(line)
|
||||
continue
|
||||
name, sep, _ = line.partition(b":")
|
||||
if not sep:
|
||||
# Malformed line — preserve and stop any in-progress drop.
|
||||
dropping = False
|
||||
out.append(line)
|
||||
continue
|
||||
name_lc = name.lower()
|
||||
if name_lc in name_set or (prefix_tuple and name_lc.startswith(prefix_tuple)):
|
||||
dropping = True
|
||||
continue
|
||||
dropping = False
|
||||
out.append(line)
|
||||
|
||||
cleaned = b"".join(out)
|
||||
if cleaned == head:
|
||||
return raw_email
|
||||
return cleaned + body
|
||||
|
||||
|
||||
def extract_base64_images_from_html(
|
||||
def extract_inline_images_html(
|
||||
html: str,
|
||||
known_images: dict[str, str] | None = None,
|
||||
) -> tuple[str, list[dict]]:
|
||||
"""Extract base64-encoded images from HTML and replace them with CID references.
|
||||
|
||||
For each `<img src="data:image/...;base64,...">` found in *html*, a unique
|
||||
CID is generated, the `src` attribute is replaced with `cid:<cid>`, and
|
||||
the decoded binary content is collected.
|
||||
For each ``<img src="data:image/...;base64,...">`` found in *html*,
|
||||
a unique CID is generated, the ``src`` attribute is replaced with
|
||||
``cid:<cid>``, and the decoded binary content is collected.
|
||||
|
||||
Args:
|
||||
html: The HTML string potentially containing base64 images.
|
||||
known_images: Optional dict mapping SHA-256 hex digests to CIDs.
|
||||
When provided, duplicate images are de-duplicated across calls
|
||||
by reusing the same CID.
|
||||
When provided, duplicate images are de-duplicated across
|
||||
calls by reusing the same CID.
|
||||
|
||||
Returns:
|
||||
A tuple of (stripped_html, images) where *images* is a list of dicts
|
||||
with keys `cid`, `content` (bytes), `content_type`, and `name` and `size`.
|
||||
A tuple of (stripped_html, images) — same shape as
|
||||
:func:`extract_inline_images_text`.
|
||||
"""
|
||||
images: list[dict] = []
|
||||
stripped_html = _HTML_BASE64_IMG_RE.sub(_make_replacer(images, known_images), html)
|
||||
@@ -11,22 +11,25 @@ from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
import rest_framework as drf
|
||||
from jmap_email import (
|
||||
compose_email,
|
||||
find_header,
|
||||
first_address_email,
|
||||
parse_email,
|
||||
)
|
||||
|
||||
from core import models
|
||||
from core.enums import MessageDeliveryStatusChoices
|
||||
from core.mda.inbound import check_local_recipient, deliver_inbound_message
|
||||
from core.mda.outbound_direct import send_message_via_mx
|
||||
from core.mda.rfc5322 import (
|
||||
EmailParseError,
|
||||
compose_email,
|
||||
create_forward_message,
|
||||
create_reply_message,
|
||||
extract_base64_images_from_html,
|
||||
extract_base64_images_from_text,
|
||||
parse_email_message,
|
||||
from core.mda.inline_images import (
|
||||
extract_inline_images_html,
|
||||
extract_inline_images_text,
|
||||
)
|
||||
from core.mda.outbound_direct import send_message_via_mx
|
||||
from core.mda.replies import make_forward, make_reply
|
||||
from core.mda.signing import sign_message_dkim, verify_message_dkim
|
||||
from core.mda.smtp import send_smtp_mail
|
||||
from core.mda.utils import current_sent_at
|
||||
from core.services.blob_gc import schedule_for_gc
|
||||
from core.services.dns.check import check_spf_status
|
||||
from core.services.throttle import check_and_increment_throttle
|
||||
@@ -104,6 +107,27 @@ def validate_attachments_size(total_size: int, message_id: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
# When a message has no To recipient, emit a "To:" header using empty-group
|
||||
# syntax (RFC 4356 §3). A missing To header is a common anti-spam negative
|
||||
# signal (and resembles a DKIM-replay shape), so this keeps such sends —
|
||||
# typically Bcc-only — looking legitimate without disclosing anyone.
|
||||
UNDISCLOSED_RECIPIENTS_TO_HEADER = b"To: undisclosed-recipients:;"
|
||||
|
||||
|
||||
def build_xmailer_value() -> str:
|
||||
"""Return the X-Mailer header value: product name plus running release.
|
||||
|
||||
A present X-Mailer is a small positive deliverability signal (its absence
|
||||
reads as bulk/templated mail to some filters, e.g. iCloud). Like other MUAs
|
||||
(e.g. Open-Xchange's "Open-Xchange Mailer vX.Y.Z-RevN"), this identifies the
|
||||
software, not the deployment — so the product name is fixed and the running
|
||||
release appended when available.
|
||||
"""
|
||||
if settings.RELEASE != "NA":
|
||||
return f"{settings.MDA_HEADER_XMAILER} {settings.RELEASE}"
|
||||
return settings.MDA_HEADER_XMAILER
|
||||
|
||||
|
||||
def compose_and_sign_mime(
|
||||
message: models.Message,
|
||||
mailbox: models.Mailbox,
|
||||
@@ -142,18 +166,18 @@ def compose_and_sign_mime(
|
||||
if parent_parsed:
|
||||
is_forward = (message.subject or "").lower().startswith("fwd:")
|
||||
if is_forward:
|
||||
nested_data = create_forward_message(
|
||||
nested_data = make_forward(
|
||||
original_message=parent_parsed,
|
||||
forward_text=text_body,
|
||||
forward_html=html_body,
|
||||
body_text=text_body,
|
||||
body_html=html_body,
|
||||
include_original=True,
|
||||
)
|
||||
else:
|
||||
nested_data = create_reply_message(
|
||||
nested_data = make_reply(
|
||||
original_message=parent_parsed,
|
||||
reply_text=text_body,
|
||||
reply_html=html_body,
|
||||
include_quote=True,
|
||||
body_text=text_body,
|
||||
body_html=html_body,
|
||||
include_original=True,
|
||||
)
|
||||
if nested_data.get("textBody"):
|
||||
text_body = nested_data["textBody"][0]["content"]
|
||||
@@ -178,15 +202,18 @@ def compose_and_sign_mime(
|
||||
|
||||
mime_data = {
|
||||
"from": [{"name": message.sender.name, "email": message.sender.email}],
|
||||
"date": timezone.now().strftime("%a, %d %b %Y %H:%M:%S %z"),
|
||||
"sentAt": current_sent_at(),
|
||||
"to": recipients_by_type.get(models.MessageRecipientTypeChoices.TO, []),
|
||||
"cc": recipients_by_type.get(models.MessageRecipientTypeChoices.CC, []),
|
||||
"subject": message.subject,
|
||||
"textBody": [{"content": text_body}] if text_body else [],
|
||||
"htmlBody": [{"content": html_body}] if html_body else [],
|
||||
"message_id": message.mime_id,
|
||||
"messageId": [message.mime_id] if message.mime_id else None,
|
||||
}
|
||||
|
||||
# Advertise the sending application via X-Mailer (see build_xmailer_value).
|
||||
mime_data["headers"] = [{"name": "X-Mailer", "value": build_xmailer_value()}]
|
||||
|
||||
if all_attachments:
|
||||
mime_data["attachments"] = all_attachments
|
||||
message.has_attachments = bool(all_attachments)
|
||||
@@ -197,6 +224,11 @@ def compose_and_sign_mime(
|
||||
prepend_headers=prepend_headers,
|
||||
)
|
||||
|
||||
# Bcc/Cc-only send: the composed MIME has no To header. Add the empty-group
|
||||
# placeholder before signing so it is covered by DKIM.
|
||||
if not mime_data["to"]:
|
||||
raw_mime = UNDISCLOSED_RECIPIENTS_TO_HEADER + b"\r\n" + raw_mime
|
||||
|
||||
dkim_header = sign_message_dkim(raw_mime, mailbox.domain)
|
||||
if dkim_header:
|
||||
raw_mime = dkim_header + b"\r\n" + raw_mime
|
||||
@@ -243,29 +275,22 @@ def append_signature_and_extract_inline_images(
|
||||
raw_images = []
|
||||
|
||||
if text_body:
|
||||
text_body, text_images = extract_base64_images_from_text(
|
||||
text_body, text_images = extract_inline_images_text(
|
||||
text_body, known_images=known_images
|
||||
)
|
||||
raw_images.extend(text_images)
|
||||
|
||||
if html_body:
|
||||
html_body, html_images = extract_base64_images_from_html(
|
||||
html_body, html_images = extract_inline_images_html(
|
||||
html_body, known_images=known_images
|
||||
)
|
||||
raw_images.extend(html_images)
|
||||
|
||||
# Normalize to the format expected by compose_email
|
||||
inline_attachments = [
|
||||
{
|
||||
"content": img["content"],
|
||||
"type": img["content_type"],
|
||||
"name": img["name"],
|
||||
"disposition": "inline",
|
||||
"cid": img["cid"],
|
||||
"size": img["size"],
|
||||
}
|
||||
for img in raw_images
|
||||
]
|
||||
# ``extract_inline_images_*`` already returns the JMAP / composer
|
||||
# attachment shape (``type`` key, etc.). Set ``disposition="inline"``
|
||||
# on each entry so the composer wraps in ``multipart/related`` and
|
||||
# emits the ``cid`` Content-ID header.
|
||||
inline_attachments = [{**img, "disposition": "inline"} for img in raw_images]
|
||||
|
||||
return text_body, html_body, inline_attachments
|
||||
|
||||
@@ -316,6 +341,21 @@ def prepare_outbound_message(
|
||||
# atomic for just the Blob INSERT + FK-establishing save —
|
||||
# this keeps the per-sha advisory lock taken inside
|
||||
# ``create_blob`` held for ms, not for the duration of DKIM.
|
||||
# Caller-supplied MIME may also lack a To header (e.g. Bcc-only); add
|
||||
# the placeholder before signing. ``parse_email`` returns None on
|
||||
# unparseable input (already rejected upstream by the submit view).
|
||||
parsed = parse_email(raw_mime)
|
||||
if parsed is not None:
|
||||
if not find_header(parsed, "to"):
|
||||
raw_mime = UNDISCLOSED_RECIPIENTS_TO_HEADER + b"\r\n" + raw_mime
|
||||
# Mirror the composed-body path: advertise the sending application
|
||||
# via X-Mailer so raw submissions get the same deliverability
|
||||
# signal. Prepend before signing so it's covered by DKIM, but keep a
|
||||
# caller-supplied X-Mailer instead of duplicating the header.
|
||||
if not find_header(parsed, "x-mailer"):
|
||||
raw_mime = (
|
||||
f"X-Mailer: {build_xmailer_value()}".encode() + b"\r\n" + raw_mime
|
||||
)
|
||||
signed_mime = _sign_mime(mailbox_sender, raw_mime)
|
||||
validate_mime_size(len(signed_mime), message.id)
|
||||
message.sender_user = user
|
||||
@@ -497,14 +537,9 @@ def send_message(message: models.Message, force_mta_out: bool = False):
|
||||
# Use context manager to batch thread stats updates for all delivery status changes
|
||||
with ThreadStatsUpdateDeferrer.defer():
|
||||
blob_content = message.blob.get_content()
|
||||
try:
|
||||
parsed_email = parse_email_message(blob_content)
|
||||
except EmailParseError as e:
|
||||
logger.error(
|
||||
"Failed to parse email for message %s: %s",
|
||||
message.id,
|
||||
e,
|
||||
)
|
||||
parsed_email = parse_email(blob_content)
|
||||
if parsed_email is None:
|
||||
logger.error("Failed to parse email for message %s", message.id)
|
||||
# Mark all recipients as failed
|
||||
for recipient in message.recipients.all():
|
||||
recipient.delivery_status = MessageDeliveryStatusChoices.FAILED
|
||||
@@ -514,7 +549,7 @@ def send_message(message: models.Message, force_mta_out: bool = False):
|
||||
)
|
||||
return
|
||||
|
||||
if parsed_email.get("from", {}).get("email") != message.sender.email:
|
||||
if first_address_email(parsed_email.get("from")) != message.sender.email:
|
||||
raise ValueError("Mailbox email does not match the raw message sender")
|
||||
|
||||
message.sent_at = timezone.now()
|
||||
@@ -754,6 +789,7 @@ def send_outbound_email(
|
||||
message_content=mime_data,
|
||||
smtp_username=mta_out_smtp_username,
|
||||
smtp_password=mta_out_smtp_password,
|
||||
smtp_tls_security_level=settings.MTA_OUT_SMTP_TLS_SECURITY_LEVEL,
|
||||
)
|
||||
return statuses
|
||||
|
||||
|
||||
@@ -14,7 +14,8 @@ from django.conf import settings
|
||||
|
||||
import dns.resolver
|
||||
|
||||
from core.mda.smtp import send_smtp_mail
|
||||
from core.mda.smtp import SmtpProxy, send_smtp_mail
|
||||
from core.services.ssrf import SSRFValidationError, assert_public_ip
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -51,13 +52,27 @@ def resolve_mx_records(domain: str) -> List[Tuple[int, str]]:
|
||||
|
||||
|
||||
def resolve_hostname_ip(hostname: str) -> Optional[str]:
|
||||
"""
|
||||
Resolve a hostname to its first A record IP address, with a direct DNS query
|
||||
"""Resolve a hostname to its first *public* A-record IP.
|
||||
|
||||
SSRF guard: a recipient domain's MX (or A-record fallback) is
|
||||
attacker-controlled, so any address that fails SSRF validation
|
||||
(loopback / link-local / private / reserved / multicast / cloud-metadata)
|
||||
is skipped — the SMTP worker must never be steered into dialing internal
|
||||
infrastructure. Returns None when the host has no usable public IP, which
|
||||
makes the caller skip this MX and ultimately permanent-fail the recipient
|
||||
rather than connecting anywhere unsafe. Because we connect to exactly the
|
||||
IP returned here, there is no DNS-rebinding window between check and dial.
|
||||
"""
|
||||
try:
|
||||
answers = dns.resolver.resolve(hostname, "A", lifetime=10)
|
||||
for r in answers:
|
||||
return str(r)
|
||||
ip_str = str(r)
|
||||
try:
|
||||
assert_public_ip(ip_str, hostname)
|
||||
except SSRFValidationError as e:
|
||||
logger.warning("Refusing non-public MX target %s: %s", hostname, e)
|
||||
continue
|
||||
return ip_str
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error resolving IP for %s: %s", hostname, e)
|
||||
return None
|
||||
@@ -85,21 +100,31 @@ def group_recipients_by_mx(recipients: List[str]) -> Dict[str, Dict[str, Any]]:
|
||||
return domain_map
|
||||
|
||||
|
||||
def select_smtp_proxy() -> Dict[str, Any]:
|
||||
def select_smtp_proxy() -> Optional[SmtpProxy]:
|
||||
"""Pick a SOCKS5 proxy at random from MTA_OUT_DIRECT_PROXIES, if any.
|
||||
|
||||
Skips entries whose URL is missing a hostname or port, so a single bad
|
||||
config line doesn't take out the whole proxy pool.
|
||||
"""
|
||||
Select an SMTP proxy to use for sending messages.
|
||||
"""
|
||||
if len(settings.MTA_OUT_DIRECT_PROXIES) > 0:
|
||||
proxy = random.choice(settings.MTA_OUT_DIRECT_PROXIES) # noqa: S311
|
||||
parsed = urlparse(proxy)
|
||||
return {
|
||||
"proxy_host": parsed.hostname,
|
||||
"proxy_port": parsed.port,
|
||||
"proxy_username": parsed.username,
|
||||
"proxy_password": parsed.password,
|
||||
"sender_hostname": parsed.hostname,
|
||||
}
|
||||
return {}
|
||||
proxies = list(settings.MTA_OUT_DIRECT_PROXIES)
|
||||
random.shuffle(proxies)
|
||||
for url in proxies:
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
except ValueError as e:
|
||||
logger.warning("Invalid SMTP proxy URL %r: %s", url, e)
|
||||
continue
|
||||
if not parsed.hostname or not parsed.port:
|
||||
logger.warning("SMTP proxy URL %r missing hostname or port, skipping", url)
|
||||
continue
|
||||
return SmtpProxy(
|
||||
host=parsed.hostname,
|
||||
port=parsed.port,
|
||||
username=parsed.username,
|
||||
password=parsed.password,
|
||||
sender_hostname=parsed.hostname,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def send_message_via_mx(envelope_from, recipient_emails, mime_data) -> Dict[str, Any]:
|
||||
@@ -151,8 +176,6 @@ def send_message_via_mx(envelope_from, recipient_emails, mime_data) -> Dict[str,
|
||||
remaining_recipients,
|
||||
)
|
||||
|
||||
proxy_settings = select_smtp_proxy()
|
||||
|
||||
# Use direct SMTP, no auth
|
||||
smtp_statuses = send_smtp_mail(
|
||||
smtp_host=mx_hostname,
|
||||
@@ -161,7 +184,8 @@ def send_message_via_mx(envelope_from, recipient_emails, mime_data) -> Dict[str,
|
||||
envelope_from=envelope_from,
|
||||
recipient_emails=remaining_recipients.copy(),
|
||||
message_content=mime_data,
|
||||
**proxy_settings,
|
||||
smtp_tls_security_level=settings.MTA_OUT_SMTP_TLS_SECURITY_LEVEL,
|
||||
proxy=select_smtp_proxy(),
|
||||
)
|
||||
|
||||
# Process results and update remaining recipients
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Raw-byte manipulation of RFC 5322 message head sections.
|
||||
|
||||
Used by the inbound MTA path to strip Messages-internal ``X-StMsg-*``
|
||||
hint headers from received bytes before re-processing, without going
|
||||
through the stdlib ``email`` parser/serialiser round-trip (which would
|
||||
re-fold every retained header and break the DKIM body hash on signed
|
||||
input).
|
||||
"""
|
||||
|
||||
import typing
|
||||
|
||||
|
||||
def remove_mime_headers(
|
||||
raw_email: bytes,
|
||||
*,
|
||||
prefixes: typing.Iterable[str] = (),
|
||||
names: typing.Iterable[str] = (),
|
||||
) -> bytes:
|
||||
"""Remove headers from the head section of a raw MIME message.
|
||||
|
||||
A header is dropped when its name (case-insensitive, ASCII) either
|
||||
equals one of *names* or starts with one of *prefixes*. RFC 5322
|
||||
§2.2.3 folded continuation lines (lines beginning with SP or HTAB)
|
||||
are dropped along with the header they continue.
|
||||
|
||||
Operates on the head as raw bytes split at the first blank line;
|
||||
the body and the bytes of every retained header are left byte-
|
||||
identical. DKIM body hashing is unaffected, and signed headers we
|
||||
keep are not refolded or re-encoded.
|
||||
|
||||
Returns the input unchanged when nothing matched.
|
||||
"""
|
||||
name_set = {n.lower().encode("ascii") for n in names}
|
||||
prefix_tuple = tuple(p.lower().encode("ascii") for p in prefixes)
|
||||
if not name_set and not prefix_tuple:
|
||||
return raw_email
|
||||
|
||||
split = raw_email.find(b"\r\n\r\n")
|
||||
if split < 0:
|
||||
split = raw_email.find(b"\n\n")
|
||||
if split < 0:
|
||||
head, body = raw_email, b""
|
||||
else:
|
||||
head, body = raw_email[:split], raw_email[split:]
|
||||
|
||||
out: list[bytes] = []
|
||||
dropping = False
|
||||
for line in head.splitlines(keepends=True):
|
||||
if line[:1] in (b" ", b"\t"):
|
||||
if not dropping:
|
||||
out.append(line)
|
||||
continue
|
||||
name, sep, _ = line.partition(b":")
|
||||
if not sep:
|
||||
# Malformed line — preserve and stop any in-progress drop.
|
||||
dropping = False
|
||||
out.append(line)
|
||||
continue
|
||||
name_lc = name.lower()
|
||||
if name_lc in name_set or (prefix_tuple and name_lc.startswith(prefix_tuple)):
|
||||
dropping = True
|
||||
continue
|
||||
dropping = False
|
||||
out.append(line)
|
||||
|
||||
cleaned = b"".join(out)
|
||||
if cleaned == head:
|
||||
return raw_email
|
||||
return cleaned + body
|
||||
@@ -0,0 +1,355 @@
|
||||
"""Reply / forward template builders for outbound composition.
|
||||
|
||||
These were originally shipped by ``jmap-email`` but moved here because
|
||||
they bake in Messages-specific UI choices that don't belong in a
|
||||
strict-JMAP library:
|
||||
|
||||
- English-only header strings (``On {date}, {sender} wrote:``,
|
||||
``---------- Forwarded message ----------``) — the frontend has
|
||||
translations for these in ``src/frontend/public/locales/common/``
|
||||
but the library has no access to a locale.
|
||||
- HTML markup choices (``<blockquote data-type="quote-separator">``)
|
||||
picked to align with the frontend's blocknote rendering.
|
||||
- An incomplete output dict (no ``from``, no ``sentAt``) that the
|
||||
Messages outbound flow finishes — the library can't finish it
|
||||
because it has no live mailbox / user context.
|
||||
|
||||
The library still owns the wire-format primitives we depend on:
|
||||
:func:`jmap_email.compose_email`, :func:`jmap_email.parse_email`,
|
||||
:func:`jmap_email.format_address`, :func:`jmap_email.format_address_list`,
|
||||
:func:`jmap_email.is_valid_msg_id`. This module is a thin layer on top.
|
||||
"""
|
||||
|
||||
import html
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import format_datetime, parsedate_to_datetime
|
||||
from typing import Any
|
||||
|
||||
from jmap_email import (
|
||||
JmapEmail,
|
||||
first_msgid,
|
||||
format_address,
|
||||
format_address_list,
|
||||
is_valid_msg_id,
|
||||
msgid_chain,
|
||||
)
|
||||
|
||||
# Bracket-aware tokeniser for a wire-form ``References`` chain. Splitting
|
||||
# on whitespace would slice ``<bad id@x>`` into ``<bad`` + ``id@x>`` and
|
||||
# the right half would survive shape validation — silent half-id
|
||||
# salvaging. Walking by ``<…>`` pairs keeps each bracketed token intact.
|
||||
_MSGID_TOKEN_RE = re.compile(r"<[^<>]*>")
|
||||
|
||||
__all__ = [
|
||||
"compute_reply_threading",
|
||||
"forward_subject",
|
||||
"make_forward",
|
||||
"make_reply",
|
||||
"reply_subject",
|
||||
]
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
# Subject prefixing
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def reply_subject(subject: str) -> str:
|
||||
"""Add ``Re: `` prefix to a subject, avoiding duplication."""
|
||||
if subject.lower().startswith("re:"):
|
||||
return subject
|
||||
return f"Re: {subject}"
|
||||
|
||||
|
||||
def forward_subject(subject: str) -> str:
|
||||
"""Add ``Fwd: `` prefix to a subject, avoiding duplication."""
|
||||
if subject.lower().startswith("fwd:"):
|
||||
return subject
|
||||
return f"Fwd: {subject}"
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
# RFC 5322 §3.6.4 threading projection
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def compute_reply_threading(
|
||||
original_message: JmapEmail,
|
||||
) -> tuple[list[str] | None, list[str] | None]:
|
||||
"""Project (``inReplyTo``, ``references``) for a reply to ``original_message``.
|
||||
|
||||
Returns a pair of JMAP ``String[] | None`` values ready to splice
|
||||
into the outbound dict::
|
||||
|
||||
in_reply_to, references = compute_reply_threading(parent)
|
||||
if in_reply_to:
|
||||
reply["inReplyTo"] = in_reply_to
|
||||
if references:
|
||||
reply["references"] = references
|
||||
|
||||
Both are ``None`` when the parent's Message-ID is missing or
|
||||
malformed — better to lose threading than corrupt the chain on the
|
||||
receiver side.
|
||||
"""
|
||||
parent_id = first_msgid(original_message.get("messageId"))
|
||||
if not parent_id or not is_valid_msg_id(parent_id):
|
||||
return None, None
|
||||
|
||||
# The stored shape strips angle brackets; wrap for the wire chain
|
||||
# then strip again for the per-id list.
|
||||
wrapped = f"<{parent_id}>"
|
||||
orig_refs = msgid_chain(original_message.get("references"))
|
||||
chain_tokens = [
|
||||
tok for tok in _MSGID_TOKEN_RE.findall(orig_refs) if is_valid_msg_id(tok)
|
||||
]
|
||||
if wrapped not in chain_tokens:
|
||||
chain_tokens.append(wrapped)
|
||||
|
||||
in_reply_to = [parent_id]
|
||||
references = [tok.strip("<>") for tok in chain_tokens]
|
||||
return in_reply_to, references
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
# Quote-block embedding (text + HTML)
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _body_content(part: dict[str, Any]) -> str:
|
||||
"""Read the ``content`` of a JMAP ``EmailBodyPart``."""
|
||||
return part.get("content", "") if isinstance(part, dict) else ""
|
||||
|
||||
|
||||
def _attach_utc_if_naive(dt: datetime) -> datetime:
|
||||
"""Ensure ``dt`` is timezone-aware, defaulting to UTC."""
|
||||
if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None:
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
def _embed_original_message(
|
||||
original_message: JmapEmail,
|
||||
new_text: str = "",
|
||||
new_html: str | None = None,
|
||||
include_original: bool = True,
|
||||
is_forward: bool = False,
|
||||
) -> tuple[str, str]:
|
||||
"""Embed original message content into new text and HTML.
|
||||
|
||||
Returns ``(text_body, html_body)``.
|
||||
"""
|
||||
if new_text is None:
|
||||
new_text = ""
|
||||
|
||||
if not include_original:
|
||||
html_body = new_html or f"<p>{html.escape(new_text)}</p>"
|
||||
if html_body:
|
||||
html_body = html_body.replace("’", "'")
|
||||
return new_text, html_body
|
||||
|
||||
# Coerce None → "" — inbound parsers may emit ``{"subject": None}``
|
||||
# when the source message has no Subject header, and downstream
|
||||
# ``str.lower()`` etc. crash on None.
|
||||
orig_subject = original_message.get("subject") or ""
|
||||
orig_from_list = original_message.get("from") or []
|
||||
orig_from = orig_from_list[0] if orig_from_list else {}
|
||||
orig_to = original_message.get("to") or []
|
||||
orig_cc = original_message.get("cc") or []
|
||||
# ``sentAt`` is the JMAP ISO-8601 string. Reply/forward callers
|
||||
# sometimes pass a tz-aware ``datetime`` here off a freshly-loaded
|
||||
# ``Message`` model where ``sent_at`` is already a ``datetime``.
|
||||
orig_date = original_message.get("sentAt") or ""
|
||||
|
||||
date_str = ""
|
||||
if isinstance(orig_date, datetime):
|
||||
date_str = format_datetime(_attach_utc_if_naive(orig_date))
|
||||
elif isinstance(orig_date, str) and orig_date:
|
||||
try:
|
||||
parsed_dt = parsedate_to_datetime(orig_date)
|
||||
except (ValueError, TypeError, IndexError):
|
||||
parsed_dt = None
|
||||
if parsed_dt:
|
||||
date_str = format_datetime(parsed_dt)
|
||||
else:
|
||||
date_str = orig_date
|
||||
else:
|
||||
date_str = "an unknown date"
|
||||
|
||||
header_text = ""
|
||||
if is_forward:
|
||||
from_display = format_address(
|
||||
orig_from.get("name", ""), orig_from.get("email", "")
|
||||
)
|
||||
to_display = format_address_list(orig_to)
|
||||
cc_display = format_address_list(orig_cc) if orig_cc else ""
|
||||
|
||||
header_text = "\r\n\r\n---------- Forwarded message ----------\r\n"
|
||||
if from_display:
|
||||
header_text += f"From: {from_display}\r\n"
|
||||
if to_display:
|
||||
header_text += f"To: {to_display}\r\n"
|
||||
if cc_display:
|
||||
header_text += f"Cc: {cc_display}\r\n"
|
||||
header_text += f"Subject: {orig_subject}\r\n"
|
||||
header_text += f"Date: {date_str}\r\n\r\n"
|
||||
else:
|
||||
from_display = format_address(
|
||||
orig_from.get("name", ""), orig_from.get("email", "")
|
||||
)
|
||||
if from_display:
|
||||
header_text = f"\r\n\r\nOn {date_str}, {from_display} wrote:\r\n"
|
||||
else:
|
||||
header_text = f"\r\n\r\nOn {date_str}, someone wrote:\r\n"
|
||||
|
||||
text_body = f"{new_text}{header_text}"
|
||||
|
||||
if original_message.get("textBody"):
|
||||
text_body_list = original_message["textBody"]
|
||||
first_text = text_body_list[0] if text_body_list else None
|
||||
orig_text = _body_content(first_text) if first_text else ""
|
||||
if orig_text:
|
||||
if is_forward:
|
||||
text_body += orig_text
|
||||
else:
|
||||
quoted_text = "\r\n".join(
|
||||
[f"> {line}" for line in orig_text.splitlines()]
|
||||
)
|
||||
text_body += quoted_text
|
||||
|
||||
html_content = new_html or f"<p>{html.escape(new_text)}</p>"
|
||||
if html_content:
|
||||
html_content = html_content.replace("’", "'")
|
||||
|
||||
html_body = html_content
|
||||
if new_html or original_message.get("htmlBody"):
|
||||
from_display_html = html.escape(
|
||||
format_address(orig_from.get("name", ""), orig_from.get("email", ""))
|
||||
)
|
||||
to_display_html = html.escape(format_address_list(orig_to))
|
||||
cc_display_html = html.escape(format_address_list(orig_cc)) if orig_cc else ""
|
||||
|
||||
if is_forward:
|
||||
header_html = "<p>---------- Forwarded message ----------<br/>"
|
||||
else:
|
||||
header_html = "<p>---------- In reply to ----------<br/>"
|
||||
|
||||
if from_display_html:
|
||||
header_html += f"<strong>From:</strong> {from_display_html}<br/>"
|
||||
if to_display_html:
|
||||
header_html += f"<strong>To:</strong> {to_display_html}<br/>"
|
||||
if cc_display_html:
|
||||
header_html += f"<strong>Cc:</strong> {cc_display_html}<br/>"
|
||||
header_html += f"<strong>Subject:</strong> {html.escape(orig_subject)}<br/>"
|
||||
header_html += f"<strong>Date:</strong> {html.escape(date_str)}<br/>"
|
||||
header_html += "</p>"
|
||||
|
||||
orig_html = ""
|
||||
if original_message.get("htmlBody"):
|
||||
html_body_list = original_message["htmlBody"]
|
||||
first_html = html_body_list[0] if html_body_list else None
|
||||
orig_html = _body_content(first_html) if first_html else ""
|
||||
|
||||
nested_html = f"""
|
||||
<blockquote data-type="quote-separator">
|
||||
{header_html}
|
||||
{orig_html}
|
||||
</blockquote>
|
||||
"""
|
||||
html_body = f"{html_content}{nested_html}"
|
||||
|
||||
return text_body, html_body
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
# Reply / forward template builders
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def make_reply(
|
||||
original_message: JmapEmail,
|
||||
body_text: str = "",
|
||||
body_html: str | None = None,
|
||||
include_original: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a JMAP Email object pre-filled as a reply to ``original_message``.
|
||||
|
||||
Returns a new Email dict (not bytes). The caller is expected to set
|
||||
``from`` and ``sentAt`` before passing the result to
|
||||
:func:`jmap_email.compose_email` — the composer is strict-by-design
|
||||
and will reject a dict missing either one.
|
||||
|
||||
Threading contract: emits ``inReplyTo`` and a per-id-validated
|
||||
``references`` chain when the parent Message-ID parses cleanly;
|
||||
emits neither when the parent id is malformed (better to lose
|
||||
threading than relay corruption).
|
||||
"""
|
||||
orig_subject = original_message.get("subject") or ""
|
||||
orig_from_list = original_message.get("from") or []
|
||||
orig_from = orig_from_list[0] if orig_from_list else None
|
||||
|
||||
if body_text is None:
|
||||
body_text = ""
|
||||
|
||||
new_subject = reply_subject(orig_subject)
|
||||
|
||||
text_body, html_body = _embed_original_message(
|
||||
original_message, body_text, body_html, include_original, is_forward=False
|
||||
)
|
||||
|
||||
reply_in_reply_to, reply_refs = compute_reply_threading(original_message)
|
||||
|
||||
reply: dict[str, Any] = {
|
||||
"subject": new_subject,
|
||||
"textBody": [{"partId": "1", "type": "text/plain", "content": text_body}],
|
||||
"from": None,
|
||||
"to": [orig_from] if orig_from and orig_from.get("email") else None,
|
||||
"cc": original_message.get("cc"),
|
||||
}
|
||||
if reply_in_reply_to:
|
||||
reply["inReplyTo"] = reply_in_reply_to
|
||||
if reply_refs:
|
||||
reply["references"] = reply_refs
|
||||
if html_body != (body_html or f"<p>{html.escape(body_text)}</p>"):
|
||||
reply["htmlBody"] = [{"partId": "2", "type": "text/html", "content": html_body}]
|
||||
|
||||
return reply
|
||||
|
||||
|
||||
def make_forward(
|
||||
original_message: JmapEmail,
|
||||
body_text: str = "",
|
||||
body_html: str | None = None,
|
||||
include_original: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a JMAP Email object pre-filled as a forward of ``original_message``.
|
||||
|
||||
Returns a new Email dict (not bytes). The caller is expected to set
|
||||
``from``, ``to``, and ``sentAt`` before passing the result to
|
||||
:func:`jmap_email.compose_email` — the composer is strict-by-design
|
||||
and will reject a dict missing any of them.
|
||||
"""
|
||||
orig_subject = original_message.get("subject") or ""
|
||||
new_subject = forward_subject(orig_subject)
|
||||
|
||||
if body_text is None:
|
||||
body_text = ""
|
||||
|
||||
text_body, html_body = _embed_original_message(
|
||||
original_message, body_text, body_html, include_original, is_forward=True
|
||||
)
|
||||
|
||||
forward: dict[str, Any] = {
|
||||
"subject": new_subject,
|
||||
"textBody": [{"partId": "1", "type": "text/plain", "content": text_body}],
|
||||
"from": None,
|
||||
"to": None,
|
||||
"cc": None,
|
||||
}
|
||||
|
||||
if html_body != (body_html or f"<p>{html.escape(body_text)}</p>"):
|
||||
forward["htmlBody"] = [
|
||||
{"partId": "2", "type": "text/html", "content": html_body}
|
||||
]
|
||||
|
||||
return forward
|
||||
@@ -1,118 +0,0 @@
|
||||
# RFC 5322 email parser & composer
|
||||
|
||||
This module handles RFC 5322 / 5321 / 2047 / 2231 email messages on both the
|
||||
inbound and outbound paths. It exposes two complementary entry points:
|
||||
|
||||
- `composer.compose_email(jmap_data)` — outbound: build raw RFC 5322 bytes from
|
||||
a JMAP-style dict.
|
||||
- `parser.parse_email_message(raw_bytes)` — inbound: parse raw RFC 5322 bytes
|
||||
into the same JMAP-style dict shape.
|
||||
|
||||
Both sides use the JMAP body conventions (`textBody`, `htmlBody`,
|
||||
`attachments`) so callers don't need to know which way data is flowing.
|
||||
|
||||
## Strict compose, lenient parse
|
||||
|
||||
The two entry points use **different libraries on purpose**, and the asymmetry
|
||||
is the point.
|
||||
|
||||
| Direction | Backend | Why |
|
||||
|---|---|---|
|
||||
| **Compose** (`composer.py`) | Python stdlib `email` | Caller-controlled input → must produce strictly RFC-compliant output. Stdlib's `email.policy.SMTP` enforces correct address-list folding, RFC 2047 encoded-word emission, RFC 2231 parameter encoding, and CRLF / line-length limits. No third-party correctness bugs to inherit. |
|
||||
| **Parse** (`parser.py`) | Mailgun's Flanker | Real-world inbound MIME comes from every MTA on the planet, including ones that violate the spec. Flanker's lenient parser recovers from common malformations (encoded structural delimiters, missing charsets, broken Content-Transfer-Encoding) where stdlib's strict policy would raise. |
|
||||
|
||||
### Compose: why stdlib
|
||||
|
||||
Until 2026 the composer was Flanker-backed. It carried two known security-flavor
|
||||
bugs in Flanker's address-header serialization:
|
||||
|
||||
- `'; '.join(...)` was used as the address-list separator instead of `', '`,
|
||||
violating RFC 5322 §3.4 whenever any recipient had a non-ASCII display name.
|
||||
- `ace_display_name` wrapped the display-name in `smart_quote()` *before*
|
||||
RFC 2047 encoding, leaking literal `"` characters into the encoded-word —
|
||||
enabling a class of address-list injection-via-display-name attacks.
|
||||
|
||||
Both were fixed in our flanker fork, but the broader pattern — Flanker is
|
||||
unmaintained upstream, so any new compose bug becomes ours to fix forever —
|
||||
made stdlib the correct destination. CPython has been quietly fixing this
|
||||
exact corner of the email spec for years (gh-100884, gh-118643, gh-121284,
|
||||
gh-127794, gh-142006, gh-142517, gh-144156). On a Python ≥ 3.14.4 floor we
|
||||
get all of those for free.
|
||||
|
||||
The composer requires **no fallback path**: caller-controlled input means we
|
||||
should never see malformed inputs, and producing strictly compliant output is
|
||||
the security-correct choice. Headers with embedded CR/LF are stripped
|
||||
defensively (`_sanitize_header_value`) to defeat header injection.
|
||||
|
||||
### Parse: why Flanker (for now)
|
||||
|
||||
The parser stays on Flanker because:
|
||||
|
||||
- Real inbound messages contain encoding violations, malformed Content-Type
|
||||
parameters, charset mismatches, and broken structural delimiters that
|
||||
stdlib's `policy.default` rejects with `MessageDefect` errors.
|
||||
- Flanker's `addresslib.address.parse_list` returns the survivors instead of
|
||||
the empty set when one address in a list is malformed (modulo the long-known
|
||||
upstream bug in mailgun/flanker#190 — orthogonal to our concerns here).
|
||||
- Migrating the parser carries real regression risk against thousands of
|
||||
message fixtures we don't fully control.
|
||||
|
||||
The parser's public surface (`parse_email_message`, `parse_email_address`,
|
||||
`parse_email_addresses`, `decode_email_header_text`, `parse_date`) does not
|
||||
expose Flanker types — callers always see plain dicts and primitives — so a
|
||||
future migration to stdlib + targeted lenience helpers (e.g. `policy.compat32`
|
||||
fallback on `MessageDefect`) is straightforward and unblocked.
|
||||
|
||||
## JMAP shape
|
||||
|
||||
Both compose and parse use the [JMAP](https://jmap.io/spec-mail.html#properties-of-the-email-object)
|
||||
email object shape:
|
||||
|
||||
- `from`, `to`, `cc`, `bcc` — `[{"name": str, "email": str}, ...]`
|
||||
- `subject` — `str`
|
||||
- `textBody`, `htmlBody` — `[{"partId": str, "type": str, "charset": str, "content": str}, ...]`
|
||||
- `attachments` — `[{"name": str, "type": str, "content": base64-str, "disposition": "attachment" | "inline", "cid": str | None}, ...]`
|
||||
- `messageId`, `headers`, `date` — as needed
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
from core.mda.rfc5322 import (
|
||||
compose_email, EmailComposeError,
|
||||
parse_email_message, EmailParseError,
|
||||
)
|
||||
|
||||
# Outbound:
|
||||
try:
|
||||
raw_bytes = compose_email({
|
||||
"from": {"name": "Alice", "email": "alice@example.com"},
|
||||
"to": [{"name": "Bob", "email": "bob@example.com"}],
|
||||
"subject": "hi",
|
||||
"textBody": [{"content": "hello"}],
|
||||
})
|
||||
except EmailComposeError as e:
|
||||
...
|
||||
|
||||
# Inbound:
|
||||
try:
|
||||
parsed = parse_email_message(raw_bytes)
|
||||
subject = parsed["subject"]
|
||||
sender = parsed["from"]
|
||||
text_parts = parsed["textBody"]
|
||||
except EmailParseError as e:
|
||||
...
|
||||
```
|
||||
|
||||
## Tests
|
||||
|
||||
- `test_rfc5322_composer.py` — compose-side: address formatting, MIME structure
|
||||
cases (text-only, html-only, alternative, related-with-inline, mixed-with-
|
||||
attachments), header-injection defense, ported regression cases from
|
||||
flanker's own composer test suite.
|
||||
- `test_rfc5322_parser.py` — parse-side: address/header/date parsing, body
|
||||
extraction, malformed-input recovery.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- Python ≥ 3.14.4 (composer relies on stdlib `email` fixes through 3.14.4 — gh-100884, gh-118643, gh-121284, gh-127794, gh-142006, gh-142517, gh-144156).
|
||||
- Flanker (parser-only, pinned via fork in `pyproject.toml`).
|
||||
@@ -1,49 +0,0 @@
|
||||
"""
|
||||
RFC5322 email format package.
|
||||
|
||||
This package provides functionality for parsing and handling email
|
||||
content according to RFC5322 standards.
|
||||
"""
|
||||
|
||||
from .composer import (
|
||||
EmailComposeError,
|
||||
compose_email,
|
||||
create_forward_message,
|
||||
create_reply_message,
|
||||
format_address,
|
||||
format_address_list,
|
||||
)
|
||||
from .parser import (
|
||||
EmailParseError,
|
||||
decode_email_header_text,
|
||||
parse_date,
|
||||
parse_email_address,
|
||||
parse_email_addresses,
|
||||
parse_email_message,
|
||||
)
|
||||
from .utils import (
|
||||
extract_base64_images_from_html,
|
||||
extract_base64_images_from_text,
|
||||
remove_mime_headers,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Parser functions
|
||||
"parse_email_address",
|
||||
"parse_email_addresses",
|
||||
"parse_email_message",
|
||||
"parse_date",
|
||||
"decode_email_header_text",
|
||||
"EmailParseError",
|
||||
# Composer functions
|
||||
"format_address",
|
||||
"format_address_list",
|
||||
"compose_email",
|
||||
"create_reply_message",
|
||||
"create_forward_message",
|
||||
"EmailComposeError",
|
||||
# Utility functions
|
||||
"extract_base64_images_from_html",
|
||||
"extract_base64_images_from_text",
|
||||
"remove_mime_headers",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,874 +0,0 @@
|
||||
"""
|
||||
RFC5322 email parser using Flanker library.
|
||||
|
||||
This module provides functions for parsing email addresses and messages
|
||||
according to RFC5322 standards. It uses the Flanker library for robust
|
||||
parsing and is intended to be the central place for all email parsing
|
||||
operations in the application.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import shlex
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from datetime import timezone as dt_timezone
|
||||
from email.header import decode_header
|
||||
from email.utils import parsedate_to_datetime
|
||||
from ntpath import basename as nt_basename
|
||||
from posixpath import basename as posix_basename
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from flanker.addresslib import address
|
||||
from flanker.mime import create
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _strip_nul_bytes(text: str) -> str:
|
||||
"""Strip NUL bytes from text.
|
||||
|
||||
PostgreSQL text fields cannot store NUL (0x00) bytes.
|
||||
This char is used to mark the end of a string in C language
|
||||
and is not valid in PostgreSQL text fields. Furthermore the
|
||||
RFC 5322 section 4 defines it as an obsolete character.
|
||||
https://datatracker.ietf.org/doc/html/rfc5322#page-31
|
||||
"""
|
||||
return text.replace("\x00", "") if text else ""
|
||||
|
||||
|
||||
class EmailParseError(Exception):
|
||||
"""Exception raised for errors during email parsing."""
|
||||
|
||||
|
||||
def decode_email_header_text(header_text: str) -> str:
|
||||
"""
|
||||
Decode email header text that might be encoded (RFC 2047).
|
||||
"""
|
||||
if not header_text:
|
||||
return ""
|
||||
|
||||
# Ensure input is a string
|
||||
header_text_str = str(header_text)
|
||||
# Use decode_header which returns a list of (decoded_string, charset) pairs
|
||||
# charset is None if the part was not encoded
|
||||
decoded_parts = decode_header(header_text_str)
|
||||
|
||||
result_parts = []
|
||||
for part, charset in decoded_parts:
|
||||
if isinstance(part, bytes):
|
||||
# Decode bytes using charset or fallbacks
|
||||
if not charset or charset == "unknown-8bit":
|
||||
try:
|
||||
result_parts.append(part.decode("utf-8", errors="replace"))
|
||||
except UnicodeDecodeError:
|
||||
result_parts.append(part.decode("latin-1", errors="replace"))
|
||||
else:
|
||||
try:
|
||||
result_parts.append(part.decode(charset, errors="replace"))
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
result_parts.append(part.decode("utf-8", errors="replace"))
|
||||
else:
|
||||
# Part is already a string
|
||||
result_parts.append(part)
|
||||
|
||||
# Join the decoded parts first.
|
||||
full_result = "".join(result_parts)
|
||||
# Now, replace folding whitespace (CRLF followed by space/tab) with a single space.
|
||||
cleaned_result = re.sub(r"\r\n[ \t]+", " ", full_result)
|
||||
# Finally, collapse any multiple spaces into one.
|
||||
return " ".join(cleaned_result.split())
|
||||
|
||||
|
||||
def _strip_name_quotes(name: str) -> str:
|
||||
"""
|
||||
Strip surrounding single quotes from display names.
|
||||
|
||||
RFC 5322 uses double quotes for display names with special characters,
|
||||
and flanker correctly strips those. However, some email clients incorrectly
|
||||
use single quotes, which flanker preserves. We strip them for consistency.
|
||||
|
||||
Examples:
|
||||
"'John Doe'" -> "John Doe"
|
||||
"John Doe" -> "John Doe"
|
||||
"'John's Name'" -> "John's Name" (only strips surrounding quotes)
|
||||
"""
|
||||
if name and len(name) >= 2 and name.startswith("'") and name.endswith("'"):
|
||||
return name[1:-1]
|
||||
return name
|
||||
|
||||
|
||||
def _contains_group_syntax(address_str: str) -> bool:
|
||||
"""
|
||||
Check if the address string contains RFC 5322 group syntax or malformed variants.
|
||||
|
||||
Group syntax format: "Group Name: addr1, addr2;" or "undisclosed-recipients:;"
|
||||
Also handles malformed variants like "undisclosed-recipients:>" (using > instead of ;)
|
||||
Returns True if any group-like syntax pattern is found.
|
||||
|
||||
The pattern is: word(s) followed by : then addresses/empty then ; or >
|
||||
Key insight: the group name comes AFTER any comma separator, so we look for
|
||||
patterns like "name:...;" where "name" doesn't contain @.
|
||||
"""
|
||||
stripped = address_str.strip()
|
||||
# Check for proper group syntax (;) or malformed variant (>)
|
||||
if ";" not in stripped and ":>" not in stripped:
|
||||
return False
|
||||
|
||||
# Use regex to find group patterns: non-@ chars followed by : then anything then ; or >
|
||||
# This handles "undisclosed-recipients:;", "Group: addr1, addr2;", ":;", and ":>"
|
||||
# Pattern: optional non-@ non-: chars, then :, then anything, then ; or just :>
|
||||
group_pattern = re.compile(r"[^@:,]*:([^;]*;|>)")
|
||||
return bool(group_pattern.search(stripped))
|
||||
|
||||
|
||||
def _remove_group_syntax(address_str: str) -> str:
|
||||
"""
|
||||
Remove RFC 5322 group syntax from address string, extracting inner addresses.
|
||||
|
||||
"Group: addr1, addr2;" -> "addr1, addr2"
|
||||
"undisclosed-recipients:;" -> ""
|
||||
"user@a.com, Group: b@c.com;" -> "user@a.com, b@c.com"
|
||||
"user@a.com, undisclosed-recipients:;" -> "user@a.com"
|
||||
"undisclosed-recipients:>" -> "" (malformed variant)
|
||||
"""
|
||||
stripped = address_str.strip()
|
||||
if ";" not in stripped and ":>" not in stripped:
|
||||
return stripped
|
||||
|
||||
# Use regex to find and process group patterns
|
||||
# Group pattern: optional word(s) without @ or : or ,, followed by :, then content, then ;
|
||||
# Also handle malformed :> variant (empty group with > instead of ;)
|
||||
# We replace "GroupName: content;" with just "content"
|
||||
group_pattern = re.compile(r"[^@:,]*:([^;]*);")
|
||||
|
||||
def replace_group(match):
|
||||
inner = match.group(1).strip()
|
||||
return inner if inner else ""
|
||||
|
||||
result = group_pattern.sub(replace_group, stripped)
|
||||
|
||||
# Handle malformed :> pattern (remove "name:>" entirely as it's an empty malformed group)
|
||||
malformed_pattern = re.compile(r"[^@:,]*:>")
|
||||
result = malformed_pattern.sub("", result)
|
||||
|
||||
# Clean up: remove empty entries, extra commas, whitespace
|
||||
parts = [p.strip() for p in result.split(",") if p.strip()]
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
def parse_email_address(address_str: str) -> Tuple[str, str]:
|
||||
"""
|
||||
Parse an email address that might include a display name.
|
||||
|
||||
Args:
|
||||
address_str: String containing an email address, possibly with display name
|
||||
|
||||
Returns:
|
||||
Tuple of (display_name, email_address)
|
||||
|
||||
Examples:
|
||||
>>> parse_email_address('user@example.com')
|
||||
('', 'user@example.com')
|
||||
>>> parse_email_address('User <user@example.com>')
|
||||
('User', 'user@example.com')
|
||||
"""
|
||||
if not address_str:
|
||||
return "", ""
|
||||
|
||||
# Handle RFC 5322 group syntax (e.g., "undisclosed-recipients:;")
|
||||
# These cause flanker warnings and should return empty for single address parsing
|
||||
if _contains_group_syntax(address_str):
|
||||
# For single address parsing, group syntax means no valid single address
|
||||
return "", ""
|
||||
|
||||
# Use flanker to parse the address
|
||||
parsed = address.parse(address_str)
|
||||
|
||||
if parsed is None:
|
||||
return "", address_str.strip()
|
||||
|
||||
# If parsed successfully, extract name and address
|
||||
# Check for display_name attribute (UrlAddress objects from flanker don't have it)
|
||||
if not hasattr(parsed, "display_name"):
|
||||
# UrlAddress or other non-standard parsed object - return address only
|
||||
addr = getattr(parsed, "address", str(parsed))
|
||||
return "", addr
|
||||
|
||||
# Strip single quotes from display name (flanker only strips double quotes per RFC 5322)
|
||||
display_name = _strip_name_quotes(parsed.display_name or "") # pylint: disable=no-member
|
||||
return display_name, parsed.address # pylint: disable=no-member
|
||||
|
||||
|
||||
def parse_email_addresses(addresses_str: str) -> List[Tuple[str, str]]:
|
||||
"""
|
||||
Parse multiple email addresses from a comma-separated string.
|
||||
|
||||
Handles RFC 5322 group syntax (e.g., "Group: addr1, addr2;") by extracting
|
||||
the addresses within groups.
|
||||
|
||||
Args:
|
||||
addresses_str: Comma-separated string of email addresses
|
||||
|
||||
Returns:
|
||||
List of tuples, each containing (display_name, email_address)
|
||||
"""
|
||||
if not addresses_str:
|
||||
return []
|
||||
|
||||
# Handle RFC 5322 group syntax (e.g., "undisclosed-recipients:;" or "Group: a@b.com;")
|
||||
# Extract addresses from within groups to avoid flanker warnings
|
||||
if _contains_group_syntax(addresses_str):
|
||||
addresses_str = _remove_group_syntax(addresses_str)
|
||||
if not addresses_str:
|
||||
return [] # Empty group like "undisclosed-recipients:;"
|
||||
|
||||
# Use flanker to parse the address list
|
||||
parsed = address.parse_list(addresses_str)
|
||||
|
||||
if parsed is None:
|
||||
return []
|
||||
|
||||
# Extract name and address for each parsed address
|
||||
# Strip single quotes from display names (flanker only strips double quotes per RFC 5322)
|
||||
# Handle UrlAddress objects which don't have display_name attribute
|
||||
result = []
|
||||
for addr in parsed:
|
||||
if hasattr(addr, "display_name"):
|
||||
name = _strip_name_quotes(addr.display_name or "")
|
||||
email = addr.address
|
||||
else:
|
||||
# UrlAddress or other non-standard parsed object
|
||||
name = ""
|
||||
email = getattr(addr, "address", str(addr))
|
||||
result.append((name, email))
|
||||
return result
|
||||
|
||||
|
||||
def parse_date(date_str: str) -> Optional[datetime]:
|
||||
"""
|
||||
Parse date string from email header.
|
||||
|
||||
Args:
|
||||
date_str: Date string in RFC5322 format
|
||||
|
||||
Returns:
|
||||
Datetime object or None if parsing fails
|
||||
"""
|
||||
if not date_str:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Use email.utils which handles RFC5322 date formats
|
||||
return parsedate_to_datetime(date_str)
|
||||
except (TypeError, ValueError) as e: # Catch specific errors
|
||||
logger.warning("Could not parse date string '%s': %s", date_str, e)
|
||||
return None
|
||||
|
||||
|
||||
def _infer_filename_from_content_type(content_type: str) -> str:
|
||||
"""
|
||||
Infer a filename with extension from a MIME content type.
|
||||
Uses the most commonly used file extensions for each MIME type.
|
||||
|
||||
Args:
|
||||
content_type: MIME type string (e.g., "image/png", "application/pdf")
|
||||
|
||||
Returns:
|
||||
Filename with appropriate extension (e.g., "unnamed.png", "unnamed.pdf")
|
||||
"""
|
||||
extension_map = {
|
||||
"text/plain": ".txt",
|
||||
"text/html": ".html",
|
||||
"text/csv": ".csv",
|
||||
"application/pdf": ".pdf",
|
||||
"image/jpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/gif": ".gif",
|
||||
"application/json": ".json",
|
||||
"application/xml": ".xml",
|
||||
"application/zip": ".zip",
|
||||
}
|
||||
ext = extension_map.get(content_type, "")
|
||||
return f"unnamed{ext}"
|
||||
|
||||
|
||||
def _sanitize_filename(filename: str, max_length: int = 255) -> str:
|
||||
"""Sanitize an attachment filename, preserving the extension when truncating."""
|
||||
|
||||
filename = nt_basename(posix_basename(filename))
|
||||
|
||||
filename = filename.strip('"/.\\')
|
||||
|
||||
# Remove null bytes and control characters
|
||||
filename = re.sub(r"[\x00-\x1f\x7f]", "", filename)
|
||||
|
||||
# Remove dangerous characters
|
||||
filename = re.sub(r'[<>:"|?*\\/]', "_", filename)
|
||||
|
||||
# Truncate while preserving extension
|
||||
if len(filename) > max_length:
|
||||
# Find the last dot for extension (but not at the start like .gitignore)
|
||||
last_dot = filename.rfind(".")
|
||||
if last_dot > 0:
|
||||
name = filename[:last_dot]
|
||||
ext = filename[last_dot:]
|
||||
# Only preserve extension if it's reasonable length (up to 10 chars including dot)
|
||||
if len(ext) <= 10:
|
||||
max_name_length = max_length - len(ext)
|
||||
if max_name_length > 0:
|
||||
return name[:max_name_length] + ext
|
||||
return filename[:max_length]
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
def _build_attachment_dict(
|
||||
body: Any,
|
||||
part_type: str,
|
||||
filename: str,
|
||||
disposition: str,
|
||||
content_id: Optional[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Helper function to build an attachment dictionary.
|
||||
Converts body to bytes, computes SHA-256 hash, and constructs the attachment dict.
|
||||
|
||||
Args:
|
||||
body: The part body (str or bytes)
|
||||
part_type: MIME type of the part
|
||||
filename: Name of the attachment file
|
||||
disposition: Content-Disposition value ("attachment", "inline", etc.)
|
||||
content_id: Content-ID if present
|
||||
|
||||
Returns:
|
||||
Dictionary representing the attachment
|
||||
"""
|
||||
if isinstance(body, str):
|
||||
body_bytes = body.encode("utf-8")
|
||||
else:
|
||||
body_bytes = body
|
||||
|
||||
content_hash = hashlib.sha256(body_bytes).hexdigest()
|
||||
|
||||
return {
|
||||
"type": part_type,
|
||||
"name": _sanitize_filename(filename) or "unnamed",
|
||||
"size": len(body_bytes),
|
||||
"disposition": disposition,
|
||||
"cid": content_id,
|
||||
"content": body_bytes,
|
||||
"sha256": content_hash,
|
||||
}
|
||||
|
||||
|
||||
def _is_inline_media_type(content_type: str) -> bool:
|
||||
"""
|
||||
Check if the content type is an inline media type (image/*, audio/*, video/*).
|
||||
|
||||
Args:
|
||||
content_type: MIME type string (e.g., "image/png", "audio/mp3")
|
||||
|
||||
Returns:
|
||||
True if the type is an inline media type
|
||||
"""
|
||||
return (
|
||||
content_type.startswith("image/")
|
||||
or content_type.startswith("audio/")
|
||||
or content_type.startswith("video/")
|
||||
)
|
||||
|
||||
|
||||
def _get_part_info(part) -> Dict[str, Any]:
|
||||
"""
|
||||
Extract relevant information from a MIME part for classification.
|
||||
|
||||
Args:
|
||||
part: A Flanker MIME part
|
||||
|
||||
Returns:
|
||||
Dictionary with type, disposition, name, body, content_id, part_id
|
||||
"""
|
||||
if not hasattr(part, "content_type") or not part.content_type:
|
||||
return {"type": "text/plain", "disposition": None, "name": None, "body": None}
|
||||
|
||||
content_type_obj = part.content_type
|
||||
part_type = f"{content_type_obj.main}/{content_type_obj.sub}"
|
||||
|
||||
# Get disposition
|
||||
disposition = None
|
||||
disposition_info = getattr(part, "content_disposition", None)
|
||||
if disposition_info and isinstance(disposition_info, tuple) and disposition_info[0]:
|
||||
disposition = disposition_info[0].lower()
|
||||
|
||||
# Get filename from disposition or content-type params
|
||||
filename = None
|
||||
if (
|
||||
disposition_info
|
||||
and isinstance(disposition_info, tuple)
|
||||
and len(disposition_info) > 1
|
||||
):
|
||||
params = disposition_info[1]
|
||||
if isinstance(params, dict):
|
||||
filename_raw = params.get("filename")
|
||||
if filename_raw:
|
||||
filename = decode_email_header_text(str(filename_raw).strip())
|
||||
|
||||
if not filename and hasattr(content_type_obj, "params"):
|
||||
filename_param = content_type_obj.params.get("name")
|
||||
if filename_param:
|
||||
filename = decode_email_header_text(filename_param.strip())
|
||||
|
||||
# Get Content-ID
|
||||
headers_dict = getattr(part, "headers", {})
|
||||
content_id_header = headers_dict.get("Content-ID")
|
||||
content_id = str(content_id_header).strip("<>") if content_id_header else None
|
||||
|
||||
# Get body - may fail if flanker can't decode the transfer encoding
|
||||
# (e.g., quoted-printable with non-ASCII characters)
|
||||
try:
|
||||
body = getattr(part, "body", None)
|
||||
except ValueError:
|
||||
# Flanker's quopri decoder failed - try to get raw body
|
||||
try:
|
||||
container = getattr(part, "_container", None)
|
||||
if container and hasattr(container, "stream"):
|
||||
body_start = getattr(container, "_body_start", 0)
|
||||
body_end = getattr(container, "end", 0)
|
||||
container.stream.seek(body_start)
|
||||
body = container.stream.read(body_end - body_start + 1)
|
||||
else:
|
||||
body = None
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
body = None
|
||||
|
||||
# Get part ID
|
||||
part_id = getattr(part, "message_id", "") or ""
|
||||
|
||||
return {
|
||||
"type": part_type,
|
||||
"disposition": disposition,
|
||||
"name": filename,
|
||||
"body": body,
|
||||
"content_id": content_id,
|
||||
"part_id": part_id,
|
||||
}
|
||||
|
||||
|
||||
def _build_body_part_dict(part_info: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Build a body part dictionary for textBody/htmlBody arrays.
|
||||
|
||||
Args:
|
||||
part_info: Dictionary from _get_part_info
|
||||
|
||||
Returns:
|
||||
Dictionary with partId, type, content
|
||||
"""
|
||||
body = part_info["body"]
|
||||
part_type = part_info["type"]
|
||||
|
||||
# Binary types (images, audio, video) need base64 encoding for JSON transport
|
||||
if _is_inline_media_type(part_type):
|
||||
if body is None:
|
||||
content = ""
|
||||
elif isinstance(body, bytes):
|
||||
content = base64.b64encode(body).decode("ascii")
|
||||
else:
|
||||
# Already a string (unlikely for binary), encode it
|
||||
content = base64.b64encode(body.encode("latin-1")).decode("ascii")
|
||||
# Text types - decode as UTF-8
|
||||
elif body is not None and not isinstance(body, str):
|
||||
content = body.decode("utf-8", errors="replace")
|
||||
else:
|
||||
content = body or ""
|
||||
|
||||
return {
|
||||
"partId": part_info["part_id"],
|
||||
"type": part_type,
|
||||
"content": _strip_nul_bytes(content),
|
||||
}
|
||||
|
||||
|
||||
def _build_attachment_from_part_info(
|
||||
part_info: Dict[str, Any], disposition_override: str = "attachment"
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Build an attachment dictionary from part info.
|
||||
|
||||
Args:
|
||||
part_info: Dictionary from _get_part_info
|
||||
disposition_override: Disposition to use if not set
|
||||
|
||||
Returns:
|
||||
Dictionary representing the attachment
|
||||
"""
|
||||
disposition = part_info["disposition"] or disposition_override
|
||||
filename = part_info["name"] or _infer_filename_from_content_type(part_info["type"])
|
||||
|
||||
return _build_attachment_dict(
|
||||
part_info["body"] or b"",
|
||||
part_info["type"],
|
||||
filename,
|
||||
disposition,
|
||||
part_info["content_id"],
|
||||
)
|
||||
|
||||
|
||||
def _parse_body_structure(
|
||||
parts: List,
|
||||
multipart_type: str,
|
||||
in_alternative: bool,
|
||||
html_body: Optional[List],
|
||||
text_body: Optional[List],
|
||||
attachments: List,
|
||||
) -> None:
|
||||
"""
|
||||
Recursively parse MIME structure following JMAP spec algorithm (Section 4.1).
|
||||
|
||||
This implements the parseStructure algorithm from the JMAP specification,
|
||||
with a modification: inline media types are NOT added to attachments when
|
||||
one of textBody/htmlBody is null (unlike the spec example).
|
||||
|
||||
Args:
|
||||
parts: List of MIME parts to process
|
||||
multipart_type: Type of parent multipart (mixed/alternative/related)
|
||||
in_alternative: Whether we're inside a multipart/alternative
|
||||
html_body: List to append HTML body parts (or None if nullified)
|
||||
text_body: List to append text body parts (or None if nullified)
|
||||
attachments: List to append attachment parts
|
||||
"""
|
||||
# Track lengths for multipart/alternative fallback
|
||||
text_length = len(text_body) if text_body is not None else -1
|
||||
html_length = len(html_body) if html_body is not None else -1
|
||||
|
||||
for i, part in enumerate(parts):
|
||||
if not hasattr(part, "content_type") or not part.content_type:
|
||||
continue
|
||||
|
||||
content_type_obj = part.content_type
|
||||
part_type = f"{content_type_obj.main}/{content_type_obj.sub}"
|
||||
is_multipart = content_type_obj.is_multipart()
|
||||
|
||||
# Get part info for classification
|
||||
part_info = _get_part_info(part)
|
||||
|
||||
# Determine if this is an inline body part (not attachment)
|
||||
# Per JMAP spec: disposition != "attachment" AND
|
||||
# (type is text/plain OR text/html OR inline media) AND
|
||||
# (first part OR (not in related AND (is inline media OR no filename)))
|
||||
is_inline = (
|
||||
part_info["disposition"] != "attachment"
|
||||
and (
|
||||
part_type in {"text/plain", "text/html"}
|
||||
or _is_inline_media_type(part_type)
|
||||
)
|
||||
and (
|
||||
i == 0
|
||||
or (
|
||||
multipart_type != "related"
|
||||
and (_is_inline_media_type(part_type) or not part_info["name"])
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
if is_multipart:
|
||||
# Recurse into multipart
|
||||
sub_multipart_type = content_type_obj.sub # e.g., "alternative", "related"
|
||||
sub_parts = getattr(part, "parts", []) or []
|
||||
_parse_body_structure(
|
||||
sub_parts,
|
||||
sub_multipart_type,
|
||||
in_alternative or sub_multipart_type == "alternative",
|
||||
html_body,
|
||||
text_body,
|
||||
attachments,
|
||||
)
|
||||
|
||||
elif is_inline:
|
||||
# Handle inline parts based on context
|
||||
if multipart_type == "alternative":
|
||||
# In direct alternative: route based on type only
|
||||
if part_type == "text/plain":
|
||||
if text_body is not None:
|
||||
text_body.append(_build_body_part_dict(part_info))
|
||||
elif part_type == "text/html":
|
||||
if html_body is not None:
|
||||
html_body.append(_build_body_part_dict(part_info))
|
||||
else:
|
||||
# Other types in alternative go to attachments
|
||||
attachments.append(_build_attachment_from_part_info(part_info))
|
||||
continue
|
||||
|
||||
# Outside alternative but within an alternative ancestor
|
||||
if in_alternative:
|
||||
# text/plain nullifies htmlBody locally
|
||||
if part_type == "text/plain":
|
||||
html_body = None
|
||||
# text/html nullifies textBody locally
|
||||
if part_type == "text/html":
|
||||
text_body = None
|
||||
|
||||
# Push to both arrays if not nullified
|
||||
if text_body is not None:
|
||||
text_body.append(_build_body_part_dict(part_info))
|
||||
if html_body is not None:
|
||||
html_body.append(_build_body_part_dict(part_info))
|
||||
|
||||
# NOTE: We intentionally skip the JMAP spec's condition:
|
||||
# if ((!textBody || !htmlBody) && isInlineMediaType) attachments.push(part)
|
||||
# This is our modification to not duplicate inline media in attachments
|
||||
|
||||
else:
|
||||
# Non-inline parts go to attachments
|
||||
attachments.append(_build_attachment_from_part_info(part_info))
|
||||
|
||||
# Handle multipart/alternative fallback:
|
||||
# If only one type was found, copy to the other array
|
||||
if (
|
||||
multipart_type == "alternative"
|
||||
and text_body is not None
|
||||
and html_body is not None
|
||||
):
|
||||
# Found HTML part only - copy to textBody
|
||||
if text_length == len(text_body) and html_length != len(html_body):
|
||||
for j in range(html_length, len(html_body)):
|
||||
text_body.append(html_body[j])
|
||||
# Found text part only - copy to htmlBody
|
||||
if html_length == len(html_body) and text_length != len(text_body):
|
||||
for j in range(text_length, len(text_body)):
|
||||
html_body.append(text_body[j])
|
||||
|
||||
|
||||
def parse_message_content(message) -> Dict[str, Any]:
|
||||
"""
|
||||
Extract text, HTML, and attachments from a message, following JMAP format.
|
||||
|
||||
This uses the JMAP spec's parseStructure algorithm (Section 4.1) to properly
|
||||
handle multipart structures including alternative, related, and mixed.
|
||||
|
||||
Key behavior:
|
||||
- text/plain parts go to textBody
|
||||
- text/html parts go to htmlBody
|
||||
- Inline media (images, audio, video) go to textBody/htmlBody, NOT attachments
|
||||
- Explicit attachments (Content-Disposition: attachment) go to attachments
|
||||
- Parts in multipart/related after the first go to attachments
|
||||
|
||||
Args:
|
||||
message: A Flanker MIME message object
|
||||
|
||||
Returns:
|
||||
Dictionary with textBody, htmlBody, and attachments arrays
|
||||
"""
|
||||
result = {"textBody": [], "htmlBody": [], "attachments": []}
|
||||
|
||||
# Handle invalid message structure
|
||||
if not hasattr(message, "content_type") or not message.content_type:
|
||||
if hasattr(message, "body") and isinstance(message.body, str):
|
||||
result["textBody"].append(
|
||||
{
|
||||
"partId": "",
|
||||
"type": "text/plain",
|
||||
"content": _strip_nul_bytes(message.body),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
try:
|
||||
# Use the JMAP-style recursive parser
|
||||
# Wrap the message in a list and treat it as if inside multipart/mixed
|
||||
_parse_body_structure(
|
||||
[message],
|
||||
"mixed",
|
||||
False,
|
||||
result["htmlBody"],
|
||||
result["textBody"],
|
||||
result["attachments"],
|
||||
)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error parsing message body structure: %s", e, exc_info=True)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _parse_labels_header(labels_str: str) -> list:
|
||||
"""Parse a labels header value, handling quoted strings.
|
||||
|
||||
Supports two formats:
|
||||
- Comma-separated (our format, OfflineIMAP): ``label1, label2, "label three"``
|
||||
- Space-separated (Dovecot): ``label1 label2 "label three"``
|
||||
"""
|
||||
result = []
|
||||
# Only use comma parsing when commas are actually present as delimiters
|
||||
if "," in labels_str:
|
||||
# Comma-separated format with optional quoted strings
|
||||
pattern = r'\s*"([^"]*)"\s*|\s*([^,]+)'
|
||||
matches = re.findall(pattern, labels_str)
|
||||
for match in matches:
|
||||
# match[0] is the quoted content (without quotes), match[1] is unquoted
|
||||
label = (match[0] if match[0] else match[1]).strip()
|
||||
if label:
|
||||
result.append(label)
|
||||
else:
|
||||
# Space-separated format (Dovecot), with shlex to handle quoted strings
|
||||
try:
|
||||
result = [
|
||||
token.strip() for token in shlex.split(labels_str) if token.strip()
|
||||
]
|
||||
except ValueError:
|
||||
# Fallback to simple split if shlex fails (e.g. unmatched quotes)
|
||||
result = [token.strip() for token in labels_str.split() if token.strip()]
|
||||
return result
|
||||
|
||||
|
||||
def parse_email_message(raw_email_bytes: bytes) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Parse a raw email message (bytes) into a structured dictionary following JMAP format.
|
||||
|
||||
Args:
|
||||
raw_email_bytes: Raw email data as bytes
|
||||
|
||||
Returns:
|
||||
Dictionary containing parsed email data, or None if parsing fails fundamentally.
|
||||
|
||||
Raises:
|
||||
EmailParseError: If parsing fails with a specific error we want to propagate.
|
||||
"""
|
||||
if not raw_email_bytes or not isinstance(raw_email_bytes, bytes):
|
||||
# Ensure input is non-empty bytes
|
||||
logger.warning(
|
||||
"Invalid input provided to parse_email_message: type=%s",
|
||||
type(raw_email_bytes),
|
||||
)
|
||||
raise EmailParseError("Input must be non-empty bytes.")
|
||||
|
||||
try:
|
||||
# Parse with flanker directly from bytes
|
||||
message = create.from_string(raw_email_bytes)
|
||||
|
||||
if message is None or not hasattr(message, "headers"):
|
||||
logger.warning(
|
||||
"Flanker failed to parse email data into a valid message object. Input length: %d",
|
||||
len(raw_email_bytes),
|
||||
)
|
||||
raise EmailParseError(
|
||||
"Flanker could not parse the input into a valid email message."
|
||||
)
|
||||
|
||||
# Extract all headers, normalizing keys to lowercase
|
||||
headers = {}
|
||||
# Also extract headers in order for position-based filtering (e.g., spam checks)
|
||||
# Flanker's message.headers.items() preserves the order from the raw email
|
||||
headers_list = []
|
||||
|
||||
for k, v in message.headers.items():
|
||||
decoded_value = decode_email_header_text(v)
|
||||
key_lower = k.lower()
|
||||
headers_list.append((key_lower, decoded_value))
|
||||
|
||||
# Build headers dict (for compatibility)
|
||||
if key_lower in headers:
|
||||
current_value = headers[key_lower]
|
||||
if isinstance(current_value, list):
|
||||
current_value.append(decoded_value)
|
||||
else:
|
||||
headers[key_lower] = [current_value, decoded_value]
|
||||
else:
|
||||
headers[key_lower] = decoded_value
|
||||
|
||||
# Split headers into blocks based on Received headers
|
||||
# Each Received header marks the END of its block - everything above it (before it in the list) is trusted
|
||||
# All values in blocks are stored as lists for consistency
|
||||
headers_blocks = []
|
||||
current_block = defaultdict(list)
|
||||
|
||||
for header_name, header_value in headers_list:
|
||||
if header_name == "received":
|
||||
# Received header marks the end of the current block
|
||||
# Add it to the current block, then finalize the block
|
||||
current_block["received"].append(header_value)
|
||||
headers_blocks.append(dict(current_block))
|
||||
current_block = defaultdict(list)
|
||||
else:
|
||||
# Add header to current block (always as list)
|
||||
current_block[header_name].append(header_value)
|
||||
|
||||
# Add the last block if it has any headers (headers after the last Received)
|
||||
if current_block:
|
||||
headers_blocks.append(dict(current_block))
|
||||
|
||||
# Extract labels from X-Gmail-Labels and X-Keywords headers
|
||||
# Both are combined into gmail_labels for backward compatibility
|
||||
gmail_labels = []
|
||||
seen_labels = set()
|
||||
|
||||
# Parse X-Gmail-Labels (Google Takeout format)
|
||||
if "x-gmail-labels" in headers:
|
||||
labels_str = headers["x-gmail-labels"]
|
||||
if isinstance(labels_str, list):
|
||||
labels_str = labels_str[0] # Take first value if multiple
|
||||
for label in _parse_labels_header(labels_str):
|
||||
if label not in seen_labels:
|
||||
seen_labels.add(label)
|
||||
gmail_labels.append(label)
|
||||
|
||||
# Parse X-Keywords (Dovecot/OfflineIMAP/mu4e format)
|
||||
if "x-keywords" in headers:
|
||||
labels_str = headers["x-keywords"]
|
||||
if isinstance(labels_str, list):
|
||||
labels_str = labels_str[0] # Take first value if multiple
|
||||
for label in _parse_labels_header(labels_str):
|
||||
if label not in seen_labels:
|
||||
seen_labels.add(label)
|
||||
gmail_labels.append(label)
|
||||
|
||||
subject = headers.get("subject", "")
|
||||
from_header_decoded = headers.get("from", "")
|
||||
from_name, from_addr = parse_email_address(from_header_decoded)
|
||||
to_recipients = parse_email_addresses(headers.get("to", ""))
|
||||
cc_recipients = parse_email_addresses(headers.get("cc", ""))
|
||||
bcc_recipients = parse_email_addresses(headers.get("bcc", ""))
|
||||
date = parse_date(headers.get("date", ""))
|
||||
message_id = headers.get("message-id", "")
|
||||
if message_id.startswith("<") and message_id.endswith(">"):
|
||||
message_id = message_id[1:-1]
|
||||
references = headers.get("references", "")
|
||||
in_reply_to = headers.get("in-reply-to", "")
|
||||
if in_reply_to.startswith("<") and in_reply_to.endswith(">"):
|
||||
in_reply_to = in_reply_to[1:-1]
|
||||
|
||||
# Extract content using parse_message_content
|
||||
body_parts = parse_message_content(message)
|
||||
|
||||
# Use datetime.timezone.utc for the default date
|
||||
default_date = datetime.now(dt_timezone.utc)
|
||||
|
||||
return {
|
||||
"subject": _strip_nul_bytes(subject or ""),
|
||||
"from": {"name": from_name, "email": from_addr},
|
||||
"to": [{"name": name, "email": email} for name, email in to_recipients],
|
||||
"cc": [{"name": name, "email": email} for name, email in cc_recipients],
|
||||
"bcc": [{"name": name, "email": email} for name, email in bcc_recipients],
|
||||
"date": date or default_date,
|
||||
# JMAP format body parts
|
||||
"textBody": body_parts["textBody"],
|
||||
"htmlBody": body_parts["htmlBody"],
|
||||
"attachments": body_parts["attachments"],
|
||||
# Raw MIME is passed in, no need to include decoded string version
|
||||
"headers": headers, # Dict for compatibility
|
||||
"headers_list": headers_list, # List of (name, value) tuples in order
|
||||
"headers_blocks": headers_blocks, # List of dicts, each block ends with a Received header
|
||||
"message_id": message_id,
|
||||
"references": references,
|
||||
"in_reply_to": in_reply_to,
|
||||
"gmail_labels": gmail_labels, # Add Gmail labels to parsed data
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
# Ensure any EmailParseError raised above is not caught again
|
||||
if isinstance(e, EmailParseError):
|
||||
raise e
|
||||
logger.exception("Unexpected error during email parsing: %s", str(e))
|
||||
raise EmailParseError("Failed to parse email") from e
|
||||
@@ -10,10 +10,17 @@ from typing import Optional, Tuple
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
from jmap_email import body_text_joined
|
||||
|
||||
from core import models
|
||||
from core.mda.draft import create_draft
|
||||
from core.mda.outbound import prepare_outbound_message, send_message
|
||||
from core.mda.selfcheck_reporting import SelfCheckResult, report_selfcheck
|
||||
from core.mda.selfcheck_reporting import (
|
||||
SelfCheckResult,
|
||||
finish_sentry_checkin,
|
||||
report_selfcheck,
|
||||
start_sentry_checkin,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -163,8 +170,8 @@ def _wait_for_message_reception(
|
||||
for message in messages:
|
||||
# Check if the message contains our secret
|
||||
parsed_data = message.get_parsed_data()
|
||||
text_body = parsed_data.get("textBody", [{}])[0].get("content", "")
|
||||
html_body = parsed_data.get("htmlBody", [{}])[0].get("content", "")
|
||||
text_body = body_text_joined(parsed_data, "textBody")
|
||||
html_body = body_text_joined(parsed_data, "htmlBody")
|
||||
|
||||
if secret in text_body or secret in html_body:
|
||||
logger.info("Found received message with secret: %s", message.id)
|
||||
@@ -186,8 +193,8 @@ def _verify_message_integrity(message: models.Message, original_secret: str) ->
|
||||
return False
|
||||
|
||||
# Check that the secret is present in the message body
|
||||
text_body = parsed_data.get("textBody", [{}])[0].get("content", "")
|
||||
html_body = parsed_data.get("htmlBody", [{}])[0].get("content", "")
|
||||
text_body = body_text_joined(parsed_data, "textBody")
|
||||
html_body = body_text_joined(parsed_data, "htmlBody")
|
||||
|
||||
if original_secret not in text_body or original_secret not in html_body:
|
||||
logger.error("Secret not found in message body")
|
||||
@@ -247,6 +254,8 @@ def run_selfcheck() -> SelfCheckResult:
|
||||
|
||||
logger.info("Starting selfcheck: %s -> %s", from_email, to_email)
|
||||
|
||||
check_in_id = start_sentry_checkin()
|
||||
|
||||
try:
|
||||
# Step 1: Create test mailboxes
|
||||
from_mailbox, to_mailbox = _create_test_mailboxes(from_email, to_email)
|
||||
@@ -340,4 +349,6 @@ that the mail delivery pipeline is working correctly.</p>
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Failed to report selfcheck result", exc_info=True)
|
||||
|
||||
finish_sentry_checkin(check_in_id, result)
|
||||
|
||||
return result
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Selfcheck reporting: webhook and structured logging."""
|
||||
"""Selfcheck reporting: webhook, Sentry crons, and structured logging."""
|
||||
|
||||
import logging
|
||||
from typing import Optional, TypedDict
|
||||
@@ -6,6 +6,8 @@ from typing import Optional, TypedDict
|
||||
from django.conf import settings
|
||||
|
||||
import requests
|
||||
from sentry_sdk.crons import capture_checkin
|
||||
from sentry_sdk.crons.consts import MonitorStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -45,6 +47,71 @@ def log_selfcheck_result(result: SelfCheckResult):
|
||||
)
|
||||
|
||||
|
||||
def _sentry_crons_enabled() -> bool:
|
||||
"""Whether Sentry cron reporting is enabled and usable.
|
||||
|
||||
Warns once per call when a slug is configured without ``SENTRY_DSN``
|
||||
— without the DSN, ``capture_checkin`` silently no-ops, so the
|
||||
operator would otherwise see "missed" alerts in Sentry with no
|
||||
hint as to why.
|
||||
"""
|
||||
if not settings.MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG:
|
||||
return False
|
||||
if not settings.SENTRY_DSN:
|
||||
logger.warning(
|
||||
"MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG is set but SENTRY_DSN is "
|
||||
"not — selfcheck Sentry cron check-in skipped."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def start_sentry_checkin() -> Optional[str]:
|
||||
"""Open a Sentry cron check-in if configured. Returns the check_in_id."""
|
||||
if not _sentry_crons_enabled():
|
||||
return None
|
||||
|
||||
try:
|
||||
return capture_checkin(
|
||||
monitor_slug=settings.MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG,
|
||||
status=MonitorStatus.IN_PROGRESS,
|
||||
)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Failed to open Sentry selfcheck check-in", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def finish_sentry_checkin(check_in_id: Optional[str], result: SelfCheckResult):
|
||||
"""Close a previously opened Sentry cron check-in with OK/ERROR.
|
||||
|
||||
Reports an explicit duration when both send and reception times are
|
||||
known — Sentry would otherwise infer it from the check-in timestamp
|
||||
delta, which also includes the post-run cleanup sleep.
|
||||
"""
|
||||
# Short-circuit on check_in_id first so the misconfig warning from
|
||||
# _sentry_crons_enabled() fires at most once per run (from start).
|
||||
if not check_in_id or not _sentry_crons_enabled():
|
||||
return
|
||||
|
||||
status = MonitorStatus.OK if result["success"] else MonitorStatus.ERROR
|
||||
send_time = result["send_time"]
|
||||
reception_time = result["reception_time"]
|
||||
duration = (
|
||||
send_time + reception_time
|
||||
if send_time is not None and reception_time is not None
|
||||
else None
|
||||
)
|
||||
try:
|
||||
capture_checkin(
|
||||
monitor_slug=settings.MESSAGES_SELFCHECK_SENTRY_MONITOR_SLUG,
|
||||
check_in_id=check_in_id,
|
||||
status=status,
|
||||
duration=duration,
|
||||
)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Failed to close Sentry selfcheck check-in", exc_info=True)
|
||||
|
||||
|
||||
def send_selfcheck_webhook(result: SelfCheckResult):
|
||||
"""POST to selfcheck webhook on success only."""
|
||||
webhook_url = settings.MESSAGES_SELFCHECK_WEBHOOK_URL
|
||||
|
||||
@@ -2,13 +2,12 @@
|
||||
|
||||
import base64
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import dns.resolver
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from dkim import DKIM
|
||||
from dkim import sign as dkim_sign
|
||||
from dkim import verify as dkim_verify
|
||||
|
||||
from core.enums import DKIMAlgorithmChoices
|
||||
|
||||
@@ -56,7 +55,7 @@ def generate_dkim_key(
|
||||
return private_key_pem, public_key_b64
|
||||
|
||||
|
||||
def sign_message_dkim(raw_mime_message: bytes, maildomain) -> Optional[bytes]:
|
||||
def sign_message_dkim(raw_mime_message: bytes, maildomain) -> bytes | None:
|
||||
"""Sign a raw MIME message with DKIM.
|
||||
|
||||
Uses the most recent active DKIM key for the domain.
|
||||
@@ -117,7 +116,7 @@ def sign_message_dkim(raw_mime_message: bytes, maildomain) -> Optional[bytes]:
|
||||
return None
|
||||
|
||||
|
||||
def verify_message_dkim(raw_mime_message: bytes) -> bool:
|
||||
def verify_message_dkim(raw_mime_message: bytes) -> str | None:
|
||||
"""Verify a DKIM signature on a raw MIME message using public DNS.
|
||||
|
||||
This verifies that the DKIM signature will pass validation when the receiving
|
||||
@@ -128,7 +127,13 @@ def verify_message_dkim(raw_mime_message: bytes) -> bool:
|
||||
raw_mime_message: The raw bytes of the MIME message with DKIM signature.
|
||||
|
||||
Returns:
|
||||
True if the DKIM signature is valid, False otherwise.
|
||||
The signing domain (the signature's ``d=`` tag, lowercased) if the DKIM
|
||||
signature is valid, otherwise ``None``. Returning the domain rather than
|
||||
a bare bool lets callers enforce identifier alignment against the From:
|
||||
header — a valid signature only proves that *some* domain signed the
|
||||
message, not that the visible From: address is authentic (that is
|
||||
DMARC's job). Callers that only care whether *any* valid signature
|
||||
exists can treat the result as truthy/falsy.
|
||||
"""
|
||||
try:
|
||||
# Create a DNS function that performs actual DNS lookups
|
||||
@@ -166,9 +171,18 @@ def verify_message_dkim(raw_mime_message: bytes) -> bool:
|
||||
|
||||
return None
|
||||
|
||||
# Verify the DKIM signature using public DNS
|
||||
return dkim_verify(raw_mime_message, dnsfunc=get_dns_txt)
|
||||
# Verify the DKIM signature using public DNS. We drive the DKIM object
|
||||
# directly (rather than the module-level ``verify`` helper) so we can
|
||||
# read back the ``d=`` domain of the signature that validated: ``verify``
|
||||
# records it on ``self.domain``.
|
||||
dkim_obj = DKIM(raw_mime_message)
|
||||
if not dkim_obj.verify(dnsfunc=get_dns_txt):
|
||||
return None
|
||||
signing_domain = dkim_obj.domain
|
||||
if not signing_domain:
|
||||
return None
|
||||
return signing_domain.decode("ascii", "replace").rstrip(".").lower()
|
||||
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error during DKIM verification: %s", e, exc_info=True)
|
||||
return False
|
||||
return None
|
||||
|
||||
+110
-70
@@ -3,10 +3,23 @@
|
||||
import logging
|
||||
import smtplib
|
||||
import ssl
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import socks
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SmtpProxy:
|
||||
"""SOCKS5 proxy + EHLO identity to use when sending."""
|
||||
|
||||
host: str
|
||||
port: int
|
||||
username: Optional[str] = None
|
||||
password: Optional[str] = None
|
||||
sender_hostname: Optional[str] = None
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -78,6 +91,58 @@ class ProxySMTP(smtplib.SMTP):
|
||||
)
|
||||
|
||||
|
||||
def _build_tls_context(level: str) -> ssl.SSLContext:
|
||||
"""Build an SSL context matching Postfix's smtp_tls_security_level semantics.
|
||||
|
||||
"secure" performs full PKI verification (CA chain + hostname). "may" creates
|
||||
an unverified context: many public MXes serve mismatched or self-signed
|
||||
certs, and rejecting them would just push delivery to cleartext anyway.
|
||||
"""
|
||||
ctx = ssl.create_default_context()
|
||||
if level != "secure":
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
return ctx
|
||||
|
||||
|
||||
def _starttls_upgrade(
|
||||
client: "ProxySMTP", level: str, sender_hostname: Optional[str]
|
||||
) -> Optional[str]:
|
||||
"""Run the STARTTLS dance, with policy-aware fallback.
|
||||
|
||||
Returns None on success (or when policy allows continuing in cleartext),
|
||||
the sentinel ``"fallback"`` to ask the caller to retry the whole session
|
||||
without TLS (``may`` only), or an error string to defer delivery.
|
||||
"""
|
||||
if level == "none":
|
||||
return None
|
||||
if not client.has_extn("starttls"):
|
||||
if level == "secure":
|
||||
return (
|
||||
"STARTTLS not advertised by server "
|
||||
"(required by smtp_tls_security_level=secure)"
|
||||
)
|
||||
return None
|
||||
try:
|
||||
code, msg = client.starttls(context=_build_tls_context(level))
|
||||
logger.debug("SMTP: STARTTLS response: %s %s", code, msg)
|
||||
if not 200 <= code <= 299:
|
||||
raise RuntimeError(f"STARTTLS rejected: {code} {msg}")
|
||||
code, msg = client.ehlo(sender_hostname)
|
||||
logger.debug("SMTP: EHLO2 response: %s %s", code, msg)
|
||||
if not 200 <= code <= 299:
|
||||
raise RuntimeError(f"EHLO after STARTTLS failed: {code} {msg}")
|
||||
return None
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
if level == "may":
|
||||
logger.warning(
|
||||
"SMTP: STARTTLS failed: %s, falling back to unencrypted socket", e
|
||||
)
|
||||
return "fallback"
|
||||
logger.error("SMTP: Failed to send email with TLS: %s", e, exc_info=True)
|
||||
return "Failed to send email with TLS"
|
||||
|
||||
|
||||
# pylint: disable=too-many-arguments
|
||||
def send_smtp_mail(
|
||||
smtp_host: str,
|
||||
@@ -88,11 +153,7 @@ def send_smtp_mail(
|
||||
smtp_username: Optional[str] = None,
|
||||
smtp_password: Optional[str] = None,
|
||||
timeout: int = 60,
|
||||
proxy_host: Optional[str] = None,
|
||||
proxy_port: Optional[int] = None,
|
||||
proxy_username: Optional[str] = None,
|
||||
proxy_password: Optional[str] = None,
|
||||
sender_hostname: Optional[str] = None,
|
||||
proxy: Optional[SmtpProxy] = None,
|
||||
smtp_ip: Optional[str] = None,
|
||||
smtp_tls_security_level: Optional[str] = "may",
|
||||
) -> Dict[str, Any]:
|
||||
@@ -109,12 +170,8 @@ def send_smtp_mail(
|
||||
smtp_username: SMTP username (optional)
|
||||
smtp_password: SMTP password (optional)
|
||||
timeout: Connection timeout in seconds
|
||||
proxy_host: SOCKS5 proxy hostname
|
||||
proxy_port: SOCKS5 proxy port
|
||||
proxy_username: SOCKS5 proxy username
|
||||
proxy_password: SOCKS5 proxy password
|
||||
sender_hostname: Local hostname to use for SMTP EHLO/HELO
|
||||
smtp_tls_security_level: SMTP TLS security level ("none", "may")
|
||||
proxy: Optional SOCKS5 proxy and local hostname to present in EHLO/HELO
|
||||
smtp_tls_security_level: SMTP TLS security level ("none", "may", "secure")
|
||||
|
||||
Returns:
|
||||
Dict mapping recipient emails to delivery status with retry flag:
|
||||
@@ -127,6 +184,8 @@ def send_smtp_mail(
|
||||
}
|
||||
"""
|
||||
statuses = {}
|
||||
sender_hostname = proxy.sender_hostname if proxy else None
|
||||
proxy_host = proxy.host if proxy else None
|
||||
|
||||
def error_for_all_recipients(error: str, retry: bool) -> Dict[str, Any]:
|
||||
return {
|
||||
@@ -145,9 +204,9 @@ def send_smtp_mail(
|
||||
port=None,
|
||||
timeout=timeout,
|
||||
proxy_host=proxy_host,
|
||||
proxy_port=proxy_port,
|
||||
proxy_username=proxy_username,
|
||||
proxy_password=proxy_password,
|
||||
proxy_port=proxy.port if proxy else None,
|
||||
proxy_username=proxy.username if proxy else None,
|
||||
proxy_password=proxy.password if proxy else None,
|
||||
local_hostname=sender_hostname,
|
||||
)
|
||||
|
||||
@@ -159,7 +218,7 @@ def send_smtp_mail(
|
||||
logger.debug("SMTP: QUIT failed %s", e)
|
||||
|
||||
try:
|
||||
client._host = smtp_host # noqa: SLF001 # pylint: disable=protected-access
|
||||
client._host = smtp_host # noqa: SLF001 # pylint: disable=protected-access,attribute-defined-outside-init
|
||||
(code, msg) = client.connect(smtp_ip or smtp_host, smtp_port)
|
||||
logger.debug(
|
||||
"SMTP: connected to %s:%s (%s %s)", smtp_host, smtp_port, code, msg
|
||||
@@ -180,63 +239,44 @@ def send_smtp_mail(
|
||||
_quit()
|
||||
return error_for_all_recipients(f"HELO failed: {code} {msg}", True)
|
||||
|
||||
if client.has_extn("starttls") and smtp_tls_security_level != "none":
|
||||
try:
|
||||
# smtplib.SMTP.starttls() doesn't validate certificates by default!
|
||||
# https://github.com/python/cpython/issues/91826
|
||||
ssl_context = ssl.create_default_context()
|
||||
ssl_context.check_hostname = True
|
||||
ssl_context.verify_mode = ssl.CERT_REQUIRED
|
||||
(code, msg) = client.starttls(context=ssl_context)
|
||||
logger.debug("SMTP: STARTTLS response: %s %s", code, msg)
|
||||
if not 200 <= code <= 299:
|
||||
_quit()
|
||||
if smtp_tls_security_level == "may":
|
||||
raise Exception(f"STARTTLS failed : {code} {msg}") # pylint: disable=broad-exception-raised
|
||||
return error_for_all_recipients(
|
||||
f"STARTTLS failed: {code} {msg}", True
|
||||
)
|
||||
|
||||
# Restart the SMTP session now that we're in TLS mode
|
||||
(code, msg) = client.ehlo(sender_hostname)
|
||||
logger.debug("SMTP: EHLO2 response: %s %s", code, msg)
|
||||
if not 200 <= code <= 299:
|
||||
_quit()
|
||||
if smtp_tls_security_level == "may":
|
||||
raise Exception(f"STARTTLS failed : {code} {msg}") # pylint: disable=broad-exception-raised
|
||||
return error_for_all_recipients(
|
||||
f"EHLO after STARTTLS failed: {code} {msg}", True
|
||||
)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
if smtp_tls_security_level == "may":
|
||||
logger.warning(
|
||||
"SMTP: STARTTLS failed: %s, falling back to unencrypted socket",
|
||||
e,
|
||||
)
|
||||
return send_smtp_mail(
|
||||
smtp_host=smtp_host,
|
||||
smtp_ip=smtp_ip,
|
||||
smtp_port=smtp_port,
|
||||
envelope_from=envelope_from,
|
||||
recipient_emails=recipient_emails,
|
||||
message_content=message_content,
|
||||
smtp_username=smtp_username,
|
||||
smtp_password=smtp_password,
|
||||
timeout=timeout,
|
||||
proxy_host=proxy_host,
|
||||
proxy_port=proxy_port,
|
||||
proxy_username=proxy_username,
|
||||
proxy_password=proxy_password,
|
||||
sender_hostname=sender_hostname,
|
||||
smtp_tls_security_level="none",
|
||||
)
|
||||
logger.error(
|
||||
"SMTP: Failed to send email with TLS: %s", e, exc_info=True
|
||||
)
|
||||
_quit()
|
||||
return error_for_all_recipients("Failed to send email with TLS", True)
|
||||
tls_result = _starttls_upgrade(client, smtp_tls_security_level, sender_hostname)
|
||||
if tls_result == "fallback":
|
||||
_quit()
|
||||
return send_smtp_mail(
|
||||
smtp_host=smtp_host,
|
||||
smtp_ip=smtp_ip,
|
||||
smtp_port=smtp_port,
|
||||
envelope_from=envelope_from,
|
||||
recipient_emails=recipient_emails,
|
||||
message_content=message_content,
|
||||
smtp_username=smtp_username,
|
||||
smtp_password=smtp_password,
|
||||
timeout=timeout,
|
||||
proxy=proxy,
|
||||
smtp_tls_security_level="none",
|
||||
)
|
||||
if tls_result is not None:
|
||||
_quit()
|
||||
return error_for_all_recipients(tls_result, True)
|
||||
|
||||
if smtp_username and smtp_password:
|
||||
# Refuse to send SMTP AUTH over an unencrypted connection — this
|
||||
# covers both an explicit ``smtp_tls_security_level="none"`` config
|
||||
# and the recursive cleartext retry taken after a "may"-level
|
||||
# STARTTLS failure. Without this guard, a network attacker who can
|
||||
# strip or fail STARTTLS would harvest cleartext relay credentials.
|
||||
if smtp_tls_security_level == "none":
|
||||
_quit()
|
||||
logger.error(
|
||||
"SMTP: refusing AUTH for user '%s' over unencrypted "
|
||||
"connection to %s:%s (TLS unavailable or disabled)",
|
||||
smtp_username,
|
||||
smtp_host,
|
||||
smtp_port,
|
||||
)
|
||||
return error_for_all_recipients(
|
||||
"SMTP AUTH blocked: connection is not TLS-encrypted", True
|
||||
)
|
||||
try:
|
||||
client.login(smtp_username, smtp_password)
|
||||
except smtplib.SMTPAuthenticationError as auth_err:
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
"""Messages-side helpers built on top of :mod:`jmap_email`.
|
||||
|
||||
Three groups of helpers live here:
|
||||
|
||||
- :func:`gmail_labels` and :func:`headers_blocks` — Messages-specific
|
||||
computations against the JMAP ``headers`` list. Neither belongs in
|
||||
the library: the headers they recognise are project conventions
|
||||
(Google Takeout / Dovecot label headers; Received-bounded trust
|
||||
scopes used by the inbound auth path).
|
||||
- :func:`thread_snippet` — the thread-listing snippet derived from
|
||||
``parse_email``'s ``preview`` field, truncated to
|
||||
:data:`SNIPPET_MAX_LENGTH`.
|
||||
- :func:`current_sent_at` — single source of truth for the
|
||||
``sentAt`` ISO-8601 string outbound paths stamp on the JMAP dict
|
||||
they hand to :func:`jmap_email.compose_email`.
|
||||
"""
|
||||
|
||||
import re
|
||||
import shlex
|
||||
from collections import defaultdict
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
from jmap_email import JmapEmail, body_part_text, decode_rfc2047_header
|
||||
|
||||
__all__ = [
|
||||
"SNIPPET_MAX_LENGTH",
|
||||
"current_sent_at",
|
||||
"gmail_labels",
|
||||
"headers_blocks",
|
||||
"thread_snippet",
|
||||
]
|
||||
|
||||
|
||||
SNIPPET_MAX_LENGTH = 140
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
# Date stamping for outbound composition
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def current_sent_at() -> str:
|
||||
"""Return the ISO-8601 ``sentAt`` value outbound composition stamps.
|
||||
|
||||
:func:`jmap_email.compose_email` is strict-by-design and rejects a
|
||||
missing or unparseable ``sentAt``. Every backend code path that
|
||||
composes "now" routes through this helper so the timestamp shape
|
||||
is uniform — currently ``timezone.now().isoformat()``.
|
||||
"""
|
||||
return timezone.now().isoformat()
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
# Thread-listing snippet
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def thread_snippet(parsed_email: JmapEmail, fallback: str = "") -> str:
|
||||
"""Return the thread-listing snippet for a parsed JMAP Email.
|
||||
|
||||
Resolution order:
|
||||
|
||||
1. ``parsed["preview"]`` — the library's spec-default ≤256-char
|
||||
plain-text excerpt, already HTML-stripped and whitespace-
|
||||
normalised.
|
||||
2. The first ``textBody`` part — used when ``parse_email`` was
|
||||
called with ``preview=False`` or when the caller hand-built the
|
||||
JMAP dict (importers, autoreply, MTA-in test fixtures).
|
||||
3. ``fallback`` — when neither preview nor a text body exists.
|
||||
|
||||
Output is always truncated to :data:`SNIPPET_MAX_LENGTH`.
|
||||
"""
|
||||
parsed = parsed_email or {}
|
||||
candidate = parsed.get("preview") or ""
|
||||
if not candidate:
|
||||
text_body = parsed.get("textBody") or []
|
||||
if text_body:
|
||||
candidate = body_part_text(parsed, text_body[0])
|
||||
if not candidate:
|
||||
candidate = fallback or ""
|
||||
return candidate[:SNIPPET_MAX_LENGTH]
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
# Gmail / Dovecot label headers
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Comma-separated form with optional quoted strings — the OfflineIMAP /
|
||||
# Google Takeout convention. Falls back to space-separated (Dovecot) when
|
||||
# no comma is present.
|
||||
_COMMA_LABEL_RE = re.compile(r'\s*"([^"]*)"\s*|\s*([^,]+)')
|
||||
|
||||
|
||||
def _parse_labels_header(labels_str: str) -> list[str]:
|
||||
"""Parse a labels header value, handling quoted strings.
|
||||
|
||||
Supports two formats:
|
||||
|
||||
- Comma-separated (OfflineIMAP / Google Takeout):
|
||||
``label1, label2, "label three"``
|
||||
- Space-separated (Dovecot): ``label1 label2 "label three"``
|
||||
"""
|
||||
result: list[str] = []
|
||||
if "," in labels_str:
|
||||
for quoted, plain in _COMMA_LABEL_RE.findall(labels_str):
|
||||
label = (quoted if quoted else plain).strip()
|
||||
if label:
|
||||
result.append(label)
|
||||
else:
|
||||
try:
|
||||
result = [
|
||||
token.strip() for token in shlex.split(labels_str) if token.strip()
|
||||
]
|
||||
except ValueError:
|
||||
# Unmatched quotes — fall back to a simple split rather than
|
||||
# losing the label list entirely.
|
||||
result = [token.strip() for token in labels_str.split() if token.strip()]
|
||||
return result
|
||||
|
||||
|
||||
def gmail_labels(parsed_email: JmapEmail) -> list[str]:
|
||||
"""Return labels harvested from ``X-Gmail-Labels`` / ``X-Keywords``.
|
||||
|
||||
Deduped in first-seen order. Empty list when neither header is
|
||||
present. Reads the raw header list directly so the library does
|
||||
not need to bake the Google / Dovecot label idiom into its
|
||||
strict-JMAP wire shape.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
labels: list[str] = []
|
||||
for header in parsed_email.get("headers") or []:
|
||||
if not isinstance(header, dict):
|
||||
continue
|
||||
name = (header.get("name") or "").lower()
|
||||
if name not in ("x-gmail-labels", "x-keywords"):
|
||||
continue
|
||||
raw_value = header.get("value") or ""
|
||||
if not raw_value:
|
||||
continue
|
||||
# ``parsed["headers"][*]["value"]`` is the RFC 8621 Raw form
|
||||
# (byte-faithful, no encoded-word decode). Labels routinely
|
||||
# ship as RFC 2047 ``=?UTF-8?Q?…?=`` words (Google Takeout uses
|
||||
# Q-encoding for non-ASCII label text) so decode before
|
||||
# splitting.
|
||||
value = decode_rfc2047_header(raw_value)
|
||||
for label in _parse_labels_header(value):
|
||||
if label not in seen:
|
||||
seen.add(label)
|
||||
labels.append(label)
|
||||
return labels
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
# Received-bounded header trust scopes
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def headers_blocks(
|
||||
parsed_email: JmapEmail,
|
||||
) -> list[dict[str, list[str]]]:
|
||||
"""Return every header grouped into Received-bounded trust scopes.
|
||||
|
||||
Each ``Received`` header marks the END of its block; everything
|
||||
above (earlier) it is in the same trust scope. The trailing
|
||||
Received-less block holds our own MTA prepend. Values inside a
|
||||
block are always lists for uniform downstream indexing.
|
||||
|
||||
Useful for inbound auth (trusted-relay cuts) and spam classifiers
|
||||
that want to discriminate per-hop. Computed on demand so the
|
||||
library's ``ext`` namespace stays free of Messages-specific
|
||||
pre-computation.
|
||||
"""
|
||||
blocks: list[dict[str, list[str]]] = []
|
||||
current: dict[str, list[str]] = defaultdict(list)
|
||||
for header in parsed_email.get("headers") or []:
|
||||
if not isinstance(header, dict):
|
||||
continue
|
||||
name = (header.get("name") or "").lower()
|
||||
value = header.get("value") or ""
|
||||
if name == "received":
|
||||
current["received"].append(value)
|
||||
blocks.append(dict(current))
|
||||
current = defaultdict(list)
|
||||
else:
|
||||
current[name].append(value)
|
||||
if current:
|
||||
blocks.append(dict(current))
|
||||
return blocks
|
||||
@@ -0,0 +1,25 @@
|
||||
# Generated by Django 5.2.11 on 2026-05-27 07:43
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0029_attachment_message_not_null'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='attachment',
|
||||
name='_deprecated_messages',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='blob',
|
||||
name='_deprecated_mailbox',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='blob',
|
||||
name='_deprecated_maildomain',
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 5.2.11 on 2026-06-09 19:04
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0030_remove_attachment__deprecated_messages_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='message',
|
||||
name='mime_id',
|
||||
field=models.CharField(blank=True, db_index=True, max_length=998, null=True, verbose_name='mime id'),
|
||||
),
|
||||
]
|
||||
+79
-80
@@ -30,6 +30,7 @@ from django.utils.text import slugify
|
||||
|
||||
import pyzstd
|
||||
from encrypted_fields.fields import EncryptedJSONField, EncryptedTextField
|
||||
from jmap_email import EmailHeader, JmapEmail, body_part_text, parse_email
|
||||
from timezone_field import TimeZoneField
|
||||
|
||||
from core.enums import (
|
||||
@@ -54,7 +55,6 @@ from core.enums import (
|
||||
thread_event_type_choices,
|
||||
user_event_type_choices,
|
||||
)
|
||||
from core.mda.rfc5322 import EmailParseError, parse_email_message
|
||||
from core.mda.signing import generate_dkim_key as _generate_dkim_key
|
||||
from core.services.tiered_storage import TieredStorageService, sha256_advisory_lock
|
||||
from core.utils import validate_json_schema
|
||||
@@ -735,6 +735,21 @@ class Mailbox(BaseModel):
|
||||
|
||||
return reset_keycloak_user_password(email)
|
||||
|
||||
def set_display_name(self, name):
|
||||
"""Set the mailbox display name through its linked Contact.
|
||||
|
||||
Ensures a Contact (matching the mailbox email) exists and carries
|
||||
``name``, creating and linking it when the mailbox has none yet. This
|
||||
guards against silently dropping the update when ``contact`` is NULL.
|
||||
"""
|
||||
contact, _ = Contact.objects.update_or_create(
|
||||
email=str(self), mailbox=self, defaults={"name": name}
|
||||
)
|
||||
if self.contact_id != contact.id:
|
||||
self.contact = contact
|
||||
self.save(update_fields=["contact"])
|
||||
return contact
|
||||
|
||||
@property
|
||||
def threads_viewer(self):
|
||||
"""Return queryset of threads where the mailbox has at least viewer access."""
|
||||
@@ -1924,7 +1939,9 @@ class Message(BaseModel):
|
||||
sent_at = models.DateTimeField("sent at", null=True, blank=True)
|
||||
archived_at = models.DateTimeField("archived at", null=True, blank=True)
|
||||
|
||||
mime_id = models.CharField("mime id", max_length=998, null=True, blank=True)
|
||||
mime_id = models.CharField(
|
||||
"mime id", max_length=998, null=True, blank=True, db_index=True
|
||||
)
|
||||
|
||||
channel = models.ForeignKey(
|
||||
"Channel",
|
||||
@@ -1974,7 +1991,7 @@ class Message(BaseModel):
|
||||
objects = MessageManager()
|
||||
|
||||
# Internal cache for parsed data
|
||||
_parsed_email_cache: Optional[Dict[str, Any]] = None
|
||||
_parsed_email_cache: Optional[JmapEmail] = None
|
||||
|
||||
class Meta:
|
||||
db_table = "messages_message"
|
||||
@@ -1985,42 +2002,72 @@ class Message(BaseModel):
|
||||
def __str__(self):
|
||||
return str(self.subject) if self.subject else "(no subject)"
|
||||
|
||||
def get_parsed_data(self) -> Dict[str, Any]:
|
||||
"""Parse raw mime message using parser and cache the result."""
|
||||
def get_parsed_data(self) -> JmapEmail:
|
||||
"""Parse the raw MIME message and cache the strict JMAP Email
|
||||
object (RFC 8621 §4) produced by ``jmap_email.parse_email``.
|
||||
|
||||
Use the helpers in :mod:`jmap_email` (``first_address``,
|
||||
``first_msgid``, ``find_header``, …) for null-safe access
|
||||
patterns over the list-typed fields. Returns ``{}`` when
|
||||
there's no blob or parsing fails.
|
||||
"""
|
||||
if self._parsed_email_cache is not None:
|
||||
return self._parsed_email_cache
|
||||
|
||||
if self.blob:
|
||||
# ``body_values=False`` keeps text-body content inlined on
|
||||
# each ``EmailBodyPart`` rather than moving it to a
|
||||
# separate ``bodyValues`` map. The library spec-default is
|
||||
# the moved form (RFC 8621 §4.2 ``defaultProperties`` for
|
||||
# ``Email/get``); this backend's consumers (snippet
|
||||
# extraction, search indexing, LLM formatting, the API
|
||||
# serializer) all read ``content`` inline, so we project
|
||||
# back to that shape at the model boundary.
|
||||
try:
|
||||
self._parsed_email_cache = parse_email_message(self.blob.get_content())
|
||||
except EmailParseError:
|
||||
raw = self.blob.get_content()
|
||||
except ValueError as exc:
|
||||
# ``Blob.get_content`` raises ``ValueError`` on
|
||||
# decompression / decryption / integrity-check failure.
|
||||
logger.warning(
|
||||
"Failed to parse email for message %s, returning empty data",
|
||||
self.id,
|
||||
"Failed to load blob content for message %s: %s", self.id, exc
|
||||
)
|
||||
self._parsed_email_cache = {}
|
||||
return self._parsed_email_cache
|
||||
parsed = parse_email(raw, body_values=False)
|
||||
self._parsed_email_cache = parsed if parsed is not None else {}
|
||||
else:
|
||||
self._parsed_email_cache = {}
|
||||
return self._parsed_email_cache
|
||||
|
||||
def get_parsed_field(self, field_name: str) -> Any:
|
||||
"""Get a parsed field from the parsed email data."""
|
||||
"""Get a parsed field from the parsed JMAP Email object."""
|
||||
return (self.get_parsed_data() or {}).get(field_name)
|
||||
|
||||
def get_mime_headers(self) -> Dict[str, str]:
|
||||
"""Get the MIME headers of the message."""
|
||||
return self.get_parsed_data().get("headers", {})
|
||||
def get_mime_headers(self) -> list[EmailHeader]:
|
||||
"""Return the MIME headers as a JMAP ``EmailHeader[]`` list.
|
||||
|
||||
Each entry is ``{"name": <wire_case>, "value": <decoded>}`` in
|
||||
document order. Use :func:`jmap_email.find_header` for
|
||||
case-insensitive scalar lookups.
|
||||
"""
|
||||
return self.get_parsed_data().get("headers", [])
|
||||
|
||||
def get_stmsg_headers(self) -> Dict[str, str]:
|
||||
"""Get the STMSG headers of the message."""
|
||||
return {
|
||||
k[len("x-stmsg-") :].lower(): v
|
||||
for k, v in self.get_parsed_data().get("headers", {}).items()
|
||||
if k.startswith("x-stmsg-")
|
||||
}
|
||||
"""Return the ``X-StMsg-*`` headers as ``{suffix_lower: value}``.
|
||||
|
||||
``X-StMsg-*`` headers are stamped by our MTA pipeline (one per
|
||||
message) and any sender-supplied copies are stripped before
|
||||
parsing; first occurrence wins on duplicates.
|
||||
"""
|
||||
result: Dict[str, str] = {}
|
||||
for h in self.get_parsed_data().get("headers", []):
|
||||
name = h.get("name", "")
|
||||
if name.lower().startswith("x-stmsg-"):
|
||||
result.setdefault(name[len("x-stmsg-") :].lower(), h.get("value", ""))
|
||||
return result
|
||||
|
||||
def generate_mime_id(self) -> str:
|
||||
"""Get the RFC5322 Message-ID of the message."""
|
||||
"""Get the RFC 5322 Message-ID of the message."""
|
||||
_id = base64.urlsafe_b64encode(uuid.uuid4().bytes).rstrip(b"=").decode("ascii")
|
||||
return f"{_id}@_lst.{self.sender.email.split('@')[1]}"
|
||||
|
||||
@@ -2051,12 +2098,15 @@ class Message(BaseModel):
|
||||
cc = [str(mr.contact) for mr in cc_contacts]
|
||||
# Subject
|
||||
subject = self.subject or "No subject"
|
||||
# Body: try to get text/plain from parsed data
|
||||
# Body: pick the first text/plain text-body part. Transparent to
|
||||
# the body_values projection — body_part_text reads inline
|
||||
# ``content`` or ``bodyValues[partId]`` depending on which the
|
||||
# parser emitted.
|
||||
body = ""
|
||||
parsed_data = self.get_parsed_data()
|
||||
for part in parsed_data.get("textBody", []):
|
||||
if part.get("type") == "text/plain":
|
||||
body = part.get("content", "")
|
||||
body = body_part_text(parsed_data, part)
|
||||
break
|
||||
# Message ID
|
||||
msg_id = str(self.id)
|
||||
@@ -2074,12 +2124,13 @@ class Message(BaseModel):
|
||||
"""Get the number of tokens in the message (subject + body)."""
|
||||
# Subject
|
||||
subject = self.subject or "No subject"
|
||||
# Body: try to get text/plain from parsed data
|
||||
# Body: pick the first text/plain text-body part. See
|
||||
# ``Message.format_for_llm`` for the body_values rationale.
|
||||
body = ""
|
||||
parsed_data = self.get_parsed_data()
|
||||
for part in parsed_data.get("textBody", []):
|
||||
if part.get("type") == "text/plain":
|
||||
body = part.get("content", "")
|
||||
body = body_part_text(parsed_data, part)
|
||||
break
|
||||
counted_text = f"{subject} {body}"
|
||||
return len(counted_text.split())
|
||||
@@ -2123,20 +2174,6 @@ class InboundMessage(BaseModel):
|
||||
class BlobManager(models.Manager):
|
||||
"""Custom Manager for Blob model."""
|
||||
|
||||
def get_queryset(self):
|
||||
"""Defer the rollback-safety ``_deprecated_*`` FKs.
|
||||
|
||||
The columns have already been dropped in some deployed databases
|
||||
but the model fields are intentionally retained for rollback
|
||||
(see ``_deprecated_mailbox`` / ``_deprecated_maildomain``). A
|
||||
default SELECT must not reference the missing columns.
|
||||
"""
|
||||
return (
|
||||
super()
|
||||
.get_queryset()
|
||||
.defer("_deprecated_mailbox", "_deprecated_maildomain")
|
||||
)
|
||||
|
||||
def create_blob(
|
||||
self,
|
||||
content: bytes,
|
||||
@@ -2383,32 +2420,6 @@ class Blob(BaseModel):
|
||||
),
|
||||
)
|
||||
|
||||
# DEPRECATED — kept for rollback safety. Blob lifetime is now governed
|
||||
# by the reference graph + GC sweep (see core/services/blob_gc.py);
|
||||
# these columns are no longer read or written by the application.
|
||||
_deprecated_mailbox = models.ForeignKey(
|
||||
"Mailbox",
|
||||
null=True,
|
||||
blank=True,
|
||||
on_delete=models.SET_NULL,
|
||||
related_name="_deprecated_blobs",
|
||||
help_text=(
|
||||
"DEPRECATED: legacy owner pre-tiered-storage. Kept for rollback; "
|
||||
"do not read or write. To be dropped in a future migration."
|
||||
),
|
||||
)
|
||||
_deprecated_maildomain = models.ForeignKey(
|
||||
"MailDomain",
|
||||
null=True,
|
||||
blank=True,
|
||||
on_delete=models.SET_NULL,
|
||||
related_name="_deprecated_blobs",
|
||||
help_text=(
|
||||
"DEPRECATED: legacy owner pre-tiered-storage. Kept for rollback; "
|
||||
"do not read or write. To be dropped in a future migration."
|
||||
),
|
||||
)
|
||||
|
||||
objects = BlobManager()
|
||||
|
||||
class Meta:
|
||||
@@ -2579,20 +2590,6 @@ class Attachment(BaseModel):
|
||||
help_text="Content-ID for inline images",
|
||||
)
|
||||
|
||||
# DEPRECATED — replaced by ``message`` (FK) above. Through table
|
||||
# ``messages_attachment__deprecated_messages`` is a frozen snapshot
|
||||
# of pre-migration links, kept for rollback safety and audit trail.
|
||||
# Not read or written by the application.
|
||||
_deprecated_messages = models.ManyToManyField(
|
||||
"Message",
|
||||
blank=True,
|
||||
related_name="_deprecated_attachments",
|
||||
help_text=(
|
||||
"DEPRECATED: legacy multi-message linking. Kept for rollback; "
|
||||
"do not read or write. To be dropped in a future migration."
|
||||
),
|
||||
)
|
||||
|
||||
class Meta:
|
||||
db_table = "messages_attachment"
|
||||
verbose_name = "attachment"
|
||||
@@ -3190,7 +3187,8 @@ class MessageTemplate(BaseModel):
|
||||
|
||||
Args:
|
||||
mailbox: Mailbox object — provides `name` via its contact
|
||||
user: User object — fallback for `name` and source of custom attributes
|
||||
user: User object — source of `user_name`, fallback for `name`,
|
||||
and source of custom attributes
|
||||
message: Message object — provides `recipient_name` from TO recipients
|
||||
|
||||
Returns:
|
||||
@@ -3201,7 +3199,8 @@ class MessageTemplate(BaseModel):
|
||||
mailbox.contact.name
|
||||
if mailbox and mailbox.contact
|
||||
else (getattr(user, "full_name", None) if user else "")
|
||||
)
|
||||
) or ""
|
||||
context["user_name"] = (getattr(user, "full_name", None) or "") if user else ""
|
||||
schema = settings.SCHEMA_CUSTOM_ATTRIBUTES_USER
|
||||
schema_properties = schema.get("properties", {})
|
||||
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
"""Storage-safe ICS rebuild.
|
||||
|
||||
Inbound .ics is attacker-controlled (it arrives as an email attachment).
|
||||
Rather than try to enumerate every dangerous extension to strip — METHOD,
|
||||
VALARM with ACTION:EMAIL (Apple amplification), X-MS-OLK-*, X-ALT-DESC
|
||||
HTML, ATTACH with data: URIs, future iCal extensions we haven't heard
|
||||
of — we rebuild a fresh VCALENDAR from a tight allowlist of RFC 5545
|
||||
properties before PUTing to the CalDAV server.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from icalendar import Calendar as ICalendar
|
||||
from icalendar import Event as ICalEvent
|
||||
|
||||
# VEVENT properties kept on the rebuilt event. Everything else is dropped:
|
||||
# notable exclusions are ATTACH (size + scheme abuse), GEO, RESOURCES,
|
||||
# RELATED-TO, REQUEST-STATUS, COMMENT, any X-* extension.
|
||||
_VEVENT_KEEP = frozenset(
|
||||
{
|
||||
# Identity / iTIP versioning
|
||||
"UID",
|
||||
"DTSTAMP",
|
||||
"SEQUENCE",
|
||||
"CREATED",
|
||||
"LAST-MODIFIED",
|
||||
# Time / recurrence
|
||||
"DTSTART",
|
||||
"DTEND",
|
||||
"DURATION",
|
||||
"RRULE",
|
||||
"RDATE",
|
||||
"EXDATE",
|
||||
"RECURRENCE-ID",
|
||||
# Display
|
||||
"SUMMARY",
|
||||
"DESCRIPTION",
|
||||
"LOCATION",
|
||||
"URL",
|
||||
"CATEGORIES",
|
||||
# Semantics
|
||||
"STATUS",
|
||||
"TRANSP",
|
||||
"CLASS",
|
||||
"PRIORITY",
|
||||
"ORGANIZER",
|
||||
"ATTENDEE",
|
||||
}
|
||||
)
|
||||
|
||||
# Per-property parameter allowlist. Anything else (X-*, unknown future
|
||||
# params, attacker-crafted noise) is dropped. Properties not in this map
|
||||
# get all parameters stripped.
|
||||
_PARAM_KEEP = {
|
||||
"DTSTART": frozenset({"TZID", "VALUE"}),
|
||||
"DTEND": frozenset({"TZID", "VALUE"}),
|
||||
"DURATION": frozenset(),
|
||||
"RECURRENCE-ID": frozenset({"TZID", "VALUE", "RANGE"}),
|
||||
"RDATE": frozenset({"TZID", "VALUE"}),
|
||||
"EXDATE": frozenset({"TZID", "VALUE"}),
|
||||
"ATTENDEE": frozenset(
|
||||
{
|
||||
"CN",
|
||||
"PARTSTAT",
|
||||
"ROLE",
|
||||
"CUTYPE",
|
||||
"RSVP",
|
||||
"MEMBER",
|
||||
"DELEGATED-TO",
|
||||
"DELEGATED-FROM",
|
||||
"SENT-BY",
|
||||
"DIR",
|
||||
"LANGUAGE",
|
||||
"SCHEDULE-AGENT",
|
||||
"SCHEDULE-STATUS",
|
||||
}
|
||||
),
|
||||
"ORGANIZER": frozenset(
|
||||
{
|
||||
"CN",
|
||||
"DIR",
|
||||
"SENT-BY",
|
||||
"LANGUAGE",
|
||||
"SCHEDULE-AGENT",
|
||||
"SCHEDULE-STATUS",
|
||||
}
|
||||
),
|
||||
"SUMMARY": frozenset({"LANGUAGE"}),
|
||||
"DESCRIPTION": frozenset({"LANGUAGE"}),
|
||||
"LOCATION": frozenset({"LANGUAGE"}),
|
||||
"CATEGORIES": frozenset({"LANGUAGE"}),
|
||||
}
|
||||
|
||||
# URL property must start with http:// or https://. We don't use
|
||||
# ``urlparse`` here — browsers tolerate whitespace, control chars and
|
||||
# weird Unicode that urlparse rejects, so a string we'd consider "safe"
|
||||
# (because urlparse couldn't extract a dangerous scheme) might still
|
||||
# resolve to ``javascript:`` in the browser. Be stricter than urlparse:
|
||||
# the value must literally start with ``http://`` or ``https://``
|
||||
# (case-insensitive, no leading whitespace).
|
||||
_SAFE_URL_RE = re.compile(r"^https?://", re.IGNORECASE)
|
||||
|
||||
# RRULE frequencies that produce ruinous expansion if unbounded.
|
||||
# Thunderbird hard-froze on a SECONDLY-frequency invite without
|
||||
# COUNT/UNTIL (Mozilla bug 1770984). Reject these unless explicitly
|
||||
# bounded.
|
||||
_RRULE_FREQ_REQUIRES_BOUND = frozenset({"SECONDLY", "MINUTELY"})
|
||||
|
||||
|
||||
def rebuild_for_storage(cal):
|
||||
"""Return a fresh VCALENDAR containing only allowlisted properties.
|
||||
|
||||
Default-deny posture: inbound .ics is attacker-controlled (it
|
||||
arrives as an email attachment), so rather than try to enumerate
|
||||
every dangerous extension to strip — METHOD, VALARM with
|
||||
ACTION:EMAIL (Apple amplification), X-MS-OLK-*, X-ALT-DESC HTML,
|
||||
ATTACH with data: URIs, future iCal extensions we haven't heard
|
||||
of — we rebuild a fresh calendar from a small allowlist of
|
||||
well-understood RFC 5545 properties.
|
||||
|
||||
VTIMEZONE blocks referenced by a kept event's ``TZID`` parameter
|
||||
are preserved so events authored in non-UTC zones still render
|
||||
correctly. VTIMEZONEs not referenced by any kept event are
|
||||
dropped.
|
||||
|
||||
See _VEVENT_KEEP / _PARAM_KEEP for the exact allowlist.
|
||||
"""
|
||||
# Work on a deep copy throughout: ``_filter_params`` mutates the
|
||||
# value objects' ``params`` dicts, and we add components by
|
||||
# reference to ``fresh``. Without the copy the caller's parsed
|
||||
# cal would be silently mutilated after this returns.
|
||||
cal = copy.deepcopy(cal)
|
||||
|
||||
fresh = ICalendar()
|
||||
# Always stamp our own PRODID. Preserving the input's would echo
|
||||
# attacker branding ("Created by Evil Corp") into the user's
|
||||
# calendar and is informationally useless for storage. VERSION
|
||||
# is fixed at 2.0 (RFC 5545); CALSCALE is preserved when present.
|
||||
fresh.add("PRODID", "-//messages//CalDAV interop//EN")
|
||||
fresh.add("VERSION", "2.0")
|
||||
if cal.get("CALSCALE"):
|
||||
fresh.add("CALSCALE", str(cal["CALSCALE"]))
|
||||
|
||||
# Pass 1: rebuild every VEVENT and learn which TZIDs they
|
||||
# actually reference.
|
||||
rebuilt_events = []
|
||||
referenced_tzids = set()
|
||||
for vevent in cal.walk("VEVENT"):
|
||||
clean = _rebuild_event(vevent)
|
||||
if clean is None:
|
||||
continue
|
||||
for prop in ("DTSTART", "DTEND", "RECURRENCE-ID", "RDATE", "EXDATE"):
|
||||
val = clean.get(prop)
|
||||
if val is None:
|
||||
continue
|
||||
for v in val if isinstance(val, list) else [val]:
|
||||
tzid = getattr(v, "params", {}).get("TZID")
|
||||
if tzid:
|
||||
referenced_tzids.add(str(tzid))
|
||||
rebuilt_events.append(clean)
|
||||
|
||||
# Pass 2: preserve only the VTIMEZONEs our kept events use.
|
||||
for vtz in cal.walk("VTIMEZONE"):
|
||||
if str(vtz.get("TZID") or "") in referenced_tzids:
|
||||
fresh.add_component(vtz)
|
||||
|
||||
for evt in rebuilt_events:
|
||||
fresh.add_component(evt)
|
||||
|
||||
return fresh
|
||||
|
||||
|
||||
def _rebuild_event(src):
|
||||
"""Build a fresh VEVENT containing only allowlisted properties.
|
||||
|
||||
Returns ``None`` if the source has no UID (malformed event per
|
||||
RFC 5545 — skip rather than store). A missing DTSTAMP is
|
||||
synthesized (see fallback chain below).
|
||||
|
||||
DTSTAMP caveat: iTIP sequencing (RFC 5546 §3.2.6) compares
|
||||
SEQUENCE first, then DTSTAMP, to decide whether an inbound
|
||||
update supersedes what's already stored. The mainstream
|
||||
generators (Google, Outlook, Apple, …) all emit DTSTAMP on
|
||||
every iTIP message, so this fallback only fires for minimal /
|
||||
hand-crafted invites. When it does fire and we synthesize
|
||||
``now``, a later legitimate UPDATE that carries an *older*
|
||||
DTSTAMP (because the organizer authored it before our store
|
||||
observed the original) can be misread as outdated and dropped
|
||||
— the user appears to be on a stale copy of the event. The
|
||||
LAST-MODIFIED → CREATED preference makes this less likely by
|
||||
preserving the authoring order when those are present; ``now``
|
||||
is a true last-resort that we accept can cause sequence
|
||||
inversion on follow-up updates.
|
||||
"""
|
||||
fresh = ICalEvent()
|
||||
for key in list(src.keys()):
|
||||
upper = key.upper()
|
||||
if upper not in _VEVENT_KEEP:
|
||||
continue
|
||||
value = src[key]
|
||||
for item in value if isinstance(value, list) else [value]:
|
||||
cleaned = _clean_property_value(upper, item)
|
||||
if cleaned is None:
|
||||
continue
|
||||
_filter_params(upper, cleaned)
|
||||
# ``encode=0``: the value is already a typed icalendar
|
||||
# object (vText / vDDDTypes / vCalAddress / vRecur); we
|
||||
# don't want add() to re-encode and lose the params.
|
||||
fresh.add(key, cleaned, encode=0)
|
||||
|
||||
if "UID" not in fresh:
|
||||
return None
|
||||
if "DTSTAMP" not in fresh:
|
||||
# Prefer iTIP versioning info already present on the event
|
||||
# (LAST-MODIFIED → CREATED) over server-now. Setting DTSTAMP
|
||||
# to "now" effectively claims this event was authored this
|
||||
# second, which breaks iTIP sequencing if the organizer ever
|
||||
# sends an update; the original timestamps preserve order.
|
||||
fallback = fresh.get("LAST-MODIFIED") or fresh.get("CREATED")
|
||||
if fallback is not None and hasattr(fallback, "dt"):
|
||||
fresh.add("DTSTAMP", fallback.dt)
|
||||
else:
|
||||
fresh.add("DTSTAMP", datetime.now(tz=timezone.utc))
|
||||
return fresh
|
||||
|
||||
|
||||
def _clean_property_value(prop, value):
|
||||
"""Per-property validation. Returns ``None`` to drop the value."""
|
||||
if prop == "URL":
|
||||
if not _SAFE_URL_RE.match(str(value)):
|
||||
return None
|
||||
elif prop == "RRULE":
|
||||
# icalendar represents RRULE as a vRecur (dict-like) where
|
||||
# each entry is a list (``{"FREQ": ["SECONDLY"]}``).
|
||||
try:
|
||||
freq = (value.get("FREQ") or [""])[0]
|
||||
except (AttributeError, TypeError):
|
||||
return None
|
||||
if str(freq).upper() in _RRULE_FREQ_REQUIRES_BOUND:
|
||||
if not (value.get("COUNT") or value.get("UNTIL")):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _filter_params(prop, value):
|
||||
"""Drop X-* / unknown params from a property value, in place.
|
||||
|
||||
Properties not listed in ``_PARAM_KEEP`` get all their parameters
|
||||
stripped — default-deny matches the rebuild posture.
|
||||
"""
|
||||
params = getattr(value, "params", None)
|
||||
if not params:
|
||||
return
|
||||
allowed = _PARAM_KEEP.get(prop, frozenset())
|
||||
for k in list(params.keys()):
|
||||
if k.upper() not in allowed:
|
||||
del params[k]
|
||||
@@ -0,0 +1,866 @@
|
||||
"""Minimal CalDAV client for calendar invite management.
|
||||
|
||||
Implements only the CalDAV operations we need (list calendars, search
|
||||
events, add event, RSVP) directly over HTTP using ``requests`` and
|
||||
``icalendar`` for parsing, rather than pulling in the full ``caldav``
|
||||
library's dependency surface.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import quote, unquote, urljoin, urlparse
|
||||
|
||||
from django.conf import settings as django_settings
|
||||
|
||||
import defusedxml.ElementTree as ET
|
||||
import requests
|
||||
from defusedxml.ElementTree import ParseError as DefusedParseError
|
||||
from icalendar import Calendar as ICalendar
|
||||
|
||||
from core.services.calendar.ics_rebuild import rebuild_for_storage
|
||||
from core.services.ssrf import (
|
||||
SSRFProtectedAdapter,
|
||||
SSRFValidationError,
|
||||
validate_hostname,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CALDAV_TIMEOUT = 20
|
||||
|
||||
DAV_NS = "DAV:"
|
||||
CALDAV_NS = "urn:ietf:params:xml:ns:caldav"
|
||||
APPLE_ICAL_NS = "http://apple.com/ns/ical/"
|
||||
CS_NS = "http://calendarserver.org/ns/"
|
||||
# suitenumerique/calendars custom extension. Currently exposes
|
||||
# ``calendar-owner-type`` (MAILBOX vs. user's own) so we don't have to
|
||||
# infer it from the principal URL shape.
|
||||
LASUITE_NS = "http://lasuite.numerique.gouv.fr/ns/"
|
||||
|
||||
# Accept only #RRGGBB / #RGB hex from the calendar server's color property —
|
||||
# anything else is treated as no color (defensive against attacker-controlled
|
||||
# values flowing into React inline styles).
|
||||
_HEX_COLOR_RE = re.compile(r"^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{6})$")
|
||||
|
||||
|
||||
def _q(ns, tag):
|
||||
return f"{{{ns}}}{tag}"
|
||||
|
||||
|
||||
def _format_utc(dt):
|
||||
# Reject naive datetimes — silently formatting them as UTC would lie
|
||||
# about the wall-clock value sent to the CalDAV server.
|
||||
if dt.tzinfo is None:
|
||||
raise ValueError("Naive datetime; pass a timezone-aware datetime.")
|
||||
return dt.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
|
||||
|
||||
class CalDAVError(Exception):
|
||||
"""CalDAV protocol or server error.
|
||||
|
||||
``status_code`` is the upstream HTTP status when the failure was an
|
||||
HTTP response (4xx/5xx); None for network-level errors, validation
|
||||
failures, or anything that didn't yield an HTTP status.
|
||||
"""
|
||||
|
||||
def __init__(self, message, status_code=None):
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class CalDAVService: # pylint: disable=too-many-instance-attributes
|
||||
"""Minimal CalDAV client (HTTP + icalendar, no full caldav lib)."""
|
||||
|
||||
def __init__(self, url, username="", password="", headers=None, ssrf_protect=False):
|
||||
self.url = url
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.extra_headers = headers or {}
|
||||
self.ssrf_protect = ssrf_protect
|
||||
self._session = None
|
||||
self._home_set = None
|
||||
# Resolved-and-pinned destination, populated lazily on session
|
||||
# creation when ``ssrf_protect`` is True.
|
||||
self._ssrf_adapter = None
|
||||
|
||||
@property
|
||||
def session(self):
|
||||
"""Lazily-created ``requests.Session`` with auth and headers applied.
|
||||
|
||||
When ``ssrf_protect`` is True (per-mailbox channels — URL is
|
||||
user-supplied and untrusted), an ``SSRFProtectedAdapter`` is
|
||||
mounted so every request resolves the hostname through
|
||||
``validate_hostname`` (rejecting private/loopback IPs) and pins
|
||||
the resulting IP to defeat DNS-rebinding. Untrusted-input
|
||||
scheme/private-IP checks at config time (see ``from_channel``)
|
||||
already filter the obvious cases; this is the per-request
|
||||
guarantee.
|
||||
"""
|
||||
if self._session is None:
|
||||
s = requests.Session()
|
||||
if self.username or self.password:
|
||||
s.auth = (self.username, self.password)
|
||||
s.headers.update(self.extra_headers)
|
||||
if self.ssrf_protect:
|
||||
adapter = self._build_ssrf_adapter()
|
||||
# Mount under the *scheme* prefix so every absolute URL
|
||||
# we issue (PROPFIND/REPORT/PUT, possibly to the server's
|
||||
# advertised href on the same origin) flows through the
|
||||
# pinned adapter.
|
||||
s.mount("https://", adapter)
|
||||
s.mount("http://", adapter)
|
||||
self._ssrf_adapter = adapter
|
||||
self._session = s
|
||||
return self._session
|
||||
|
||||
def _build_ssrf_adapter(self):
|
||||
"""Resolve the configured URL once, pin the IP, return adapter.
|
||||
|
||||
Called lazily on session creation. Raises ``CalDAVError`` if the
|
||||
hostname resolves to a blocked range — propagating up turns this
|
||||
into a 502 to the caller, surfaced as
|
||||
"CalDAV server returned an error" in the UI.
|
||||
"""
|
||||
parsed = urlparse(self.url)
|
||||
if parsed.scheme not in {"http", "https"}:
|
||||
raise CalDAVError(
|
||||
f"CalDAV URL scheme '{parsed.scheme}' is not allowed (http/https only)."
|
||||
)
|
||||
if not parsed.hostname:
|
||||
raise CalDAVError("CalDAV URL has no hostname.")
|
||||
try:
|
||||
ips = validate_hostname(parsed.hostname, allow_ip_literal=False)
|
||||
except SSRFValidationError as exc:
|
||||
raise CalDAVError(f"CalDAV URL host rejected: {exc}") from exc
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
return SSRFProtectedAdapter(
|
||||
dest_ip=ips[0],
|
||||
dest_port=port,
|
||||
original_hostname=parsed.hostname,
|
||||
original_scheme=parsed.scheme,
|
||||
)
|
||||
|
||||
def _request(self, method, url, **kwargs):
|
||||
# Belt-and-braces SSRF guard: the session carries Basic Auth that
|
||||
# would otherwise leak to any host the server response steers us
|
||||
# toward (PROPFIND hrefs are server-controlled and can be absolute,
|
||||
# cross-origin URLs). Pin every outbound request to the configured
|
||||
# CalDAV origin.
|
||||
if not self._same_origin(url):
|
||||
logger.warning(
|
||||
"CalDAV SSRF guard tripped: refused %s to %s (configured: %s)",
|
||||
method,
|
||||
url,
|
||||
self.url,
|
||||
)
|
||||
raise CalDAVError(
|
||||
f"Refusing {method} to {url}: not on configured CalDAV origin."
|
||||
)
|
||||
kwargs.setdefault("timeout", CALDAV_TIMEOUT)
|
||||
# Never follow redirects. The same-origin guard above only validates
|
||||
# the *initial* URL; without this, a CalDAV server (especially a
|
||||
# third-party one configured via a per-mailbox Channel) could 302
|
||||
# us to an attacker-controlled host, bypassing the guard.
|
||||
kwargs.setdefault("allow_redirects", False)
|
||||
try:
|
||||
resp = self.session.request(method, url, **kwargs)
|
||||
except requests.exceptions.RequestException as exc:
|
||||
# Surface network-level failures (timeout, connection reset, DNS,
|
||||
# SSL) as CalDAVError so callers can handle them uniformly with
|
||||
# protocol errors instead of leaking ``requests`` exceptions.
|
||||
raise CalDAVError(f"{method} {url} failed: {exc}") from exc
|
||||
if resp.status_code >= 400:
|
||||
raise CalDAVError(
|
||||
f"{method} {url} failed: HTTP {resp.status_code}",
|
||||
status_code=resp.status_code,
|
||||
)
|
||||
return resp
|
||||
|
||||
def _propfind(self, url, body, depth="0"):
|
||||
return self._request(
|
||||
"PROPFIND",
|
||||
url,
|
||||
data=body.encode("utf-8"),
|
||||
headers={
|
||||
"Depth": str(depth),
|
||||
"Content-Type": "application/xml; charset=utf-8",
|
||||
},
|
||||
)
|
||||
|
||||
@property
|
||||
def home_set(self):
|
||||
"""Calendar-home-set URL, resolved lazily via principal discovery.
|
||||
|
||||
Falls back to the configured URL if discovery fails (for servers or
|
||||
URLs that already point directly at the home set).
|
||||
"""
|
||||
if self._home_set is not None:
|
||||
return self._home_set
|
||||
try:
|
||||
self._home_set = self._discover_home_set() or self.url
|
||||
except (CalDAVError, DefusedParseError, AttributeError) as exc:
|
||||
# CalDAVError: protocol/HTTP/network. DefusedParseError: malformed
|
||||
# PROPFIND XML. AttributeError: a defusedxml node was None where
|
||||
# we expected it (server returned partial body).
|
||||
# Anything else (programmer error) should NOT be swallowed —
|
||||
# let it bubble so a true bug surfaces instead of silently
|
||||
# falling back to the configured URL.
|
||||
logger.debug(
|
||||
"home-set discovery failed for %s (%s), using URL directly",
|
||||
self.url,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
self._home_set = self.url
|
||||
return self._home_set
|
||||
|
||||
def _discover_home_set(self):
|
||||
body = (
|
||||
'<?xml version="1.0"?>'
|
||||
'<d:propfind xmlns:d="DAV:">'
|
||||
"<d:prop><d:current-user-principal/></d:prop>"
|
||||
"</d:propfind>"
|
||||
)
|
||||
root = ET.fromstring(self._propfind(self.url, body, depth="0").text)
|
||||
principal_href = root.findtext(
|
||||
f".//{_q(DAV_NS, 'current-user-principal')}/{_q(DAV_NS, 'href')}"
|
||||
)
|
||||
principal_url = (
|
||||
urljoin(self.url, principal_href.strip()) if principal_href else self.url
|
||||
)
|
||||
|
||||
body = (
|
||||
'<?xml version="1.0"?>'
|
||||
'<d:propfind xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">'
|
||||
"<d:prop><c:calendar-home-set/></d:prop>"
|
||||
"</d:propfind>"
|
||||
)
|
||||
root = ET.fromstring(self._propfind(principal_url, body, depth="0").text)
|
||||
home_href = root.findtext(
|
||||
f".//{_q(CALDAV_NS, 'calendar-home-set')}/{_q(DAV_NS, 'href')}"
|
||||
)
|
||||
if not home_href:
|
||||
return principal_url
|
||||
return urljoin(self.url, home_href.strip())
|
||||
|
||||
def list_calendars(self, writable_only=False):
|
||||
"""List all calendars with a single PROPFIND depth=1 (no N+1).
|
||||
|
||||
When ``writable_only`` is True, calendars the current user cannot
|
||||
write to (read-only shares, subscribed calendars) are filtered out
|
||||
based on the DAV ``current-user-privilege-set``. Servers that do
|
||||
not advertise the privilege set are trusted (the calendar is kept)
|
||||
to avoid hiding legitimate writable calendars on minimal CalDAV
|
||||
implementations.
|
||||
"""
|
||||
body = (
|
||||
'<?xml version="1.0"?>'
|
||||
'<d:propfind xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav"'
|
||||
' xmlns:a="http://apple.com/ns/ical/"'
|
||||
' xmlns:cs="http://calendarserver.org/ns/"'
|
||||
' xmlns:ls="http://lasuite.numerique.gouv.fr/ns/">'
|
||||
"<d:prop>"
|
||||
"<d:displayname/><d:resourcetype/><a:calendar-color/>"
|
||||
"<a:calendar-order/>"
|
||||
"<d:current-user-privilege-set/>"
|
||||
"<cs:invite/>"
|
||||
"<ls:calendar-owner-type/>"
|
||||
"</d:prop>"
|
||||
"</d:propfind>"
|
||||
)
|
||||
root = ET.fromstring(self._propfind(self.home_set, body, depth="1").text)
|
||||
result = []
|
||||
for response in root.findall(_q(DAV_NS, "response")):
|
||||
href = response.findtext(_q(DAV_NS, "href"))
|
||||
if not href:
|
||||
continue
|
||||
href = href.strip()
|
||||
rtype = response.find(f".//{_q(DAV_NS, 'resourcetype')}")
|
||||
if rtype is None or rtype.find(_q(CALDAV_NS, "calendar")) is None:
|
||||
continue
|
||||
if writable_only and not self._response_is_writable(response):
|
||||
continue
|
||||
displayname = response.findtext(f".//{_q(DAV_NS, 'displayname')}") or href
|
||||
color = self._parse_color(
|
||||
response.findtext(f".//{_q(APPLE_ICAL_NS, 'calendar-color')}")
|
||||
)
|
||||
order = self._parse_order(
|
||||
response.findtext(f".//{_q(APPLE_ICAL_NS, 'calendar-order')}")
|
||||
)
|
||||
owner_email, owner_type = self._parse_owner(response)
|
||||
result.append(
|
||||
{
|
||||
"id": urljoin(self.url, href),
|
||||
"name": displayname.strip(),
|
||||
"color": color,
|
||||
"order": order,
|
||||
"owner_email": owner_email,
|
||||
"owner_type": owner_type,
|
||||
}
|
||||
)
|
||||
# Honour the user's manual ordering from the Calendars UI (Apple's
|
||||
# ``calendar-order`` property, written via PROPPATCH by the
|
||||
# suitenumerique/calendars frontend). Calendars without an order
|
||||
# value go after the ordered ones, in original server order — so
|
||||
# adding a new calendar doesn't push it ahead of explicitly-ranked
|
||||
# ones. Python's ``sorted`` is stable, which preserves server
|
||||
# order within each bucket.
|
||||
result.sort(key=lambda c: (c["order"] is None, c["order"] or 0))
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _parse_owner(response):
|
||||
"""Extract (owner_email, owner_type) from a calendar PROPFIND response.
|
||||
|
||||
Owner type comes from the suitenumerique
|
||||
``ls:calendar-owner-type`` extension: ``MAILBOX`` for shared-mailbox
|
||||
calendars, absent (404 in the propstat) for the user's own
|
||||
calendars — which we report as ``USER``. We deliberately avoid
|
||||
guessing the type from the principal URL shape; the extension is
|
||||
authoritative.
|
||||
|
||||
The owner's email is the last non-empty path segment of
|
||||
``cs:invite/cs:organizer/d:href`` — the suite emits a principal
|
||||
href like ``/.../principals/users/<email>`` or
|
||||
``/.../principals/mailboxes/<email>``, and the trailing segment is
|
||||
the address. URL-decoded so percent-encoded ``@`` (RFC-legal but
|
||||
emitted by some servers) matches the plain mailto: addresses in
|
||||
event ATTENDEE entries.
|
||||
|
||||
Returns ``(None, None)`` for CalDAV servers that don't expose
|
||||
``cs:invite`` (non-suitenumerique implementations) — callers should
|
||||
treat that as "owner unknown" and fall back to mailbox-email
|
||||
matching.
|
||||
"""
|
||||
organizer_href = response.findtext(
|
||||
f".//{_q(CS_NS, 'invite')}/{_q(CS_NS, 'organizer')}/{_q(DAV_NS, 'href')}"
|
||||
)
|
||||
if not organizer_href:
|
||||
return None, None
|
||||
last_segment = organizer_href.strip().rstrip("/").rsplit("/", 1)[-1]
|
||||
if not last_segment:
|
||||
return None, None
|
||||
owner_email = unquote(last_segment)
|
||||
|
||||
raw_type = response.findtext(f".//{_q(LASUITE_NS, 'calendar-owner-type')}")
|
||||
owner_type = "MAILBOX" if (raw_type or "").strip() == "MAILBOX" else "USER"
|
||||
return owner_email, owner_type
|
||||
|
||||
@staticmethod
|
||||
def _parse_order(raw):
|
||||
"""Parse Apple ``calendar-order`` (integer sort key) defensively.
|
||||
|
||||
The property is written by the suitenumerique/calendars frontend
|
||||
as a decimal integer, but the value flows through user input
|
||||
(PROPPATCH from the browser) so we must not trust the format.
|
||||
Returns None for missing/malformed values — those calendars sort
|
||||
last while preserving server order among themselves.
|
||||
"""
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return int(raw.strip())
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _parse_color(raw):
|
||||
"""Validate a CalDAV ``calendar-color`` value as a 3- or 6-digit hex.
|
||||
|
||||
Some servers return 8-hex (#RRGGBBAA) — trim alpha for CSS. Anything
|
||||
else (named colors, rgb(), garbage) is rejected to keep
|
||||
attacker-controlled strings out of the frontend's inline ``style``.
|
||||
"""
|
||||
if not raw:
|
||||
return None
|
||||
value = raw.strip()
|
||||
if len(value) == 9 and value.startswith("#"):
|
||||
value = value[:7]
|
||||
return value if _HEX_COLOR_RE.match(value) else None
|
||||
|
||||
@staticmethod
|
||||
def _response_is_writable(response):
|
||||
"""Whether the DAV response advertises a write privilege.
|
||||
|
||||
Absence of ``current-user-privilege-set`` is treated as writable
|
||||
(minimal CalDAV servers don't advertise ACLs); presence with no
|
||||
write-family privilege is treated as read-only.
|
||||
"""
|
||||
priv_set = response.find(f".//{_q(DAV_NS, 'current-user-privilege-set')}")
|
||||
if priv_set is None:
|
||||
return True
|
||||
write_tags = {
|
||||
_q(DAV_NS, "write"),
|
||||
_q(DAV_NS, "write-content"),
|
||||
_q(DAV_NS, "all"),
|
||||
}
|
||||
for priv in priv_set.iter(_q(DAV_NS, "privilege")):
|
||||
for child in priv:
|
||||
if child.tag in write_tags:
|
||||
return True
|
||||
return False
|
||||
|
||||
def check_conflicts(self, start, end, exclude_uid=None, attendee_email=None):
|
||||
"""Find conflicts and the existing PARTSTAT per identity for the UID.
|
||||
|
||||
Returns ``{"conflicts": [...], "existing_partstats": {identity: str}}``.
|
||||
|
||||
Events whose UID matches ``exclude_uid`` are NOT returned as
|
||||
conflicts (a prior import of the same invite should not flag the
|
||||
event as conflicting with itself).
|
||||
|
||||
A mailbox can act through several attendee-owned calendars (the
|
||||
suitenumerique/calendars CalDAV server exposes each calendar's
|
||||
``owner_email``). The stored copy living in calendar X speaks for
|
||||
X's owner, so the excluded event is inspected for *that owner's*
|
||||
PARTSTAT, keyed by identity in ``existing_partstats`` — letting the
|
||||
UI pre-select the right prior RSVP for whichever calendar is
|
||||
selected (and avoid re-prompting for a choice already made).
|
||||
``attendee_email`` is the fallback identity for servers that don't
|
||||
expose ``owner_email``.
|
||||
"""
|
||||
conflicts = []
|
||||
existing_partstats = {}
|
||||
for cal in self.list_calendars():
|
||||
try:
|
||||
events = self._calendar_query(cal["id"], start, end)
|
||||
except CalDAVError:
|
||||
logger.exception(
|
||||
"Error searching for conflicts on calendar %s", cal["name"]
|
||||
)
|
||||
continue
|
||||
# The identity a copy in this calendar speaks for: its owner
|
||||
# when the server exposes it, otherwise the acting mailbox
|
||||
# (servers without owner metadata behave as before).
|
||||
owner_lc = (cal.get("owner_email") or attendee_email or "").lower() or None
|
||||
for ics_text in events:
|
||||
summary = self._summarize_event(ics_text, cal["name"])
|
||||
if summary is None:
|
||||
continue
|
||||
if exclude_uid and summary.get("uid") == exclude_uid:
|
||||
# Per-identity PARTSTAT, keyed by the calendar owner.
|
||||
# First match wins per identity.
|
||||
if owner_lc and owner_lc not in existing_partstats:
|
||||
owner_partstat = self._extract_partstat(ics_text, owner_lc)
|
||||
if owner_partstat is not None:
|
||||
existing_partstats[owner_lc] = owner_partstat
|
||||
continue
|
||||
# UIDs are used only for the self-exclusion filter above —
|
||||
# they can carry internal routing info (incident IDs, etc.)
|
||||
# so don't leak them to the API client.
|
||||
summary.pop("uid", None)
|
||||
conflicts.append(summary)
|
||||
return {"conflicts": conflicts, "existing_partstats": existing_partstats}
|
||||
|
||||
@staticmethod
|
||||
def _extract_partstat(ics_text, attendee_email_lc):
|
||||
"""Return PARTSTAT of ``attendee_email_lc`` (lowercased) in ``ics_text``.
|
||||
|
||||
Returns ``None`` if the event is unparseable or the attendee
|
||||
is absent — callers should treat ``None`` as "no prior RSVP".
|
||||
"""
|
||||
try:
|
||||
cal = ICalendar.from_ical(ics_text)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
return None
|
||||
for comp in cal.walk("VEVENT"):
|
||||
attendees = comp.get("ATTENDEE")
|
||||
if attendees is None:
|
||||
continue
|
||||
if not isinstance(attendees, list):
|
||||
attendees = [attendees]
|
||||
for att in attendees:
|
||||
addr = str(att).strip().lower()
|
||||
if addr.startswith("mailto:"):
|
||||
addr = addr[len("mailto:") :]
|
||||
if addr == attendee_email_lc:
|
||||
val = att.params.get("PARTSTAT")
|
||||
return str(val) if val else None
|
||||
return None
|
||||
|
||||
def _calendar_query(self, calendar_url, start, end):
|
||||
body = (
|
||||
'<?xml version="1.0"?>'
|
||||
'<c:calendar-query xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">'
|
||||
"<d:prop><c:calendar-data/></d:prop>"
|
||||
"<c:filter>"
|
||||
'<c:comp-filter name="VCALENDAR">'
|
||||
'<c:comp-filter name="VEVENT">'
|
||||
f'<c:time-range start="{_format_utc(start)}" end="{_format_utc(end)}"/>'
|
||||
"</c:comp-filter>"
|
||||
"</c:comp-filter>"
|
||||
"</c:filter>"
|
||||
"</c:calendar-query>"
|
||||
)
|
||||
resp = self._request(
|
||||
"REPORT",
|
||||
calendar_url,
|
||||
data=body.encode("utf-8"),
|
||||
headers={
|
||||
"Depth": "1",
|
||||
"Content-Type": "application/xml; charset=utf-8",
|
||||
},
|
||||
)
|
||||
root = ET.fromstring(resp.text)
|
||||
data_key = f".//{_q(CALDAV_NS, 'calendar-data')}"
|
||||
return [
|
||||
data
|
||||
for r in root.findall(_q(DAV_NS, "response"))
|
||||
if (data := r.findtext(data_key))
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _summarize_event(ics_text, calendar_name):
|
||||
try:
|
||||
cal = ICalendar.from_ical(ics_text)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Could not parse conflicting event", exc_info=True)
|
||||
return None
|
||||
for comp in cal.walk("VEVENT"):
|
||||
dtstart = comp.get("DTSTART")
|
||||
dtend = comp.get("DTEND")
|
||||
uid = comp.get("UID")
|
||||
# All-day events carry a ``date`` (not ``datetime``) — surface
|
||||
# the distinction so the UI can format without TZ conversion.
|
||||
# ``new Date("2026-06-01")`` in JS parses as midnight UTC, then
|
||||
# converts to local time; for users west of UTC the date can
|
||||
# display as the day *before*. The ``all_day`` flag lets the
|
||||
# client opt into a date-only formatter for those.
|
||||
all_day = bool(
|
||||
dtstart
|
||||
and hasattr(dtstart, "dt")
|
||||
and not isinstance(dtstart.dt, datetime)
|
||||
)
|
||||
return {
|
||||
"uid": str(uid) if uid else None,
|
||||
"summary": str(comp.get("SUMMARY") or "Untitled event"),
|
||||
"start": dtstart.dt.isoformat() if dtstart else None,
|
||||
"end": dtend.dt.isoformat() if dtend else None,
|
||||
"all_day": all_day,
|
||||
"calendar_name": calendar_name,
|
||||
}
|
||||
return None
|
||||
|
||||
def add_event(self, ics_data, calendar_id=None):
|
||||
"""Store an event on the selected calendar (or the default one)."""
|
||||
cal = ICalendar.from_ical(ics_data)
|
||||
cal = rebuild_for_storage(cal)
|
||||
# Suppress server-side iTIP REQUEST fan-out. sabre/dav's Schedule
|
||||
# plugin (used by suitenumerique/calendars) auto-dispatches one
|
||||
# iTIP REQUEST per ATTENDEE on any PUT where the calendar owner is
|
||||
# the ORGANIZER — turning a single /add/ call into a mass-mailer.
|
||||
# RFC 6638 §7.1 SCHEDULE-AGENT=CLIENT on ORGANIZER tells the server
|
||||
# the client owns scheduling, so the server takes no action.
|
||||
# /add/ is a personal-calendar copy, not an invitation send, so
|
||||
# this is always the right value here. The RSVP path keeps
|
||||
# SCHEDULE-AGENT=SERVER so the REPLY reaches the organizer.
|
||||
self._set_schedule_agent_client(cal)
|
||||
self._put_event(
|
||||
self._pick_calendar_url(calendar_id),
|
||||
cal.to_ical().decode("utf-8"),
|
||||
)
|
||||
return True
|
||||
|
||||
def respond_to_event(self, ics_data, response, attendee_email, calendar_id=None):
|
||||
"""Store an RSVP'd copy of the event on the user's calendar.
|
||||
|
||||
Relies on the CalDAV server's scheduling extension (RFC 6638) to
|
||||
dispatch the iTIP REPLY back to the organizer — ORGANIZER keeps
|
||||
its default SCHEDULE-AGENT=SERVER, so the broker emits the REPLY
|
||||
on PUT. This includes ``DECLINED``: until the organizer removes
|
||||
the user from ATTENDEEs, they remain invited, so the canonical
|
||||
place to record the decline is a stored copy with
|
||||
``PARTSTAT=DECLINED``. The user can re-accept later by changing
|
||||
PARTSTAT on the same event.
|
||||
|
||||
When ``calendar_id`` points to a calendar whose owner principal is
|
||||
itself an ATTENDEE on the event, the PARTSTAT update targets that
|
||||
owner address rather than ``attendee_email``. This is how a user
|
||||
viewing an invite from their *personal* mailbox can RSVP on behalf
|
||||
of a *shared mailbox* by picking the mailbox calendar: the iTIP
|
||||
REPLY is then sent from the right identity (the mailbox, not the
|
||||
personal address). Falls back to ``attendee_email`` when the
|
||||
calendar's owner is not on the invite or the CalDAV server doesn't
|
||||
expose owner info.
|
||||
|
||||
Raises ``CalDAVError`` if neither candidate is on the ATTENDEE
|
||||
list — without that, the iTIP REPLY would never reach the
|
||||
organizer (the broker uses ATTENDEE matching to decide who to
|
||||
notify) and the user would see a "Response saved" toast for a
|
||||
no-op write.
|
||||
"""
|
||||
cal = ICalendar.from_ical(ics_data)
|
||||
calendar = self._pick_calendar(calendar_id)
|
||||
|
||||
# Prefer the selected calendar's owner address — that's the identity
|
||||
# this RSVP speaks for. Fall back to the mailbox address passed in
|
||||
# by the viewset for backends that don't expose owner info.
|
||||
candidates = []
|
||||
owner = calendar.get("owner_email")
|
||||
if owner:
|
||||
candidates.append(owner)
|
||||
if attendee_email and attendee_email not in candidates:
|
||||
candidates.append(attendee_email)
|
||||
|
||||
# Update PARTSTAT on the input first, then rebuild — the rebuild
|
||||
# preserves ATTENDEE entries (with our updated PARTSTAT) and
|
||||
# drops everything else.
|
||||
if not any(
|
||||
self._update_partstat(cal, candidate, response) for candidate in candidates
|
||||
):
|
||||
raise CalDAVError(
|
||||
"Mailbox is not an attendee of this event; "
|
||||
"RSVP would not notify the organizer."
|
||||
)
|
||||
|
||||
cal = rebuild_for_storage(cal)
|
||||
|
||||
self._put_event(
|
||||
calendar["id"],
|
||||
cal.to_ical().decode("utf-8"),
|
||||
)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _set_schedule_agent_client(cal):
|
||||
"""Stamp ``SCHEDULE-AGENT=CLIENT`` on every ORGANIZER (RFC 6638 §7.1).
|
||||
|
||||
Tells the CalDAV server's scheduling extension that the client is
|
||||
handling iTIP delivery itself, so the server MUST NOT auto-dispatch
|
||||
REQUEST/REPLY/CANCEL messages on this PUT. Used by ``add_event``
|
||||
to keep "save a personal copy of this invite" from turning into a
|
||||
mass-mailer via the server's scheduling plugin.
|
||||
|
||||
**Call only from import paths.** The RSVP path
|
||||
(``respond_to_event``) *wants* the server to dispatch the iTIP
|
||||
REPLY back to the organizer, so it must NOT call this helper —
|
||||
flipping SCHEDULE-AGENT to CLIENT there would silently suppress
|
||||
decline/accept notifications while leaving the PUT itself
|
||||
successful (and the user's success toast unchanged). The
|
||||
``test_rsvp_stores_sanitized_copy`` regression test pins the
|
||||
invariant; do not relax it without replacing the notification
|
||||
channel.
|
||||
"""
|
||||
for comp in cal.walk("VEVENT"):
|
||||
organizer = comp.get("ORGANIZER")
|
||||
if organizer is not None:
|
||||
organizer.params["SCHEDULE-AGENT"] = "CLIENT"
|
||||
|
||||
@staticmethod
|
||||
def _update_partstat(cal, attendee_email, new_partstat):
|
||||
"""Update PARTSTAT (and drop RSVP=TRUE) for the given attendee, in-place.
|
||||
|
||||
Returns True if at least one matching ATTENDEE was updated, False
|
||||
otherwise. The boolean lets callers distinguish "RSVP recorded"
|
||||
from "no-op write" — for ``respond_to_event``, a False result
|
||||
means the iTIP REPLY would never reach the organizer.
|
||||
"""
|
||||
email_lower = attendee_email.lower()
|
||||
updated = False
|
||||
for comp in cal.walk("VEVENT"):
|
||||
attendees = comp.get("ATTENDEE")
|
||||
if attendees is None:
|
||||
continue
|
||||
if not isinstance(attendees, list):
|
||||
attendees = [attendees]
|
||||
for att in attendees:
|
||||
addr = str(att).strip().lower()
|
||||
if addr.startswith("mailto:"):
|
||||
addr = addr[len("mailto:") :]
|
||||
if addr != email_lower:
|
||||
continue
|
||||
att.params["PARTSTAT"] = new_partstat
|
||||
att.params.pop("RSVP", None)
|
||||
updated = True
|
||||
return updated
|
||||
|
||||
def _pick_calendar(self, calendar_id):
|
||||
"""Resolve ``calendar_id`` to the matching calendar dict.
|
||||
|
||||
Used by both add_event (needs the URL) and respond_to_event (also
|
||||
needs ``owner_email`` to know which identity the RSVP speaks for).
|
||||
Returns the full calendar dict from ``list_calendars(writable_only=True)``.
|
||||
"""
|
||||
if calendar_id and not self._same_origin(calendar_id):
|
||||
raise CalDAVError(
|
||||
"Calendar URL host does not match the configured CalDAV server."
|
||||
)
|
||||
# Filter to writable calendars: PUT to a read-only share would fail
|
||||
# at the server anyway, and accepting a read-only id here turned
|
||||
# the writeability filter on the list endpoint into UX polish
|
||||
# rather than an enforced precondition. Now the two paths agree.
|
||||
calendars = self.list_calendars(writable_only=True)
|
||||
if not calendars:
|
||||
raise CalDAVError("No writable calendars available on this CalDAV server.")
|
||||
if calendar_id:
|
||||
for cal in calendars:
|
||||
if cal["id"] == calendar_id:
|
||||
return cal
|
||||
raise CalDAVError("Calendar is not in this user's writable calendar list.")
|
||||
return calendars[0]
|
||||
|
||||
def _pick_calendar_url(self, calendar_id):
|
||||
return self._pick_calendar(calendar_id)["id"]
|
||||
|
||||
def _same_origin(self, candidate_url):
|
||||
"""Whether ``candidate_url`` shares scheme + host + port with ``self.url``.
|
||||
|
||||
Normalizes default ports so ``https://host/`` and
|
||||
``https://host:443/`` compare equal — otherwise the SSRF guard
|
||||
falsely rejects legit deployments where the configured URL and
|
||||
server-returned hrefs disagree on whether to include the port.
|
||||
"""
|
||||
cand = urlparse(candidate_url)
|
||||
base = urlparse(self.url)
|
||||
if not cand.scheme or not cand.hostname:
|
||||
return False
|
||||
defaults = {"http": 80, "https": 443}
|
||||
|
||||
def _norm(parsed):
|
||||
scheme = parsed.scheme.lower()
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError:
|
||||
# Malformed port (e.g. non-numeric) → treat as no match.
|
||||
return None
|
||||
return (scheme, parsed.hostname.lower(), port or defaults.get(scheme))
|
||||
|
||||
return _norm(cand) is not None and _norm(cand) == _norm(base)
|
||||
|
||||
def _put_event(self, calendar_url, ics_data):
|
||||
uid = ""
|
||||
try:
|
||||
cal = ICalendar.from_ical(ics_data)
|
||||
for comp in cal.walk("VEVENT"):
|
||||
uid = str(comp.get("UID") or "")
|
||||
break
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.debug("Could not extract UID from ICS, using random", exc_info=True)
|
||||
# UID comes from attacker-controlled ICS data — reject path
|
||||
# separators, traversal sequences and CRLF/NUL before percent-encoding
|
||||
# so the event URL cannot escape the calendar collection.
|
||||
if uid and (any(c in uid for c in "/\\\r\n\x00") or ".." in uid):
|
||||
uid = ""
|
||||
if not uid:
|
||||
uid = str(uuid.uuid4())
|
||||
|
||||
event_url = calendar_url.rstrip("/") + "/" + quote(uid, safe="") + ".ics"
|
||||
data = ics_data.encode("utf-8") if isinstance(ics_data, str) else ics_data
|
||||
self._request(
|
||||
"PUT",
|
||||
event_url,
|
||||
data=data,
|
||||
headers={"Content-Type": "text/calendar; charset=utf-8"},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_channel(cls, channel):
|
||||
"""Create a CalDAVService from a Channel model instance.
|
||||
|
||||
TODO(caldav-per-channel): there is no DRF write path for this yet.
|
||||
``ChannelSerializer.RESERVED_SETTINGS_KEYS`` rejects
|
||||
``username``/``password`` in plaintext ``settings``, and
|
||||
``encrypted_settings`` is not a serializer field — so the only
|
||||
way to provision per-mailbox CalDAV credentials today is via the
|
||||
Django admin / management commands / test factories. The code
|
||||
path is kept live so the test suite exercises it and so the
|
||||
future write path is a small, well-scoped change rather than a
|
||||
new feature.
|
||||
|
||||
Reads non-secret config from ``channel.settings``:
|
||||
- ``url`` — CalDAV server URL.
|
||||
Reads secrets from ``channel.encrypted_settings``:
|
||||
- ``username`` — Basic Auth user.
|
||||
- ``password`` — Basic Auth password.
|
||||
|
||||
Storing credentials in ``settings`` (the plaintext JSONField) is
|
||||
rejected at the serializer layer; the secrets MUST live in
|
||||
``encrypted_settings`` so a DB read does not surface them.
|
||||
"""
|
||||
settings = channel.settings or {}
|
||||
secrets = channel.encrypted_settings or {}
|
||||
url = settings.get("url")
|
||||
if not url:
|
||||
raise ValueError("CalDAV channel is missing 'url' in settings.")
|
||||
# Per-channel URLs are user-supplied (channel configured by a
|
||||
# mailbox admin via Django admin). Opt into the SSRF-pinned
|
||||
# session: ``validate_hostname`` at session-creation time rejects
|
||||
# private/loopback/metadata IPs, and the ``SSRFProtectedAdapter``
|
||||
# pins the resolved IP per request to defeat DNS-rebinding —
|
||||
# neither check applies to ``from_instance_config`` because
|
||||
# operator-supplied env vars are trusted (they may legitimately
|
||||
# point at a private CalDAV instance on the same network).
|
||||
return cls(
|
||||
url=url,
|
||||
username=secrets.get("username") or "",
|
||||
password=secrets.get("password") or "",
|
||||
ssrf_protect=True,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_instance_config(cls, username):
|
||||
"""Create a CalDAVService from the deployment-default CalDAV config.
|
||||
|
||||
Used when no per-mailbox Channel overrides the integration (users
|
||||
can point a Channel at any CalDAV provider; see
|
||||
``from_channel_or_instance``).
|
||||
|
||||
Authenticates with HTTP Basic Auth: ``username`` is the requesting
|
||||
user's *OIDC identity email* (``User.email``) — NOT the mailbox
|
||||
address. The companion CalDAV provider (suitenumerique/calendars)
|
||||
keys principals on the OIDC email claim, and provisions a
|
||||
principal on first request, so the right addressing identity is
|
||||
the human's OIDC email even when they are acting on a mailbox
|
||||
whose ``local_part@domain.name`` differs.
|
||||
|
||||
The password is the single ``CALDAV_DEFAULT_PASSWORD`` value — at
|
||||
the protocol level it is just an HTTP Basic password, but the same
|
||||
value is sent for every user, so it effectively authenticates
|
||||
messages-as-a-service rather than any individual user.
|
||||
|
||||
Trust model: see the comment block on ``CALDAV_DEFAULT_PASSWORD``
|
||||
in ``messages/settings.py``. In short: the CalDAV server trusts
|
||||
whichever email messages claims to act as, so the load-bearing
|
||||
safety property is that the OIDC identity provider does not let
|
||||
one human assert another human's email claim.
|
||||
"""
|
||||
url = django_settings.CALDAV_DEFAULT_URL
|
||||
password = django_settings.CALDAV_DEFAULT_PASSWORD
|
||||
if not url or not password:
|
||||
raise ValueError(
|
||||
"Instance-level CalDAV is not configured "
|
||||
"(CALDAV_DEFAULT_URL and CALDAV_DEFAULT_PASSWORD are required)."
|
||||
)
|
||||
return cls(url=url, username=username, password=password)
|
||||
|
||||
@classmethod
|
||||
def from_channel_or_instance(cls, channel, username):
|
||||
"""Prefer a per-mailbox Channel, falling back to the default config.
|
||||
|
||||
The per-mailbox path lets users override the integration to point
|
||||
at a CalDAV provider of their choice with credentials they own.
|
||||
The default path (``CALDAV_DEFAULT_*`` env vars) is the
|
||||
deployment-wide fallback — see ``from_instance_config`` for its
|
||||
trust model.
|
||||
|
||||
``username`` is the requesting user's OIDC identity email; it is
|
||||
only used by the default path (per-channel credentials are
|
||||
self-contained). See ``from_instance_config`` for why it must
|
||||
be the OIDC email rather than the mailbox address.
|
||||
Returns None if neither is available.
|
||||
"""
|
||||
# TODO(caldav-per-channel): no DRF write path exists for CalDAV
|
||||
# channels yet (see ``from_channel``). In practice this branch is
|
||||
# only reached via admin/management/factory-provisioned rows.
|
||||
if channel:
|
||||
return cls.from_channel(channel)
|
||||
|
||||
if (
|
||||
django_settings.CALDAV_DEFAULT_URL
|
||||
and django_settings.CALDAV_DEFAULT_PASSWORD
|
||||
):
|
||||
return cls.from_instance_config(username)
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Celery tasks for CalDAV calendar operations."""
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from sentry_sdk import capture_exception
|
||||
|
||||
from core.enums import ChannelTypes
|
||||
from core.models import Channel
|
||||
from core.services.calendar.service import CalDAVError, CalDAVService
|
||||
|
||||
from messages.celery_app import app as celery_app
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
# Generic, user-facing error wording. The exception text is logged + sent to
|
||||
# Sentry for diagnosis, but is never surfaced to the API client — exception
|
||||
# strings can include the CalDAV server URL, internal hostnames, or other
|
||||
# details we do not want to render in a toast.
|
||||
_RSVP_FAILURE = "Failed to send the RSVP."
|
||||
_ADD_FAILURE = "Failed to add the event to the calendar."
|
||||
|
||||
|
||||
def _get_caldav_service(channel_id: str | None, user_email: str):
|
||||
"""Build a CalDAVService from a channel ID or instance-level config.
|
||||
|
||||
``user_email`` is the requesting user's OIDC identity email — NOT
|
||||
the acting mailbox's email. It is the Basic Auth username for the
|
||||
instance-level path because the calendars CalDAV provider keys
|
||||
principals on the OIDC ``email`` claim. Per-channel auth is
|
||||
self-contained so the email is ignored there.
|
||||
"""
|
||||
if channel_id:
|
||||
channel = Channel.objects.get(id=channel_id, type=ChannelTypes.CALDAV)
|
||||
return CalDAVService.from_channel(channel)
|
||||
|
||||
return CalDAVService.from_instance_config(user_email)
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def calendar_rsvp_task(
|
||||
self, # pylint: disable=unused-argument
|
||||
channel_id: str | None,
|
||||
user_email: str,
|
||||
ics_data: str,
|
||||
response: str,
|
||||
attendee_email: str,
|
||||
calendar_id: str | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Respond to a calendar event via CalDAV (RSVP).
|
||||
|
||||
Args:
|
||||
channel_id: UUID of the CalDAV channel, or None for instance config
|
||||
user_email: Requesting user's OIDC identity email (Basic Auth user
|
||||
for instance config — addresses the user's principal on the
|
||||
CalDAV server, which keys on the OIDC email claim).
|
||||
ics_data: Raw ICS content
|
||||
response: ACCEPTED, DECLINED, or TENTATIVE
|
||||
attendee_email: Email of the responding attendee in the .ics
|
||||
ATTENDEE list (the mailbox address the invitation was sent
|
||||
to — this is what iTIP matches on, NOT the user's OIDC email).
|
||||
calendar_id: Optional specific calendar URL to use
|
||||
"""
|
||||
try:
|
||||
service = _get_caldav_service(channel_id, user_email)
|
||||
except Channel.DoesNotExist as e:
|
||||
# Race: the row existed when the viewset enqueued the task, gone
|
||||
# by the time the worker ran. Worth a Sentry breadcrumb so we
|
||||
# can see if this happens at any volume.
|
||||
capture_exception(e)
|
||||
logger.warning(
|
||||
"CalDAV channel %s vanished between enqueue and execute", channel_id
|
||||
)
|
||||
return {
|
||||
"status": "FAILURE",
|
||||
"result": None,
|
||||
"error": "CalDAV channel not found.",
|
||||
}
|
||||
except ValueError as e:
|
||||
# Configuration error (URL/password missing). User-facing message
|
||||
# is intentionally generic; full detail is on the worker logs.
|
||||
logger.warning("CalDAV service unavailable: %s", e)
|
||||
return {
|
||||
"status": "FAILURE",
|
||||
"result": None,
|
||||
"error": "CalDAV service is not configured.",
|
||||
}
|
||||
|
||||
try:
|
||||
service.respond_to_event(
|
||||
ics_data=ics_data,
|
||||
response=response,
|
||||
attendee_email=attendee_email,
|
||||
calendar_id=calendar_id,
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "SUCCESS",
|
||||
"result": {"response": response},
|
||||
"error": None,
|
||||
}
|
||||
except CalDAVError as e:
|
||||
# CalDAVError is the protocol-shaped error the service raises for
|
||||
# known failure modes (no attendee match, SSRF-blocked URL, 4xx/5xx
|
||||
# from server). Its message is safe to surface — it is composed
|
||||
# by us, not by ``requests`` or the upstream — and is informative
|
||||
# to the user (e.g. "Mailbox is not an attendee of this event").
|
||||
logger.warning("RSVP failed: %s", e)
|
||||
return {
|
||||
"status": "FAILURE",
|
||||
"result": None,
|
||||
"error": str(e),
|
||||
}
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
# Anything else is unexpected. Stack to Sentry, generic copy to
|
||||
# the client — raw ``str(e)`` from ``requests``/``icalendar`` can
|
||||
# leak the CalDAV URL or other internal details.
|
||||
capture_exception(e)
|
||||
logger.exception("Error responding to calendar event")
|
||||
return {
|
||||
"status": "FAILURE",
|
||||
"result": None,
|
||||
"error": _RSVP_FAILURE,
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def calendar_add_event_task(
|
||||
self, # pylint: disable=unused-argument
|
||||
channel_id: str | None,
|
||||
user_email: str,
|
||||
ics_data: str,
|
||||
calendar_id: str | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Add a calendar event to a CalDAV calendar.
|
||||
|
||||
Args:
|
||||
channel_id: UUID of the CalDAV channel, or None for instance config
|
||||
user_email: Requesting user's OIDC identity email (Basic Auth user
|
||||
for instance config). The event is stored on a calendar owned
|
||||
by this user's CalDAV principal, not on the mailbox's.
|
||||
ics_data: Raw ICS content
|
||||
calendar_id: Optional specific calendar URL to use
|
||||
"""
|
||||
try:
|
||||
service = _get_caldav_service(channel_id, user_email)
|
||||
except Channel.DoesNotExist as e:
|
||||
# Race: the row existed when the viewset enqueued the task, gone
|
||||
# by the time the worker ran. Worth a Sentry breadcrumb so we
|
||||
# can see if this happens at any volume.
|
||||
capture_exception(e)
|
||||
logger.warning(
|
||||
"CalDAV channel %s vanished between enqueue and execute", channel_id
|
||||
)
|
||||
return {
|
||||
"status": "FAILURE",
|
||||
"result": None,
|
||||
"error": "CalDAV channel not found.",
|
||||
}
|
||||
except ValueError as e:
|
||||
# Configuration error (URL/password missing). User-facing message
|
||||
# is intentionally generic; full detail is on the worker logs.
|
||||
logger.warning("CalDAV service unavailable: %s", e)
|
||||
return {
|
||||
"status": "FAILURE",
|
||||
"result": None,
|
||||
"error": "CalDAV service is not configured.",
|
||||
}
|
||||
|
||||
try:
|
||||
service.add_event(ics_data=ics_data, calendar_id=calendar_id)
|
||||
|
||||
return {
|
||||
"status": "SUCCESS",
|
||||
"result": {"added": True},
|
||||
"error": None,
|
||||
}
|
||||
except CalDAVError as e:
|
||||
# See ``calendar_rsvp_task`` — CalDAVError messages are
|
||||
# user-composed and safe to surface.
|
||||
logger.warning("Add-event failed: %s", e)
|
||||
return {
|
||||
"status": "FAILURE",
|
||||
"result": None,
|
||||
"error": str(e),
|
||||
}
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
capture_exception(e)
|
||||
logger.exception("Error adding calendar event")
|
||||
return {
|
||||
"status": "FAILURE",
|
||||
"result": None,
|
||||
"error": _ADD_FAILURE,
|
||||
}
|
||||
@@ -5,8 +5,6 @@ import html
|
||||
import io
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from email.message import EmailMessage
|
||||
from email.utils import format_datetime
|
||||
from typing import Any, Dict
|
||||
|
||||
from django.conf import settings
|
||||
@@ -14,11 +12,12 @@ from django.core.files.storage import storages
|
||||
from django.db.models import OuterRef, Subquery
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from jmap_email import JmapEmail, compose_email, parse_email
|
||||
from sentry_sdk import capture_exception
|
||||
|
||||
from core.api.utils import generate_presigned_url
|
||||
from core.mda.inbound import deliver_inbound_message
|
||||
from core.mda.rfc5322.parser import parse_email_message
|
||||
from core.mda.utils import current_sent_at
|
||||
from core.models import Label, Mailbox, Message, ThreadAccess
|
||||
|
||||
from messages.celery_app import app as celery_app
|
||||
@@ -664,14 +663,6 @@ def _create_notification_message(
|
||||
Returns:
|
||||
True if message was delivered successfully, False otherwise
|
||||
"""
|
||||
# Build the notification email
|
||||
msg = EmailMessage()
|
||||
msg["From"] = f"noreply@{settings.MESSAGES_TECHNICAL_DOMAIN}"
|
||||
msg["To"] = mailbox_email
|
||||
msg["Subject"] = "Your mailbox export is ready"
|
||||
msg["Date"] = format_datetime(datetime.now(timezone.utc))
|
||||
|
||||
# Create message body
|
||||
body_text = f"""Your mailbox export is ready for download.
|
||||
|
||||
Export Summary:
|
||||
@@ -703,16 +694,21 @@ This file is in MBOX format and can be imported into most email clients.
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
msg.set_content(body_text)
|
||||
msg.add_alternative(body_html, subtype="html")
|
||||
notification: JmapEmail = {
|
||||
"from": [{"email": f"noreply@{settings.MESSAGES_TECHNICAL_DOMAIN}"}],
|
||||
"to": [{"email": mailbox_email}],
|
||||
"subject": "Your mailbox export is ready",
|
||||
"sentAt": current_sent_at(),
|
||||
"textBody": [{"partId": "1", "type": "text/plain", "content": body_text}],
|
||||
"htmlBody": [{"partId": "2", "type": "text/html", "content": body_html}],
|
||||
}
|
||||
raw_data = compose_email(notification)
|
||||
parsed_email = parse_email(raw_data)
|
||||
if parsed_email is None:
|
||||
# We just composed this; failing to parse it back means the
|
||||
# composer is broken — bubble up so Sentry catches it.
|
||||
raise RuntimeError("Exporter notification failed to round-trip parse_email")
|
||||
|
||||
# Convert to bytes for delivery
|
||||
raw_data = msg.as_bytes()
|
||||
|
||||
# Parse the email
|
||||
parsed_email = parse_email_message(raw_data)
|
||||
|
||||
# Deliver to the mailbox
|
||||
return deliver_inbound_message(
|
||||
recipient_email=mailbox_email,
|
||||
parsed_email=parsed_email,
|
||||
|
||||
@@ -7,10 +7,10 @@ from django.conf import settings
|
||||
from django.core.files.storage import storages
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from jmap_email import first_address_email, parse_email
|
||||
from sentry_sdk import capture_exception
|
||||
|
||||
from core.mda.inbound import deliver_inbound_message
|
||||
from core.mda.rfc5322 import parse_email_message
|
||||
from core.models import Mailbox
|
||||
from core.utils import ThreadReindexDeferrer, ThreadStatsUpdateDeferrer
|
||||
|
||||
@@ -99,14 +99,28 @@ def process_eml_file_task(self, file_key: str, recipient_id: str) -> Dict[str, A
|
||||
}
|
||||
|
||||
# Parse the email message
|
||||
parsed_email = parse_email_message(file_content)
|
||||
parsed_email = parse_email(file_content)
|
||||
if parsed_email is None:
|
||||
error_msg = "Failed to parse email message"
|
||||
logger.error("%s for key %s", error_msg, file_key)
|
||||
return {
|
||||
"status": "FAILURE",
|
||||
"result": {
|
||||
"status": "FAILURE",
|
||||
"current_message": 1,
|
||||
"success_count": 0,
|
||||
"failure_count": 1,
|
||||
"type": "eml",
|
||||
},
|
||||
"error": error_msg,
|
||||
}
|
||||
|
||||
# Treat the EML as a sent message when From matches the destination
|
||||
# mailbox — the same heuristic IMAP uses against the account
|
||||
# username. Without this flag, importing one's own sent mails would
|
||||
# land them in the inbox view.
|
||||
recipient_email = str(recipient)
|
||||
sender_email = (parsed_email.get("from") or {}).get("email") or ""
|
||||
sender_email = first_address_email(parsed_email.get("from"))
|
||||
# TODO: better heuristic to determine if the message is from the sender
|
||||
is_import_sender = sender_email.lower() == recipient_email.lower()
|
||||
|
||||
|
||||
@@ -19,9 +19,9 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
from django.conf import settings
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from jmap_email import first_address_email, parse_email
|
||||
|
||||
from core.mda.inbound import deliver_inbound_message
|
||||
from core.mda.rfc5322 import parse_email_message
|
||||
from core.services.ssrf import SSRFValidationError, validate_hostname
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
@@ -63,19 +63,61 @@ def decode_imap_utf7(s):
|
||||
return re.sub(r"&([^-]*)-", decode_match, s)
|
||||
|
||||
|
||||
def _validate_imap_host(server: str) -> None:
|
||||
"""Validate that the IMAP server hostname is not a private/internal address.
|
||||
def _validate_imap_host(server: str) -> str:
|
||||
"""Validate the IMAP server hostname and return the vetted IP to pin to.
|
||||
|
||||
Wraps the shared SSRF validator but allows public IP literals, which are
|
||||
legitimate addresses for customer-supplied IMAP servers.
|
||||
Wraps the shared SSRF validator (allowing public IP literals, which are
|
||||
legitimate for customer-supplied IMAP servers) and returns the first
|
||||
validated IP address. The caller connects to *exactly* that address so the
|
||||
address we vetted is the address we dial — closing the DNS-rebinding
|
||||
(TOCTOU) window where stock imaplib would re-resolve the hostname and could
|
||||
land on an internal IP.
|
||||
|
||||
Raises:
|
||||
ValueError: If the hostname resolves to a blocked IP address.
|
||||
ValueError: If the hostname resolves to a blocked / non-public address.
|
||||
"""
|
||||
try:
|
||||
validate_hostname(server, allow_ip_literal=True)
|
||||
valid_ips = validate_hostname(server, allow_ip_literal=True)
|
||||
except SSRFValidationError as exc:
|
||||
raise ValueError(f"IMAP server {server} is not allowed: {exc}") from exc
|
||||
if not valid_ips:
|
||||
raise ValueError(f"IMAP server {server} did not resolve to a usable address")
|
||||
return valid_ips[0]
|
||||
|
||||
|
||||
class _IPPinnedIMAP4(imaplib.IMAP4):
|
||||
"""``imaplib.IMAP4`` that dials a pre-validated IP instead of re-resolving.
|
||||
|
||||
SSRF hardening: ``validate_hostname`` vets the server name, but stock
|
||||
imaplib re-resolves the hostname when it opens the socket — a DNS-rebinding
|
||||
window in which the second lookup can return an internal address. We pin the
|
||||
connection to the already-validated IP. The original hostname is kept as
|
||||
``self.host`` (used for STARTTLS SNI/cert verification upstream).
|
||||
"""
|
||||
|
||||
def __init__(self, host, port, *, connect_ip, timeout=None):
|
||||
self._connect_ip = connect_ip
|
||||
super().__init__(host, port, timeout)
|
||||
|
||||
def _create_socket(self, timeout):
|
||||
return socket.create_connection((self._connect_ip, self.port), timeout)
|
||||
|
||||
|
||||
class _IPPinnedIMAP4SSL(imaplib.IMAP4_SSL):
|
||||
"""SSL variant of :class:`_IPPinnedIMAP4`.
|
||||
|
||||
Connects to the pinned IP but verifies the TLS certificate against the
|
||||
original hostname (``server_hostname`` SNI), so pinning never weakens
|
||||
certificate validation.
|
||||
"""
|
||||
|
||||
def __init__(self, host, port, *, connect_ip, timeout=None):
|
||||
self._connect_ip = connect_ip
|
||||
super().__init__(host, port, timeout=timeout)
|
||||
|
||||
def _create_socket(self, timeout):
|
||||
sock = socket.create_connection((self._connect_ip, self.port), timeout)
|
||||
return self.ssl_context.wrap_socket(sock, server_hostname=self.host)
|
||||
|
||||
|
||||
class IMAPConnectionManager:
|
||||
@@ -92,8 +134,10 @@ class IMAPConnectionManager:
|
||||
self.connection = None
|
||||
|
||||
def __enter__(self):
|
||||
# Validate the server hostname to prevent SSRF
|
||||
_validate_imap_host(self.server)
|
||||
# Validate the server hostname AND pin the vetted IP to prevent SSRF
|
||||
# (including DNS-rebinding TOCTOU): we connect to exactly the address
|
||||
# that passed validation, never a freshly re-resolved one.
|
||||
connect_ip = _validate_imap_host(self.server)
|
||||
|
||||
# Port 143 typically uses STARTTLS, port 993 uses SSL direct
|
||||
# If use_ssl=True and port is 143, use STARTTLS instead of SSL direct
|
||||
@@ -104,8 +148,11 @@ class IMAPConnectionManager:
|
||||
if self.use_ssl and not use_starttls:
|
||||
# SSL direct (typically port 993)
|
||||
try:
|
||||
self.connection = imaplib.IMAP4_SSL(
|
||||
self.server, self.port, timeout=settings.IMAP_TIMEOUT
|
||||
self.connection = _IPPinnedIMAP4SSL(
|
||||
self.server,
|
||||
self.port,
|
||||
connect_ip=connect_ip,
|
||||
timeout=settings.IMAP_TIMEOUT,
|
||||
)
|
||||
except ssl.SSLError as e:
|
||||
# SSL handshake failed - likely wrong port or server doesn't support SSL
|
||||
@@ -118,8 +165,11 @@ class IMAPConnectionManager:
|
||||
raise IMAPSecurityError(error_msg) from e
|
||||
else:
|
||||
# Non-encrypted connection initially (will upgrade to TLS if use_ssl=True)
|
||||
self.connection = imaplib.IMAP4(
|
||||
self.server, self.port, timeout=settings.IMAP_TIMEOUT
|
||||
self.connection = _IPPinnedIMAP4(
|
||||
self.server,
|
||||
self.port,
|
||||
connect_ip=connect_ip,
|
||||
timeout=settings.IMAP_TIMEOUT,
|
||||
)
|
||||
|
||||
if use_starttls:
|
||||
@@ -147,7 +197,7 @@ class IMAPConnectionManager:
|
||||
# else: use_ssl=False, connection remains unencrypted (explicit user choice)
|
||||
|
||||
# Set UTF-8 encoding for the IMAP connection
|
||||
self.connection._encoding = "utf-8" # noqa: SLF001
|
||||
self.connection._encoding = "utf-8" # noqa: SLF001 # pylint: disable=attribute-defined-outside-init
|
||||
|
||||
# Login
|
||||
self.connection.login(self.username, self.password)
|
||||
@@ -515,24 +565,33 @@ def process_folder_messages( # pylint: disable=too-many-arguments
|
||||
failure_count += 1
|
||||
else:
|
||||
# Parse message
|
||||
parsed_email = parse_email_message(raw_email)
|
||||
|
||||
# TODO: better heuristic to determine if the message is from the sender
|
||||
is_sender = parsed_email["from"]["email"].lower() == username.lower()
|
||||
|
||||
# Deliver message
|
||||
if deliver_inbound_message(
|
||||
str(recipient),
|
||||
parsed_email,
|
||||
raw_email,
|
||||
is_import=True,
|
||||
is_import_sender=is_sender,
|
||||
imap_labels=[display_name],
|
||||
imap_flags=flags,
|
||||
):
|
||||
success_count += 1
|
||||
else:
|
||||
parsed_email = parse_email(raw_email)
|
||||
if parsed_email is None:
|
||||
logger.warning(
|
||||
"IMAP: skipping unparseable message %s",
|
||||
msg_num,
|
||||
)
|
||||
failure_count += 1
|
||||
else:
|
||||
# TODO: better heuristic to determine if the message is from the sender
|
||||
is_sender = (
|
||||
first_address_email(parsed_email.get("from")).lower()
|
||||
== username.lower()
|
||||
)
|
||||
|
||||
# Deliver message
|
||||
if deliver_inbound_message(
|
||||
str(recipient),
|
||||
parsed_email,
|
||||
raw_email,
|
||||
is_import=True,
|
||||
is_import_sender=is_sender,
|
||||
imap_labels=[display_name],
|
||||
imap_flags=flags,
|
||||
):
|
||||
success_count += 1
|
||||
else:
|
||||
failure_count += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Label and flag processing for imported messages."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from jmap_email import JmapEmail
|
||||
|
||||
from core import models
|
||||
from core.mda.utils import gmail_labels
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -61,17 +63,16 @@ IMAP_LABELS_TO_IGNORE = [
|
||||
|
||||
|
||||
def compute_labels_and_flags(
|
||||
parsed_email: Dict[str, Any],
|
||||
imap_labels: Optional[List[str]],
|
||||
imap_flags: Optional[List[str]],
|
||||
) -> Tuple[Set[str], Dict[str, bool]]:
|
||||
parsed_email: JmapEmail,
|
||||
imap_labels: list[str] | None,
|
||||
imap_flags: list[str] | None,
|
||||
) -> tuple[set[str], dict[str, bool]]:
|
||||
"""Compute labels and flags for a parsed email."""
|
||||
|
||||
# Combine both imap_labels and gmail_labels from parsed email
|
||||
gmail_labels = parsed_email.get("gmail_labels", [])
|
||||
imap_labels = imap_labels or []
|
||||
imap_flags = imap_flags or []
|
||||
all_labels = list(imap_labels) + list(gmail_labels)
|
||||
all_labels = list(imap_labels) + gmail_labels(parsed_email)
|
||||
|
||||
message_flags = {}
|
||||
labels_to_add = set()
|
||||
@@ -117,9 +118,9 @@ def compute_labels_and_flags(
|
||||
|
||||
def handle_duplicate_message(
|
||||
existing_message: models.Message,
|
||||
parsed_email: Dict[str, Any],
|
||||
imap_labels: List[str],
|
||||
imap_flags: List[str],
|
||||
parsed_email: JmapEmail,
|
||||
imap_labels: list[str],
|
||||
imap_flags: list[str],
|
||||
mailbox: models.Mailbox,
|
||||
) -> None:
|
||||
"""Handle duplicate message by updating labels and flags."""
|
||||
|
||||
@@ -10,11 +10,11 @@ from django.conf import settings
|
||||
from django.core.files.storage import storages
|
||||
|
||||
from celery.utils.log import get_task_logger
|
||||
from jmap_email import first_address_email, parse_email
|
||||
from jmap_email.parser import parse_date
|
||||
from sentry_sdk import capture_exception
|
||||
|
||||
from core.mda.inbound import deliver_inbound_message
|
||||
from core.mda.rfc5322 import parse_email_message
|
||||
from core.mda.rfc5322.parser import parse_date
|
||||
from core.models import Mailbox
|
||||
from core.utils import ThreadReindexDeferrer, ThreadStatsUpdateDeferrer
|
||||
|
||||
@@ -306,7 +306,14 @@ def process_mbox_file_task(self, file_key: str, recipient_id: str) -> Dict[str,
|
||||
failure_count += 1
|
||||
continue
|
||||
|
||||
parsed_email = parse_email_message(message_content)
|
||||
parsed_email = parse_email(message_content)
|
||||
if parsed_email is None:
|
||||
logger.warning(
|
||||
"mbox: skipping unparseable message (%d bytes)",
|
||||
len(message_content),
|
||||
)
|
||||
failure_count += 1
|
||||
continue
|
||||
|
||||
# Treat the message as a sent one when From matches
|
||||
# the destination mailbox — same heuristic as IMAP
|
||||
@@ -314,9 +321,7 @@ def process_mbox_file_task(self, file_key: str, recipient_id: str) -> Dict[str,
|
||||
# one's own sent mails would land them in the inbox
|
||||
# view.
|
||||
recipient_email = str(recipient)
|
||||
sender_email = (parsed_email.get("from") or {}).get(
|
||||
"email"
|
||||
) or ""
|
||||
sender_email = first_address_email(parsed_email.get("from"))
|
||||
# TODO: better heuristic to determine if the message is from the sender
|
||||
is_import_sender = (
|
||||
sender_email.lower() == recipient_email.lower()
|
||||
|
||||
@@ -14,9 +14,15 @@ import re
|
||||
import struct
|
||||
import uuid
|
||||
from email import message_from_string
|
||||
from typing import Generator, Optional, Tuple
|
||||
from typing import Generator
|
||||
|
||||
from core.mda.rfc5322 import compose_email, parse_email_address, parse_email_addresses
|
||||
from jmap_email import (
|
||||
EmailAddress,
|
||||
compose_email,
|
||||
is_valid_msg_id,
|
||||
parse_address,
|
||||
parse_addresses,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -92,8 +98,11 @@ MSGFLAG_UNSENT = 0x8
|
||||
# Flag status values
|
||||
FLAG_STATUS_FOLLOWUP = 2 # Flagged for follow-up
|
||||
|
||||
# Container class prefix for email folders (MAPI standard)
|
||||
EMAIL_CONTAINER_CLASS_PREFIX = "IPF.Note"
|
||||
# Container class prefixes for folders that hold mail items (MAPI standard).
|
||||
# IPF.Note — standard Outlook mail folders.
|
||||
# IPF.Imap — IMAP accounts archived to PST keep this class on their mail
|
||||
# folders (e.g. the inbox), so they must be treated as mail too.
|
||||
EMAIL_CONTAINER_CLASS_PREFIXES = ("IPF.Note", "IPF.Imap")
|
||||
|
||||
# Maximum recursion depth for PST folder traversal
|
||||
MAX_FOLDER_DEPTH = 50
|
||||
@@ -150,7 +159,7 @@ def get_mapi_property(item, property_tag):
|
||||
return None
|
||||
|
||||
|
||||
def get_mapi_property_data(item, property_tag) -> Optional[bytes]:
|
||||
def get_mapi_property_data(item, property_tag) -> bytes | None:
|
||||
"""Get raw data bytes for a MAPI property."""
|
||||
entry = get_mapi_property(item, property_tag)
|
||||
if entry is not None:
|
||||
@@ -161,7 +170,7 @@ def get_mapi_property_data(item, property_tag) -> Optional[bytes]:
|
||||
return None
|
||||
|
||||
|
||||
def get_mapi_property_integer(item, property_tag) -> Optional[int]:
|
||||
def get_mapi_property_integer(item, property_tag) -> int | None:
|
||||
"""Get an integer value for a MAPI property."""
|
||||
entry = get_mapi_property(item, property_tag)
|
||||
if entry is not None:
|
||||
@@ -172,7 +181,7 @@ def get_mapi_property_integer(item, property_tag) -> Optional[int]:
|
||||
return None
|
||||
|
||||
|
||||
def get_mapi_property_string(item, property_tag) -> Optional[str]:
|
||||
def get_mapi_property_string(item, property_tag) -> str | None:
|
||||
"""Get a string value for a MAPI property."""
|
||||
entry = get_mapi_property(item, property_tag)
|
||||
if entry is not None:
|
||||
@@ -189,7 +198,7 @@ def get_mapi_property_string(item, property_tag) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
def _folder_id_from_entry_id(entry_id: Optional[bytes]) -> Optional[int]:
|
||||
def _folder_id_from_entry_id(entry_id: bytes | None) -> int | None:
|
||||
"""Extract the folder identifier from a MAPI entry ID.
|
||||
|
||||
PST entry IDs are 24 bytes: 4 flags + 16 UID + 4 folder_id (LE uint32).
|
||||
@@ -224,7 +233,7 @@ def build_special_folder_map(pst_file) -> dict:
|
||||
return special_map
|
||||
|
||||
|
||||
def get_store_owner_email(pst_file) -> Optional[str]:
|
||||
def get_store_owner_email(pst_file) -> str | None:
|
||||
"""Get the mailbox owner's email from the message store PR_DISPLAY_NAME."""
|
||||
try:
|
||||
store = pst_file.get_message_store()
|
||||
@@ -349,7 +358,7 @@ def _is_email_folder(folder) -> bool:
|
||||
return True
|
||||
try:
|
||||
container_class = entry.data_as_string
|
||||
return container_class.startswith(EMAIL_CONTAINER_CLASS_PREFIX)
|
||||
return container_class.startswith(EMAIL_CONTAINER_CLASS_PREFIXES)
|
||||
except Exception:
|
||||
logger.debug("Failed to read container class for folder")
|
||||
return True
|
||||
@@ -534,17 +543,19 @@ def _get_message_flags(message) -> int:
|
||||
return flags if flags is not None else 0
|
||||
|
||||
|
||||
def _get_flag_status(message) -> Optional[int]:
|
||||
def _get_flag_status(message) -> int | None:
|
||||
"""Get PR_FLAG_STATUS from a message (follow-up flag)."""
|
||||
return get_mapi_property_integer(message, PR_FLAG_STATUS)
|
||||
|
||||
|
||||
def _addr_tuple_to_dict(name: str, addr: str) -> dict:
|
||||
"""Convert a (name, email) tuple to a JMAP address dict."""
|
||||
def _addr_tuple_to_dict(name: str, addr: str) -> EmailAddress:
|
||||
"""Bridge the ``(name, email)`` shape returned by
|
||||
:func:`jmap_email.parse_address` to a JMAP ``EmailAddress`` dict.
|
||||
"""
|
||||
return {"name": name, "email": addr}
|
||||
|
||||
|
||||
def _resolve_smtp_address(item) -> Optional[str]:
|
||||
def _resolve_smtp_address(item) -> str | None:
|
||||
"""Resolve an SMTP email address from a MAPI item (recipient or message).
|
||||
|
||||
Handles Exchange "EX" address types by checking PR_SMTP_ADDRESS first.
|
||||
@@ -582,9 +593,9 @@ def _safe_sender_name(message) -> str:
|
||||
|
||||
def _extract_sender_from_mapi(
|
||||
message,
|
||||
store_email: Optional[str] = None,
|
||||
preferred_name: Optional[str] = None,
|
||||
) -> Optional[dict]:
|
||||
store_email: str | None = None,
|
||||
preferred_name: str | None = None,
|
||||
) -> EmailAddress | None:
|
||||
"""Extract sender address from MAPI properties on the message itself.
|
||||
|
||||
Order of attempts (first hit wins):
|
||||
@@ -605,11 +616,11 @@ def _extract_sender_from_mapi(
|
||||
"""
|
||||
|
||||
def _build(
|
||||
fallback_name: Optional[str],
|
||||
fallback_name: str | None,
|
||||
smtp: str,
|
||||
*,
|
||||
sender_name_fallback: bool = True,
|
||||
) -> dict:
|
||||
) -> EmailAddress:
|
||||
# ``sender_name`` is only a valid display-name fallback when the SMTP
|
||||
# came from a *different* source (PR_SENDER_*, store_email…). When the
|
||||
# SMTP was itself extracted from ``sender_name``, reusing it as a name
|
||||
@@ -655,7 +666,7 @@ def _extract_sender_from_mapi(
|
||||
# 6. Try to parse sender_name as an email address.
|
||||
try:
|
||||
if message.sender_name:
|
||||
parsed_name, addr = parse_email_address(message.sender_name)
|
||||
parsed_name, addr = parse_address(message.sender_name, lenient=True)
|
||||
if addr and "@" in addr:
|
||||
return _build(parsed_name, addr, sender_name_fallback=False)
|
||||
except Exception:
|
||||
@@ -668,10 +679,11 @@ def _extract_sender_from_mapi(
|
||||
return None
|
||||
|
||||
|
||||
def _extract_recipients_from_mapi(message) -> dict:
|
||||
def _extract_recipients_from_mapi(message) -> dict[str, list[EmailAddress]]:
|
||||
"""Extract To/Cc/Bcc recipients from MAPI recipient table.
|
||||
|
||||
Returns dict with 'to', 'cc', 'bcc' keys mapping to lists of address dicts.
|
||||
Returns dict with ``to`` / ``cc`` / ``bcc`` keys mapping to lists of
|
||||
JMAP ``EmailAddress`` entries.
|
||||
"""
|
||||
result = {"to": [], "cc": [], "bcc": []}
|
||||
|
||||
@@ -724,39 +736,42 @@ def _extract_recipients_from_mapi(message) -> dict:
|
||||
return result
|
||||
|
||||
|
||||
def _parse_display_recipients(display_string: Optional[str]) -> list:
|
||||
def _parse_display_recipients(
|
||||
display_string: str | None,
|
||||
) -> list[EmailAddress]:
|
||||
"""Parse Outlook's semicolon-separated To/Cc/Bcc display string.
|
||||
|
||||
Returns a list of JMAP address dicts for entries containing an email
|
||||
address. Name-only entries (no '@') are dropped on purpose — a Contact
|
||||
without an email cannot be created downstream, and silently inventing
|
||||
one would corrupt the address book.
|
||||
Returns a list of JMAP ``EmailAddress`` entries for tokens that
|
||||
contain a usable email address. Name-only entries (no ``@``) are
|
||||
dropped on purpose — a Contact without an email cannot be created
|
||||
downstream, and silently inventing one would corrupt the address
|
||||
book.
|
||||
"""
|
||||
if not display_string:
|
||||
return []
|
||||
|
||||
addresses = []
|
||||
addresses: list[EmailAddress] = []
|
||||
for raw in display_string.split(";"):
|
||||
token = raw.strip()
|
||||
if not token:
|
||||
continue
|
||||
try:
|
||||
name, addr = parse_email_address(token)
|
||||
name, addr = parse_address(token, lenient=True)
|
||||
except Exception:
|
||||
logger.debug("Failed to parse display recipient token")
|
||||
continue
|
||||
if addr and "@" in addr:
|
||||
addresses.append(_addr_tuple_to_dict(name or "", addr))
|
||||
elif "@" in token:
|
||||
# parse_email_address sometimes hands back the address as the
|
||||
# name field when the token is a bare email — recover it.
|
||||
addresses.append(_addr_tuple_to_dict("", token))
|
||||
else:
|
||||
logger.debug("Dropping display recipient with no email")
|
||||
return addresses
|
||||
|
||||
|
||||
def _extract_display_recipients_from_mapi(message) -> dict:
|
||||
def _extract_display_recipients_from_mapi(
|
||||
message,
|
||||
) -> dict[str, list[EmailAddress]]:
|
||||
"""Fall back to PR_DISPLAY_TO/CC/BCC when the recipient table is empty.
|
||||
|
||||
Some PSTs exported from Exchange Online expose ``number_of_recipients=0``
|
||||
@@ -841,34 +856,28 @@ def _apply_recipient_fallback_chain(message, jmap_data: dict) -> None:
|
||||
jmap_data[key] = display_recipients[key]
|
||||
|
||||
|
||||
# Shape mirror of compose_email's _MSG_ID_RE, applied to the bracket-stripped
|
||||
# value. PST archives routinely carry Message-IDs that would crash strict
|
||||
# composition (empty, missing '@', embedded whitespace, nested brackets) —
|
||||
# pre-validating here lets us fall back to MAPI/synth instead of failing the
|
||||
# entire message reconstruction. Multiple '@' are accepted (Outlook/MAPI emit
|
||||
# obs-id-left ids like `foo$@local@domain`); the composer routes In-Reply-To /
|
||||
# References through UnstructuredHeader so those preserve on the wire.
|
||||
_VALID_MSG_ID_INNER_RE = re.compile(r"^[^\s<>]+@[^\s<>]+$")
|
||||
|
||||
|
||||
def _sanitize_message_id(raw: Optional[str]) -> Optional[str]:
|
||||
def _sanitize_message_id(raw: str | None) -> str | None:
|
||||
"""Return ``raw`` stripped of brackets/whitespace if it's a valid msg-id.
|
||||
|
||||
Returns None for anything compose_email would reject (empty, no '@',
|
||||
whitespace, nested brackets…). Keeps the importer "lenient parse,
|
||||
strict compose" contract from collapsing on malformed archives.
|
||||
PST archives routinely carry Message-IDs that strict composition
|
||||
would reject (empty, missing '@', embedded whitespace, nested
|
||||
brackets). Falling back to MAPI / synthesis here keeps the importer
|
||||
"lenient parse, strict compose" contract from collapsing on a
|
||||
malformed archive. The shape predicate is owned by
|
||||
:func:`jmap_email.is_valid_msg_id` so this path stays in lockstep
|
||||
with the composer's strict check.
|
||||
"""
|
||||
if not raw:
|
||||
return None
|
||||
candidate = raw.strip()
|
||||
if candidate.startswith("<") and candidate.endswith(">"):
|
||||
candidate = candidate[1:-1].strip()
|
||||
if not candidate or not _VALID_MSG_ID_INNER_RE.match(candidate):
|
||||
if not is_valid_msg_id(candidate):
|
||||
return None
|
||||
return candidate
|
||||
|
||||
|
||||
def _extract_message_id_from_mapi(message) -> Optional[str]:
|
||||
def _extract_message_id_from_mapi(message) -> str | None:
|
||||
"""Read the native Internet Message-ID stored on the PST message.
|
||||
|
||||
Returns the value of PR_INTERNET_MESSAGE_ID stripped of surrounding
|
||||
@@ -880,7 +889,7 @@ def _extract_message_id_from_mapi(message) -> Optional[str]:
|
||||
)
|
||||
|
||||
|
||||
def _synthesize_message_id(message, recipient_email: Optional[str]) -> str:
|
||||
def _synthesize_message_id(message, recipient_email: str | None) -> str:
|
||||
"""Build a deterministic Message-ID from stable MAPI properties.
|
||||
|
||||
Used as last resort when no native Message-ID is available (typical of
|
||||
@@ -951,14 +960,14 @@ def _synthesize_message_id(message, recipient_email: Optional[str]) -> str:
|
||||
|
||||
def reconstruct_eml(
|
||||
message,
|
||||
store_email: Optional[str] = None,
|
||||
recipient_email: Optional[str] = None,
|
||||
store_email: str | None = None,
|
||||
recipient_email: str | None = None,
|
||||
) -> bytes: # pylint: disable=too-many-branches
|
||||
"""Convert a pypff message to RFC5322 bytes.
|
||||
"""Convert a pypff message to RFC 5322 bytes.
|
||||
|
||||
If transport_headers is available, uses those for threading headers.
|
||||
Otherwise, constructs headers from MAPI properties.
|
||||
Uses the core/mda/rfc5322 compose_email API for MIME construction.
|
||||
Uses the ``jmap-email`` library's ``compose_email`` for MIME construction.
|
||||
|
||||
``recipient_email`` is the import target mailbox; its domain is used to
|
||||
synthesize ``unknown-sender@<domain>`` when no sender can be extracted,
|
||||
@@ -966,7 +975,7 @@ def reconstruct_eml(
|
||||
"""
|
||||
# pylint: disable=too-many-locals,too-many-branches,too-many-statements
|
||||
jmap_data = {}
|
||||
extra_headers = {}
|
||||
extra_headers: list[dict[str, str]] = []
|
||||
|
||||
# Try to get original transport headers for threading-critical fields
|
||||
transport_headers = None
|
||||
@@ -983,11 +992,11 @@ def reconstruct_eml(
|
||||
# parses to a real SMTP address; otherwise fall back to MAPI below,
|
||||
# preserving the human-readable name from the header.
|
||||
from_str = parsed_headers.get("From", "")
|
||||
from_name_hint: Optional[str] = None
|
||||
from_name_hint: str | None = None
|
||||
if from_str:
|
||||
name, addr = parse_email_address(from_str)
|
||||
name, addr = parse_address(from_str, lenient=True)
|
||||
if addr and "@" in addr:
|
||||
jmap_data["from"] = _addr_tuple_to_dict(name, addr)
|
||||
jmap_data["from"] = [_addr_tuple_to_dict(name, addr)]
|
||||
else:
|
||||
from_name_hint = name or None
|
||||
|
||||
@@ -998,25 +1007,25 @@ def reconstruct_eml(
|
||||
preferred_name=from_name_hint,
|
||||
)
|
||||
if sender_dict:
|
||||
jmap_data["from"] = sender_dict
|
||||
jmap_data["from"] = [sender_dict]
|
||||
|
||||
# To / Cc / Bcc — header values are authoritative when present.
|
||||
to_str = parsed_headers.get("To", "")
|
||||
if to_str:
|
||||
jmap_data["to"] = [
|
||||
_addr_tuple_to_dict(n, a) for n, a in parse_email_addresses(to_str)
|
||||
_addr_tuple_to_dict(n, a) for n, a in parse_addresses(to_str)
|
||||
]
|
||||
|
||||
cc_str = parsed_headers.get("Cc", "")
|
||||
if cc_str:
|
||||
jmap_data["cc"] = [
|
||||
_addr_tuple_to_dict(n, a) for n, a in parse_email_addresses(cc_str)
|
||||
_addr_tuple_to_dict(n, a) for n, a in parse_addresses(cc_str)
|
||||
]
|
||||
|
||||
bcc_str = parsed_headers.get("Bcc", "")
|
||||
if bcc_str:
|
||||
jmap_data["bcc"] = [
|
||||
_addr_tuple_to_dict(n, a) for n, a in parse_email_addresses(bcc_str)
|
||||
_addr_tuple_to_dict(n, a) for n, a in parse_addresses(bcc_str)
|
||||
]
|
||||
|
||||
# Subject
|
||||
@@ -1024,10 +1033,10 @@ def reconstruct_eml(
|
||||
if subject:
|
||||
jmap_data["subject"] = subject
|
||||
|
||||
# Date
|
||||
# sentAt
|
||||
date_str = parsed_headers.get("Date")
|
||||
if date_str:
|
||||
jmap_data["date"] = date_str
|
||||
jmap_data["sentAt"] = date_str
|
||||
|
||||
# Message-ID — header is preferred, but Exchange/O365 exports
|
||||
# sometimes strip it or carry a malformed value (empty, missing
|
||||
@@ -1036,31 +1045,31 @@ def reconstruct_eml(
|
||||
# cases.
|
||||
message_id = _sanitize_message_id(parsed_headers.get("Message-ID"))
|
||||
if message_id:
|
||||
jmap_data["messageId"] = message_id
|
||||
jmap_data["messageId"] = [message_id]
|
||||
|
||||
# In-Reply-To and References — pass as custom headers to preserve
|
||||
# exact original values (the in_reply_to parameter on compose_email
|
||||
# would append to References, which we don't want for imports)
|
||||
in_reply_to_val = parsed_headers.get("In-Reply-To")
|
||||
if in_reply_to_val:
|
||||
extra_headers["In-Reply-To"] = in_reply_to_val
|
||||
extra_headers.append({"name": "In-Reply-To", "value": in_reply_to_val})
|
||||
|
||||
references = parsed_headers.get("References")
|
||||
if references:
|
||||
extra_headers["References"] = references
|
||||
extra_headers.append({"name": "References", "value": references})
|
||||
|
||||
else:
|
||||
# Build from MAPI properties — sender
|
||||
sender_dict = _extract_sender_from_mapi(message, store_email=store_email)
|
||||
if sender_dict:
|
||||
jmap_data["from"] = sender_dict
|
||||
jmap_data["from"] = [sender_dict]
|
||||
|
||||
# Date
|
||||
# sentAt
|
||||
try:
|
||||
if message.delivery_time:
|
||||
jmap_data["date"] = message.delivery_time.isoformat()
|
||||
jmap_data["sentAt"] = message.delivery_time.isoformat()
|
||||
elif message.client_submit_time:
|
||||
jmap_data["date"] = message.client_submit_time.isoformat()
|
||||
jmap_data["sentAt"] = message.client_submit_time.isoformat()
|
||||
except Exception:
|
||||
logger.debug("Failed to read message date")
|
||||
|
||||
@@ -1085,9 +1094,18 @@ def reconstruct_eml(
|
||||
if "messageId" not in jmap_data:
|
||||
native_id = _extract_message_id_from_mapi(message)
|
||||
if native_id:
|
||||
jmap_data["messageId"] = native_id
|
||||
jmap_data["messageId"] = [native_id]
|
||||
else:
|
||||
jmap_data["messageId"] = _synthesize_message_id(message, recipient_email)
|
||||
jmap_data["messageId"] = [_synthesize_message_id(message, recipient_email)]
|
||||
|
||||
# ``sentAt`` fallback: the composer is strict-by-design and rejects
|
||||
# a missing Date header. For archive imports we'd rather log a
|
||||
# warning and surface a sentinel epoch than fail the whole message.
|
||||
# The synthesized date is the Unix epoch so a downstream UI can flag
|
||||
# the "no original date" state explicitly.
|
||||
if "sentAt" not in jmap_data:
|
||||
logger.warning("PST message has no resolvable Date; falling back to epoch")
|
||||
jmap_data["sentAt"] = "1970-01-01T00:00:00+00:00"
|
||||
|
||||
# No sender resolvable: synthesize one using the recipient's domain so
|
||||
# compose_email accepts the message. inbound_create.py keeps this value
|
||||
@@ -1104,7 +1122,7 @@ def reconstruct_eml(
|
||||
logger.warning(
|
||||
"PST message has no resolvable sender; using synthesized sender address"
|
||||
)
|
||||
jmap_data["from"] = {"name": "Unknown Sender", "email": fallback_email}
|
||||
jmap_data["from"] = [{"name": "Unknown Sender", "email": fallback_email}]
|
||||
|
||||
# Body parts
|
||||
try:
|
||||
@@ -1140,6 +1158,15 @@ def reconstruct_eml(
|
||||
att_size = attachment.get_size()
|
||||
att_data = attachment.read_buffer(att_size)
|
||||
|
||||
# Skip empty / whitespace-only parts. DSN and read-receipt
|
||||
# reports routinely expose blank diagnostic parts (e.g. an empty
|
||||
# text/rfc822-headers) that libpff surfaces as attachments;
|
||||
# importing them yields 0-byte attachments that render as broken
|
||||
# in the UI while carrying no information.
|
||||
if not att_data or not att_data.strip():
|
||||
logger.debug("Skipping empty attachment %d", i)
|
||||
continue
|
||||
|
||||
# Filename from MAPI properties
|
||||
filename = (
|
||||
get_mapi_property_string(attachment, PR_ATTACH_LONG_FILENAME)
|
||||
@@ -1153,6 +1180,15 @@ def reconstruct_eml(
|
||||
if not mime_type:
|
||||
mime_type = "application/octet-stream"
|
||||
|
||||
# text/rfc822-headers (the original-headers part of a DSN/read
|
||||
# receipt) composes fine but on re-parse the display parser
|
||||
# used to drop the body of this subtype, surfacing it as a
|
||||
# 0-byte attachment in the UI. The content is plain RFC822
|
||||
# header text, so normalize the label to text/plain, which
|
||||
# round-trips intact.
|
||||
if mime_type.split(";")[0].strip().lower() == "text/rfc822-headers":
|
||||
mime_type = "text/plain"
|
||||
|
||||
# Content-ID for inline images
|
||||
content_id = get_mapi_property_string(attachment, PR_ATTACH_CONTENT_ID)
|
||||
|
||||
@@ -1215,7 +1251,7 @@ def _find_ipm_subtree(pst_file):
|
||||
return root
|
||||
|
||||
|
||||
def count_pst_messages(pst_file, special_folder_map: Optional[dict] = None) -> int:
|
||||
def count_pst_messages(pst_file, special_folder_map: dict | None = None) -> int:
|
||||
"""Recursively count email messages across all email folders in a PST file."""
|
||||
if special_folder_map is None:
|
||||
special_folder_map = build_special_folder_map(pst_file)
|
||||
@@ -1252,9 +1288,9 @@ def count_pst_messages(pst_file, special_folder_map: Optional[dict] = None) -> i
|
||||
def walk_pst_messages(
|
||||
pst_file,
|
||||
special_folder_map: dict,
|
||||
store_email: Optional[str] = None,
|
||||
recipient_email: Optional[str] = None,
|
||||
) -> Generator[Tuple[str, str, int, Optional[int], Optional[bytes]], None, None]:
|
||||
store_email: str | None = None,
|
||||
recipient_email: str | None = None,
|
||||
) -> Generator[tuple[str, str, int, int | None, bytes | None], None, None]:
|
||||
"""Walk all email messages in a PST file, yielding them in chronological order.
|
||||
|
||||
First pass: collect lightweight metadata (folder ref + message index).
|
||||
@@ -1395,7 +1431,7 @@ def walk_pst_messages(
|
||||
)
|
||||
except Exception:
|
||||
# Yield None so pst_tasks.py counts this as a failure instead of
|
||||
# silently dropping it (was hidden at debug level previously).
|
||||
# silently dropping it.
|
||||
logger.exception(
|
||||
"Failed to reconstruct EML for message %d in folder %s",
|
||||
msg_idx,
|
||||
|
||||
@@ -8,10 +8,9 @@ from django.core.files.storage import storages
|
||||
|
||||
import pypff
|
||||
from celery.utils.log import get_task_logger
|
||||
from sentry_sdk import capture_exception
|
||||
from jmap_email import parse_email
|
||||
|
||||
from core.mda.inbound import deliver_inbound_message
|
||||
from core.mda.rfc5322 import parse_email_message
|
||||
from core.models import Mailbox
|
||||
from core.utils import ThreadReindexDeferrer, ThreadStatsUpdateDeferrer
|
||||
|
||||
@@ -177,7 +176,14 @@ def process_pst_file_task(self, file_key: str, recipient_id: str) -> Dict[str, A
|
||||
failure_count += 1
|
||||
continue
|
||||
|
||||
parsed_email = parse_email_message(eml_bytes)
|
||||
parsed_email = parse_email(eml_bytes)
|
||||
if parsed_email is None:
|
||||
logger.warning(
|
||||
"PST: skipping unparseable message (%d bytes)",
|
||||
len(eml_bytes),
|
||||
)
|
||||
failure_count += 1
|
||||
continue
|
||||
|
||||
# Compute IMAP-compatible flags from PST message flags
|
||||
imap_flags = []
|
||||
@@ -233,7 +239,8 @@ def process_pst_file_task(self, file_key: str, recipient_id: str) -> Dict[str, A
|
||||
else:
|
||||
failure_count += 1
|
||||
except Exception as e:
|
||||
capture_exception(e)
|
||||
# logger.exception routes to Sentry via the
|
||||
# LoggingIntegration; no separate capture needed.
|
||||
logger.exception(
|
||||
"Error processing message from PST file for recipient %s: %s",
|
||||
recipient_id,
|
||||
@@ -275,7 +282,7 @@ def process_pst_file_task(self, file_key: str, recipient_id: str) -> Dict[str, A
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
capture_exception(e)
|
||||
# logger.exception routes to Sentry via LoggingIntegration.
|
||||
logger.exception(
|
||||
"Error processing PST file for recipient %s: %s",
|
||||
recipient_id,
|
||||
|
||||
@@ -66,17 +66,24 @@ class ImportService:
|
||||
Key=file_key,
|
||||
Range="bytes=0-2047",
|
||||
)["Body"].read()
|
||||
content_type = magic.from_buffer(head, mime=True)
|
||||
|
||||
# Disambiguate ambiguous MIME types using filename extension
|
||||
if content_type in ("text/plain", "application/octet-stream") and filename:
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||
extension_map = {
|
||||
"eml": "message/rfc822",
|
||||
"mbox": "application/mbox",
|
||||
"pst": "application/vnd.ms-outlook",
|
||||
}
|
||||
content_type = extension_map.get(ext, content_type)
|
||||
# RFC 4155: an mbox file starts with a "From " envelope line at offset 0.
|
||||
# Trust that signature first — libmagic can otherwise misclassify mbox
|
||||
# files whose first message body contains HTML as text/html.
|
||||
if head.startswith(b"From "):
|
||||
content_type = "application/mbox"
|
||||
else:
|
||||
content_type = magic.from_buffer(head, mime=True)
|
||||
|
||||
# Disambiguate ambiguous MIME types using filename extension
|
||||
if content_type in ("text/plain", "application/octet-stream") and filename:
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||
extension_map = {
|
||||
"eml": "message/rfc822",
|
||||
"mbox": "application/mbox",
|
||||
"pst": "application/vnd.ms-outlook",
|
||||
}
|
||||
content_type = extension_map.get(ext, content_type)
|
||||
|
||||
if content_type not in enums.ARCHIVE_SUPPORTED_MIME_TYPES:
|
||||
return False, {
|
||||
|
||||
@@ -7,12 +7,12 @@ import logging
|
||||
from django.conf import settings
|
||||
from django.db.models import Prefetch, prefetch_related_objects
|
||||
|
||||
from jmap_email import body_text_joined, parse_email
|
||||
from opensearchpy import OpenSearch
|
||||
from opensearchpy.exceptions import NotFoundError, TransportError
|
||||
from opensearchpy.helpers import bulk
|
||||
|
||||
from core import enums, models
|
||||
from core.mda.rfc5322 import parse_email_message
|
||||
from core.services.search.exceptions import (
|
||||
RETRYABLE_EXCEPTIONS,
|
||||
RETRYABLE_TRANSPORT_STATUS,
|
||||
@@ -208,32 +208,26 @@ def _build_message_doc(message, mailbox_ids, recipients=None):
|
||||
Returns:
|
||||
dict or None if the message blob cannot be parsed.
|
||||
"""
|
||||
parsed_data = {}
|
||||
try:
|
||||
if message.blob:
|
||||
parsed_data = parse_email_message(message.blob.get_content())
|
||||
except models.Blob.DoesNotExist:
|
||||
pass
|
||||
# pylint: disable=broad-exception-caught
|
||||
except Exception as e:
|
||||
logger.error("Error parsing blob content for message %s: %s", message.id, e)
|
||||
return None
|
||||
parsed_data: dict = {}
|
||||
if message.blob:
|
||||
try:
|
||||
raw = message.blob.get_content()
|
||||
except models.Blob.DoesNotExist:
|
||||
raw = None
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error reading blob content for message %s: %s", message.id, e)
|
||||
return None
|
||||
if raw is not None:
|
||||
parsed_data = parse_email(raw)
|
||||
if parsed_data is None:
|
||||
logger.error("parse_email returned None for message %s", message.id)
|
||||
return None
|
||||
|
||||
if recipients is None:
|
||||
recipients = list(message.recipients.select_related("contact").all())
|
||||
|
||||
text_body = ""
|
||||
html_body = ""
|
||||
|
||||
if parsed_data.get("textBody"):
|
||||
text_body = " ".join(
|
||||
item.get("content", "") for item in parsed_data["textBody"]
|
||||
)
|
||||
|
||||
if parsed_data.get("htmlBody"):
|
||||
html_body = " ".join(
|
||||
item.get("content", "") for item in parsed_data["htmlBody"]
|
||||
)
|
||||
text_body = body_text_joined(parsed_data, "textBody")
|
||||
html_body = body_text_joined(parsed_data, "htmlBody")
|
||||
|
||||
return {
|
||||
"relation": {"name": "message", "parent": str(message.thread_id)},
|
||||
|
||||
@@ -41,6 +41,17 @@ def search_threads( # pylint: disable=too-many-branches
|
||||
logger.debug("OpenSearch search is disabled, returning empty results")
|
||||
return {"threads": [], "total": 0, "from": from_offset, "size": size}
|
||||
|
||||
# Scope is mandatory. An empty/None ``mailbox_ids`` would otherwise fall
|
||||
# through to the ``if mailbox_ids:`` filter below and run an unscoped,
|
||||
# cluster-wide search: the returned thread bodies are access-filtered by the
|
||||
# caller, but the hit-total and pagination are not, leaking match counts and
|
||||
# content-existence across every mailbox. The only caller passes the
|
||||
# requesting user's accessible mailboxes, so "no mailboxes" must mean
|
||||
# "no results", never "all mailboxes".
|
||||
if not mailbox_ids:
|
||||
logger.debug("search_threads called without mailbox_ids; returning empty")
|
||||
return {"threads": [], "total": 0, "from": from_offset, "size": size}
|
||||
|
||||
try: # pylint: disable=too-many-nested-blocks
|
||||
es = get_opensearch_client()
|
||||
|
||||
|
||||
@@ -38,6 +38,31 @@ def _check_ip(ip_addr: ipaddress._BaseAddress, hostname: str) -> None:
|
||||
raise SSRFValidationError(f"{hostname} resolves to reserved address")
|
||||
if ip_addr.is_private:
|
||||
raise SSRFValidationError(f"{hostname} resolves to private IP address")
|
||||
# Catch-all for anything not globally routable that the specific checks
|
||||
# above miss — notably shared address space / CGNAT (100.64.0.0/10), which
|
||||
# is neither is_private nor is_reserved in Python's ipaddress module.
|
||||
if not ip_addr.is_global:
|
||||
raise SSRFValidationError(f"{hostname} resolves to non-global address")
|
||||
|
||||
|
||||
def assert_public_ip(ip: str, hostname: str = "") -> None:
|
||||
"""Raise ``SSRFValidationError`` unless ``ip`` is a public address.
|
||||
|
||||
Companion to ``validate_hostname`` for callers that have *already*
|
||||
resolved a destination to a concrete IP and dial that exact IP — e.g.
|
||||
outbound SMTP, which pins an MX host's A record and connects to it
|
||||
directly (so there is no DNS-rebinding window to defend, only the IP to
|
||||
vet). Blocks loopback / link-local / multicast / reserved / private
|
||||
ranges and the cloud-metadata endpoints, plus a final ``is_global``
|
||||
catch-all (see ``_check_ip``) that rejects any remaining non-globally-
|
||||
routable address — notably CGNAT / shared address space (100.64.0.0/10),
|
||||
which is neither ``is_private`` nor ``is_reserved`` in Python's ipaddress.
|
||||
"""
|
||||
try:
|
||||
ip_addr = ipaddress.ip_address(ip)
|
||||
except ValueError as exc:
|
||||
raise SSRFValidationError(f"Invalid IP address {ip!r}") from exc
|
||||
_check_ip(ip_addr, hostname or ip)
|
||||
|
||||
|
||||
def validate_hostname(hostname: str, *, allow_ip_literal: bool = False) -> list[str]:
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
from core.mda.inbound_tasks import * # noqa: F403
|
||||
from core.mda.outbound_tasks import * # noqa: F403
|
||||
from core.services.blob_gc import * # noqa: F403
|
||||
from core.services.calendar.tasks import * # noqa: F403
|
||||
from core.services.dns.tasks import * # noqa: F403
|
||||
from core.services.importer.eml_tasks import * # noqa: F403
|
||||
from core.services.importer.imap_tasks import * # noqa: F403
|
||||
|
||||
@@ -825,14 +825,21 @@ class TestAdminMailDomainMailboxViewSet:
|
||||
response = api_client.post(url, data=data, format="json")
|
||||
assert response.status_code == status.HTTP_201_CREATED
|
||||
|
||||
def test_admin_maildomains_mailbox_create_personal_blocked_when_no_identity_sync(
|
||||
@override_settings(IDENTITY_PROVIDER=None)
|
||||
def test_admin_maildomains_mailbox_create_personal_allowed_when_no_identity_sync(
|
||||
self,
|
||||
api_client,
|
||||
domain_admin_user,
|
||||
domain_admin_access1,
|
||||
mail_domain1,
|
||||
):
|
||||
"""Creating a personal mailbox should fail when identity_sync is disabled."""
|
||||
"""Creating a personal mailbox should succeed when identity_sync is disabled,
|
||||
even when no identity provider (e.g. Keycloak) is configured at all.
|
||||
|
||||
This lets admins pre-create mailboxes for users who connect through a
|
||||
third-party (non-synced) identity provider. No one-time password is
|
||||
provisioned, but the mailbox and its identity user are created.
|
||||
"""
|
||||
mail_domain1.identity_sync = False
|
||||
mail_domain1.save()
|
||||
|
||||
@@ -847,8 +854,20 @@ class TestAdminMailDomainMailboxViewSet:
|
||||
},
|
||||
}
|
||||
response = api_client.post(url, data=data, format="json")
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "identity_sync" in response.data
|
||||
assert response.status_code == status.HTTP_201_CREATED
|
||||
assert response.data["is_identity"] is True
|
||||
# No password is provisioned when identity sync is disabled.
|
||||
assert "one_time_password" not in response.data
|
||||
|
||||
mailbox = models.Mailbox.objects.get(local_part="john.doe", domain=mail_domain1)
|
||||
assert mailbox.is_identity is True
|
||||
# No password can be provisioned without a configured identity provider.
|
||||
assert mailbox.can_reset_password is False
|
||||
# The identity user and its access were created.
|
||||
user = models.User.objects.get(email=str(mailbox))
|
||||
assert mailbox.accesses.filter(
|
||||
user=user, role=models.MailboxRoleChoices.ADMIN
|
||||
).exists()
|
||||
|
||||
def test_admin_maildomains_mailbox_create_shared_allowed_when_no_identity_sync(
|
||||
self,
|
||||
@@ -918,15 +937,21 @@ class TestAdminMailDomainMailboxViewSet:
|
||||
response = api_client.post(url, data=data, format="json")
|
||||
assert response.status_code == status.HTTP_201_CREATED
|
||||
|
||||
@patch("core.services.identity.keycloak.reset_keycloak_user_password")
|
||||
@patch("core.signals.sync_mailbox_to_keycloak_user")
|
||||
@override_settings(IDENTITY_PROVIDER="keycloak")
|
||||
def test_admin_maildomains_mailbox_create_personal_without_maildomain_identity_sync(
|
||||
self,
|
||||
mock_sync_mailbox,
|
||||
mock_reset_password,
|
||||
api_client,
|
||||
domain_admin_user,
|
||||
domain_admin_access1,
|
||||
mail_domain1,
|
||||
):
|
||||
"""Test that personal mailbox creation is blocked when maildomain identity_sync is False."""
|
||||
"""Personal mailbox creation succeeds without a password and without
|
||||
Keycloak sync when the maildomain has identity_sync disabled, even with
|
||||
Keycloak configured as the identity provider."""
|
||||
api_client.force_authenticate(user=domain_admin_user)
|
||||
url = self.mailboxes_url(mail_domain1.pk)
|
||||
|
||||
@@ -940,8 +965,13 @@ class TestAdminMailDomainMailboxViewSet:
|
||||
|
||||
response = api_client.post(url, data, format="json")
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "identity_sync" in response.data
|
||||
assert response.status_code == status.HTTP_201_CREATED
|
||||
assert response.data["local_part"] == "testuser"
|
||||
# No password is provisioned and no Keycloak sync happens when
|
||||
# identity_sync is disabled, regardless of the identity provider.
|
||||
assert "one_time_password" not in response.data
|
||||
mock_reset_password.assert_not_called()
|
||||
mock_sync_mailbox.assert_not_called()
|
||||
|
||||
@patch("core.services.identity.keycloak.reset_keycloak_user_password")
|
||||
@override_settings(IDENTITY_PROVIDER="other_provider")
|
||||
@@ -1187,6 +1217,71 @@ class TestAdminMailDomainMailboxViewSet:
|
||||
mailbox2_domain1.refresh_from_db()
|
||||
assert mailbox2_domain1.contact.name == "Helpdesk"
|
||||
|
||||
def test_admin_maildomains_mailbox_partial_update_shared_creates_missing_contact(
|
||||
self,
|
||||
api_client,
|
||||
domain_admin_user,
|
||||
domain_admin_access1,
|
||||
mail_domain1,
|
||||
):
|
||||
"""Renaming a shared mailbox that has no contact yet should create and
|
||||
link one, instead of silently dropping the update."""
|
||||
api_client.force_authenticate(user=domain_admin_user)
|
||||
|
||||
mailbox = factories.MailboxFactory(
|
||||
domain=mail_domain1,
|
||||
local_part="orphan-shared",
|
||||
is_identity=False,
|
||||
contact=None,
|
||||
)
|
||||
assert mailbox.contact_id is None
|
||||
|
||||
patch_url = self.mailbox_detail_url(mail_domain1.pk, mailbox.pk)
|
||||
patch_response = api_client.patch(
|
||||
patch_url, data={"metadata": {"name": "Helpdesk"}}, format="json"
|
||||
)
|
||||
assert patch_response.status_code == status.HTTP_200_OK
|
||||
|
||||
mailbox.refresh_from_db()
|
||||
assert mailbox.contact is not None
|
||||
assert mailbox.contact.name == "Helpdesk"
|
||||
assert mailbox.contact.email == str(mailbox)
|
||||
|
||||
def test_admin_maildomains_mailbox_partial_update_personal_creates_missing_contact(
|
||||
self,
|
||||
api_client,
|
||||
domain_admin_user,
|
||||
domain_admin_access1,
|
||||
mail_domain1,
|
||||
):
|
||||
"""Renaming a personal mailbox that has no contact yet should create and
|
||||
link one, and still update the owner full name."""
|
||||
api_client.force_authenticate(user=domain_admin_user)
|
||||
|
||||
mailbox = factories.MailboxFactory(
|
||||
domain=mail_domain1,
|
||||
local_part="orphan-personal",
|
||||
is_identity=True,
|
||||
contact=None,
|
||||
users_admin=[
|
||||
factories.UserFactory(email=f"orphan-personal@{mail_domain1.name}")
|
||||
],
|
||||
)
|
||||
assert mailbox.contact_id is None
|
||||
|
||||
patch_url = self.mailbox_detail_url(mail_domain1.pk, mailbox.pk)
|
||||
patch_response = api_client.patch(
|
||||
patch_url, data={"metadata": {"full_name": "Jane D."}}, format="json"
|
||||
)
|
||||
assert patch_response.status_code == status.HTTP_200_OK
|
||||
|
||||
mailbox.refresh_from_db()
|
||||
assert mailbox.contact is not None
|
||||
assert mailbox.contact.name == "Jane D."
|
||||
|
||||
user = models.User.objects.get(email=str(mailbox))
|
||||
assert user.full_name == "Jane D."
|
||||
|
||||
def test_admin_maildomains_mailbox_partial_update_forbidden_not_admin(
|
||||
self,
|
||||
api_client,
|
||||
|
||||
@@ -54,6 +54,27 @@ class TestBlobAPI:
|
||||
)
|
||||
return test_file
|
||||
|
||||
def test_upload_session_auth_requires_csrf_token(self, api_client, user_mailbox):
|
||||
"""A cookie-session upload without a CSRF token is rejected (403).
|
||||
|
||||
The upload action carries no ``@csrf_exempt`` and uses the default DRF
|
||||
auth classes, so ``SessionAuthentication`` enforces CSRF on
|
||||
cookie-authenticated requests. Unlike the other tests (which use
|
||||
``force_authenticate`` and bypass the auth/CSRF path entirely), this
|
||||
logs in via a real session and asserts the request is refused without a
|
||||
token — pinning that the endpoint is not, and was never, CSRF-exempt.
|
||||
"""
|
||||
_, user = api_client
|
||||
csrf_client = APIClient(enforce_csrf_checks=True)
|
||||
csrf_client.force_login(user) # real session → SessionAuthentication path
|
||||
url = reverse("blob-upload", kwargs={"mailbox_id": user_mailbox.id})
|
||||
|
||||
response = csrf_client.post(
|
||||
url, {"file": self._create_test_file()}, format="multipart"
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
def test_upload_download_blob(
|
||||
self,
|
||||
api_client,
|
||||
@@ -131,6 +152,15 @@ class TestBlobAPI:
|
||||
# Should be denied
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
def test_download_malformed_blob_id_returns_400(self, api_client):
|
||||
"""Non-UUID, non-``msg_`` pk must surface as 400, not 500."""
|
||||
client, _ = api_client
|
||||
|
||||
url = reverse("blob-download", kwargs={"pk": "not-a-uuid"})
|
||||
response = client.get(url)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"role",
|
||||
[
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Tests for the blob preview endpoint."""
|
||||
|
||||
import uuid
|
||||
|
||||
from django.urls import reverse
|
||||
|
||||
import pytest
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from core import factories
|
||||
from core.enums import MailboxRoleChoices, PreviewRefusalCode
|
||||
from core.services.blob_gc import upload_and_reserve_blob
|
||||
|
||||
# Minimal but real magic byte sequences for the formats the preview endpoint
|
||||
# allowlists. Using real bytes (rather than mocking ``magic.from_buffer``)
|
||||
# proves the validation path end-to-end and matches the convention already
|
||||
# used by ``tests/importer/test_import_service.py``.
|
||||
PNG_BYTES = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
b"\x00\x00\x00\rIHDR"
|
||||
b"\x00\x00\x00\x01\x00\x00\x00\x01"
|
||||
b"\x08\x06\x00\x00\x00\x1f\x15\xc4\x89"
|
||||
b"\x00\x00\x00\rIDATx\x9cc\x00\x01\x00\x00\x05\x00\x01\r\n-\xb4"
|
||||
b"\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
)
|
||||
PDF_BYTES = (
|
||||
b"%PDF-1.4\n"
|
||||
b"1 0 obj<<>>endobj\n"
|
||||
b"xref\n0 1\n0000000000 65535 f\n"
|
||||
b"trailer<<>>\n"
|
||||
b"startxref\n9\n"
|
||||
b"%%EOF\n"
|
||||
)
|
||||
ZIP_BYTES = b"PK\x03\x04" + b"\x00" * 26 + b"PK\x05\x06" + b"\x00" * 18
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBlobPreview:
|
||||
"""Cover the security contract of GET /api/v1.0/blob/{id}/preview/."""
|
||||
|
||||
@pytest.fixture
|
||||
def authed_client(self):
|
||||
"""Authenticated APIClient and its user."""
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
return client, user
|
||||
|
||||
@pytest.fixture
|
||||
def stranger_client(self):
|
||||
"""Authenticated APIClient for a user with no mailbox access."""
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
return client
|
||||
|
||||
@pytest.fixture
|
||||
def mailbox(self, authed_client):
|
||||
"""Mailbox the authed user can edit (so it can own uploaded blobs)."""
|
||||
_, user = authed_client
|
||||
mb = factories.MailboxFactory()
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mb, user=user, role=MailboxRoleChoices.EDITOR
|
||||
)
|
||||
return mb
|
||||
|
||||
@staticmethod
|
||||
def _preview_url(blob_id):
|
||||
"""Build the /api/v1.0/blob/{id}/preview/ URL."""
|
||||
return reverse("blob-preview", kwargs={"pk": blob_id})
|
||||
|
||||
def test_preview_png_returns_inline_with_security_headers(
|
||||
self, authed_client, mailbox
|
||||
):
|
||||
"""Allowlisted PNG is served inline with the hardening headers set."""
|
||||
client, _ = authed_client
|
||||
blob = upload_and_reserve_blob(mailbox, PNG_BYTES, "image/png")
|
||||
|
||||
response = client.get(self._preview_url(blob.id))
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response["Content-Type"] == "image/png"
|
||||
assert response["Content-Disposition"].startswith("inline;")
|
||||
assert response["X-Content-Type-Options"] == "nosniff"
|
||||
assert response["Referrer-Policy"] == "no-referrer"
|
||||
csp = response["Content-Security-Policy"]
|
||||
assert "default-src 'none'" in csp
|
||||
assert "sandbox" in csp
|
||||
assert response["Cache-Control"] == "private, max-age=2592000"
|
||||
assert response.content == PNG_BYTES
|
||||
|
||||
def test_preview_pdf_returns_200(self, authed_client, mailbox):
|
||||
"""PDF is in the allowlist and served as application/pdf."""
|
||||
client, _ = authed_client
|
||||
blob = upload_and_reserve_blob(mailbox, PDF_BYTES, "application/pdf")
|
||||
|
||||
response = client.get(self._preview_url(blob.id))
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response["Content-Type"] == "application/pdf"
|
||||
|
||||
def test_preview_zip_refused_415_unsupported(self, authed_client, mailbox):
|
||||
"""A non-allowlisted MIME (even if declared correctly) must be refused.
|
||||
|
||||
The declared type isn't previewable, so the refusal is reported as
|
||||
``unsupported`` (not suspicious): the client never expected a preview.
|
||||
"""
|
||||
client, _ = authed_client
|
||||
blob = upload_and_reserve_blob(mailbox, ZIP_BYTES, "application/zip")
|
||||
|
||||
response = client.get(self._preview_url(blob.id))
|
||||
|
||||
assert response.status_code == status.HTTP_415_UNSUPPORTED_MEDIA_TYPE
|
||||
assert response.json()["code"] == PreviewRefusalCode.UNSUPPORTED
|
||||
|
||||
def test_preview_mime_mismatch_refused_415_suspicious(self, authed_client, mailbox):
|
||||
"""ZIP bytes uploaded as image/png must never be served inline.
|
||||
|
||||
The declared type (image/png) was previewable, so a refusal means the
|
||||
bytes betrayed the declared type → reported as ``suspicious`` so the UI
|
||||
can warn the user instead of showing a blank preview.
|
||||
"""
|
||||
client, _ = authed_client
|
||||
# Declared as PNG to pass the allowlist check; magic detection
|
||||
# will see ZIP magic bytes and refuse the mismatch.
|
||||
blob = upload_and_reserve_blob(mailbox, ZIP_BYTES, "image/png")
|
||||
|
||||
response = client.get(self._preview_url(blob.id))
|
||||
|
||||
assert response.status_code == status.HTTP_415_UNSUPPORTED_MEDIA_TYPE
|
||||
assert response.json()["code"] == PreviewRefusalCode.SUSPICIOUS
|
||||
|
||||
def test_preview_accepts_declared_type_with_parameters_and_case(
|
||||
self, authed_client, mailbox
|
||||
):
|
||||
"""Declared Content-Type with charset/case variants is normalized.
|
||||
|
||||
Uploads can store ``image/PNG; charset=binary`` verbatim. The preview
|
||||
check must compare against the canonical ``image/png`` (parameters
|
||||
stripped, lowercased) so valid bytes aren't refused as suspicious.
|
||||
"""
|
||||
client, _ = authed_client
|
||||
blob = upload_and_reserve_blob(mailbox, PNG_BYTES, "image/PNG; charset=binary")
|
||||
|
||||
response = client.get(self._preview_url(blob.id))
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response["Content-Type"] == "image/png"
|
||||
|
||||
def test_preview_without_access_returns_403(
|
||||
self, authed_client, mailbox, stranger_client
|
||||
):
|
||||
"""Users without mailbox access cannot preview its blobs."""
|
||||
_, _ = authed_client
|
||||
blob = upload_and_reserve_blob(mailbox, PNG_BYTES, "image/png")
|
||||
|
||||
response = stranger_client.get(self._preview_url(blob.id))
|
||||
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
def test_preview_unknown_blob_returns_403(self, authed_client):
|
||||
"""Hide blob existence: unknown UUID must look like a denied blob."""
|
||||
client, _ = authed_client
|
||||
|
||||
response = client.get(self._preview_url(uuid.uuid4()))
|
||||
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
def test_preview_malformed_blob_id_returns_400(self, authed_client):
|
||||
"""Non-UUID, non-``msg_`` pk must surface as 400, not 500."""
|
||||
client, _ = authed_client
|
||||
|
||||
response = client.get(self._preview_url("not-a-uuid"))
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
def test_preview_unauthenticated_returns_401(self, mailbox):
|
||||
"""Anonymous requests are rejected before any blob lookup."""
|
||||
blob = upload_and_reserve_blob(mailbox, PNG_BYTES, "image/png")
|
||||
anon = APIClient()
|
||||
|
||||
response = anon.get(self._preview_url(blob.id))
|
||||
|
||||
assert response.status_code in (
|
||||
status.HTTP_401_UNAUTHORIZED,
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -80,6 +80,7 @@ def test_api_config(is_authenticated):
|
||||
"sdk_url": "/sdk",
|
||||
"api_url": "/api/v1.0",
|
||||
"file_url": "/explorer/items/files",
|
||||
"preview_url": "/media/preview/item",
|
||||
"app_name": "Drive App",
|
||||
}
|
||||
)
|
||||
@@ -93,6 +94,7 @@ def test_api_config_with_external_services():
|
||||
"sdk_url": "http://localhost:8902/sdk",
|
||||
"api_url": "http://localhost:8902/api/v1.0",
|
||||
"file_url": "http://localhost:8902/explorer/items/files",
|
||||
"preview_url": "http://localhost:8902/media/preview/item",
|
||||
"app_name": "Drive App",
|
||||
}
|
||||
|
||||
|
||||
@@ -95,6 +95,7 @@ class TestDriveAPIView:
|
||||
"sdk_url": "/sdk",
|
||||
"api_url": "/api/v1.0",
|
||||
"file_url": "/explorer/items/files",
|
||||
"preview_url": "/media/preview/item",
|
||||
}
|
||||
|
||||
def test_api_third_party_drive_get_anonymous(self):
|
||||
|
||||
@@ -16,7 +16,6 @@ from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from core import enums, factories, models
|
||||
from core.mda.rfc5322 import EmailParseError
|
||||
|
||||
|
||||
@pytest.fixture(name="api_client")
|
||||
@@ -118,7 +117,7 @@ class TestMTAInboundEmail:
|
||||
"""Test the MTA inbound email endpoint."""
|
||||
|
||||
@patch("core.api.viewsets.inbound.mta.deliver_inbound_message")
|
||||
@patch("core.api.viewsets.inbound.mta.parse_email_message")
|
||||
@patch("core.api.viewsets.inbound.mta.parse_email")
|
||||
@pytest.mark.django_db
|
||||
def test_valid_email_submission(
|
||||
self,
|
||||
@@ -165,7 +164,7 @@ class TestMTAInboundEmail:
|
||||
assert second_call_args[1]["subject"] == "Test Email"
|
||||
|
||||
@patch("core.api.viewsets.inbound.mta.deliver_inbound_message")
|
||||
@patch("core.api.viewsets.inbound.mta.parse_email_message")
|
||||
@patch("core.api.viewsets.inbound.mta.parse_email")
|
||||
def test_email_parse_failure(
|
||||
self,
|
||||
mock_parse,
|
||||
@@ -175,7 +174,7 @@ class TestMTAInboundEmail:
|
||||
valid_jwt_token,
|
||||
):
|
||||
"""Test that if email parsing fails, a 400 is returned."""
|
||||
mock_parse.side_effect = EmailParseError("Parsing failed")
|
||||
mock_parse.return_value = None
|
||||
|
||||
email = "recipient@example.com"
|
||||
token = valid_jwt_token(sample_email, {"original_recipients": [email]})
|
||||
@@ -193,7 +192,7 @@ class TestMTAInboundEmail:
|
||||
mock_deliver.assert_not_called() # Delivery should not be attempted
|
||||
|
||||
@patch("core.api.viewsets.inbound.mta.deliver_inbound_message")
|
||||
@patch("core.api.viewsets.inbound.mta.parse_email_message")
|
||||
@patch("core.api.viewsets.inbound.mta.parse_email")
|
||||
def test_delivery_partial_failure(
|
||||
self,
|
||||
mock_parse,
|
||||
@@ -235,7 +234,7 @@ class TestMTAInboundEmail:
|
||||
assert mock_deliver.call_count == 2 # Called for both recipients
|
||||
|
||||
@patch("core.api.viewsets.inbound.mta.deliver_inbound_message")
|
||||
@patch("core.api.viewsets.inbound.mta.parse_email_message")
|
||||
@patch("core.api.viewsets.inbound.mta.parse_email")
|
||||
def test_delivery_total_failure(
|
||||
self,
|
||||
mock_parse,
|
||||
@@ -509,6 +508,37 @@ class TestEmailAddressParsing:
|
||||
).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestMTAJWTHardening:
|
||||
"""exp / body-hash guards on the shared-secret-authenticated MTA JWT."""
|
||||
|
||||
@staticmethod
|
||||
def _token(body):
|
||||
"""Mint a token binding the given body, signed with the shared secret."""
|
||||
payload = {
|
||||
"body_hash": hashlib.sha256(body).hexdigest(),
|
||||
"exp": datetime.datetime.now(datetime.UTC) + datetime.timedelta(seconds=30),
|
||||
"original_recipients": ["recipient@example.com"],
|
||||
}
|
||||
return jwt.encode(payload, settings.MDA_API_SECRET, algorithm="HS256")
|
||||
|
||||
def test_body_hash_enforced_on_empty_body(self, api_client):
|
||||
"""body_hash is checked even when the request body is empty.
|
||||
|
||||
A token minted for a non-empty body but presented with an empty body
|
||||
must fail — closing the old ``if request.body:`` bypass that skipped
|
||||
the hash check (and let the bodyless /check path accept any token).
|
||||
"""
|
||||
token = self._token(b"some real body")
|
||||
response = api_client.post(
|
||||
"/api/v1.0/inbound/mta/deliver/",
|
||||
data=b"",
|
||||
content_type="message/rfc822",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
assert response.status_code == status.HTTP_401_UNAUTHORIZED
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestMTAInboundEmailThreading:
|
||||
"""Test the threading logic for MTA inbound emails."""
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.test import override_settings
|
||||
|
||||
import pytest
|
||||
from rest_framework import status
|
||||
@@ -10,9 +12,23 @@ from rest_framework.exceptions import AuthenticationFailed
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from core import factories, models
|
||||
from core.api.viewsets.inbound import widget as widget_module
|
||||
from core.api.viewsets.inbound.widget import WidgetAuthentication
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_throttle_cache():
|
||||
"""Reset throttle state between tests.
|
||||
|
||||
The widget deliver endpoint is rate-limited per IP, and the test client
|
||||
always presents the same address, so DRF's throttle history would
|
||||
otherwise accumulate across tests in the in-process LocMem cache and trip
|
||||
unrelated cases. Clearing the cache keeps each test independent.
|
||||
"""
|
||||
cache.clear()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(name="api_client")
|
||||
def fixture_api_client():
|
||||
"""Return an API client."""
|
||||
@@ -298,7 +314,7 @@ class TestInboundWidgetDeliver:
|
||||
call_args = mock_deliver.call_args[0]
|
||||
parsed_email = call_args[1]
|
||||
|
||||
assert parsed_email["from"]["email"] == "sender@example.com"
|
||||
assert parsed_email["from"][0]["email"] == "sender@example.com"
|
||||
assert (
|
||||
"Test message with custom settings"
|
||||
in parsed_email["htmlBody"][0]["content"]
|
||||
@@ -471,3 +487,106 @@ class TestInboundWidgetDeliver:
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestInboundWidgetAbuse:
|
||||
"""Throttling and body-size cap on the public widget deliver path."""
|
||||
|
||||
@patch(
|
||||
"core.api.viewsets.inbound.widget.deliver_inbound_message", return_value=True
|
||||
)
|
||||
def test_per_ip_throttle_blocks_flood(self, _mock_deliver, api_client, channel):
|
||||
"""Once the per-IP rate is exhausted, further posts get 429.
|
||||
|
||||
The rate is forced low so the test is deterministic and fast.
|
||||
"""
|
||||
data = {"email": "sender@example.com", "textBody": "hi"}
|
||||
|
||||
with patch.object(
|
||||
widget_module.WidgetIPThrottle, "get_rate", return_value="1/minute"
|
||||
):
|
||||
first = api_client.post(
|
||||
"/api/v1.0/inbound/widget/deliver/",
|
||||
data=data,
|
||||
HTTP_X_CHANNEL_ID=str(channel.id),
|
||||
)
|
||||
second = api_client.post(
|
||||
"/api/v1.0/inbound/widget/deliver/",
|
||||
data=data,
|
||||
HTTP_X_CHANNEL_ID=str(channel.id),
|
||||
)
|
||||
|
||||
assert first.status_code == status.HTTP_200_OK
|
||||
assert second.status_code == status.HTTP_429_TOO_MANY_REQUESTS
|
||||
# The throttled request must be rejected before the view runs delivery:
|
||||
# only the first (200) post reached deliver_inbound_message.
|
||||
_mock_deliver.assert_called_once()
|
||||
|
||||
@patch(
|
||||
"core.api.viewsets.inbound.widget.deliver_inbound_message", return_value=True
|
||||
)
|
||||
def test_per_channel_throttle_blocks_flood(
|
||||
self, _mock_deliver, api_client, channel
|
||||
):
|
||||
"""The per-channel cap trips even when the per-IP cap is generous."""
|
||||
data = {"email": "sender@example.com", "textBody": "hi"}
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
widget_module.WidgetChannelThrottle, "get_rate", return_value="1/minute"
|
||||
),
|
||||
patch.object(
|
||||
widget_module.WidgetIPThrottle, "get_rate", return_value="1000/minute"
|
||||
),
|
||||
):
|
||||
first = api_client.post(
|
||||
"/api/v1.0/inbound/widget/deliver/",
|
||||
data=data,
|
||||
HTTP_X_CHANNEL_ID=str(channel.id),
|
||||
)
|
||||
second = api_client.post(
|
||||
"/api/v1.0/inbound/widget/deliver/",
|
||||
data=data,
|
||||
HTTP_X_CHANNEL_ID=str(channel.id),
|
||||
)
|
||||
|
||||
assert first.status_code == status.HTTP_200_OK
|
||||
assert second.status_code == status.HTTP_429_TOO_MANY_REQUESTS
|
||||
|
||||
@patch(
|
||||
"core.api.viewsets.inbound.widget.deliver_inbound_message", return_value=True
|
||||
)
|
||||
@override_settings(MAX_INCOMING_EMAIL_SIZE=1024)
|
||||
def test_oversized_body_rejected(self, mock_deliver, api_client, channel):
|
||||
"""A body over MAX_INCOMING_EMAIL_SIZE is rejected before delivery."""
|
||||
data = {
|
||||
"email": "sender@example.com",
|
||||
"textBody": "x" * 2048, # exceeds the 1 KB limit
|
||||
}
|
||||
|
||||
response = api_client.post(
|
||||
"/api/v1.0/inbound/widget/deliver/",
|
||||
data=data,
|
||||
HTTP_X_CHANNEL_ID=str(channel.id),
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_413_REQUEST_ENTITY_TOO_LARGE
|
||||
mock_deliver.assert_not_called()
|
||||
|
||||
@patch(
|
||||
"core.api.viewsets.inbound.widget.deliver_inbound_message", return_value=True
|
||||
)
|
||||
@override_settings(MAX_INCOMING_EMAIL_SIZE=1024)
|
||||
def test_body_within_limit_accepted(self, mock_deliver, api_client, channel):
|
||||
"""A body within the limit still goes through."""
|
||||
data = {"email": "sender@example.com", "textBody": "x" * 100}
|
||||
|
||||
response = api_client.post(
|
||||
"/api/v1.0/inbound/widget/deliver/",
|
||||
data=data,
|
||||
HTTP_X_CHANNEL_ID=str(channel.id),
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
mock_deliver.assert_called_once()
|
||||
|
||||
@@ -396,6 +396,7 @@ class TestMailboxViewSet:
|
||||
# Check response data
|
||||
assert response.data["id"] == str(mailbox.id)
|
||||
assert response.data["email"] == str(mailbox)
|
||||
assert response.data["domain_id"] == str(mailbox.domain_id)
|
||||
assert response.data["role"] == "editor"
|
||||
assert response.data["count_unread_threads"] == 1
|
||||
assert response.data["count_threads"] == 1
|
||||
@@ -786,3 +787,128 @@ class TestMailboxAbilitiesAPI:
|
||||
response = api_client.get(url)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["role"] == "editor"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestMailboxPartialUpdate:
|
||||
"""Test renaming a mailbox via PATCH /mailboxes/{id}/ (mailbox admins)."""
|
||||
|
||||
def _grant(self, mailbox, user, role):
|
||||
return factories.MailboxAccessFactory(mailbox=mailbox, user=user, role=role)
|
||||
|
||||
def test_admin_can_rename_and_creates_missing_contact(
|
||||
self, api_client, user, mailbox
|
||||
):
|
||||
"""A mailbox admin can rename a mailbox; a missing contact is created."""
|
||||
self._grant(mailbox, user, models.MailboxRoleChoices.ADMIN)
|
||||
assert mailbox.contact_id is None
|
||||
api_client.force_authenticate(user=user)
|
||||
|
||||
url = reverse("mailboxes-detail", args=[mailbox.id])
|
||||
response = api_client.patch(url, data={"name": "Helpdesk"}, format="json")
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["name"] == "Helpdesk"
|
||||
mailbox.refresh_from_db()
|
||||
assert mailbox.contact is not None
|
||||
assert mailbox.contact.name == "Helpdesk"
|
||||
assert mailbox.contact.email == str(mailbox)
|
||||
|
||||
def test_admin_can_rename_existing_contact(self, api_client, user):
|
||||
"""Renaming a mailbox that already has a contact updates its name."""
|
||||
contact = factories.ContactFactory(name="Old")
|
||||
mailbox = factories.MailboxFactory(contact=contact)
|
||||
contact.mailbox = mailbox
|
||||
contact.email = str(mailbox)
|
||||
contact.save()
|
||||
self._grant(mailbox, user, models.MailboxRoleChoices.ADMIN)
|
||||
api_client.force_authenticate(user=user)
|
||||
|
||||
url = reverse("mailboxes-detail", args=[mailbox.id])
|
||||
response = api_client.patch(url, data={"name": "New"}, format="json")
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
mailbox.refresh_from_db()
|
||||
assert mailbox.contact.name == "New"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"role",
|
||||
[models.MailboxRoleChoices.VIEWER, models.MailboxRoleChoices.EDITOR],
|
||||
)
|
||||
def test_non_admin_member_forbidden(self, api_client, user, mailbox, role):
|
||||
"""A non-admin member of the mailbox cannot rename it."""
|
||||
self._grant(mailbox, user, role)
|
||||
api_client.force_authenticate(user=user)
|
||||
|
||||
url = reverse("mailboxes-detail", args=[mailbox.id])
|
||||
response = api_client.patch(url, data={"name": "Nope"}, format="json")
|
||||
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
def test_non_member_not_found(self, api_client, user, mailbox):
|
||||
"""A user with no access to the mailbox gets a 404."""
|
||||
api_client.force_authenticate(user=user)
|
||||
url = reverse("mailboxes-detail", args=[mailbox.id])
|
||||
response = api_client.patch(url, data={"name": "Nope"}, format="json")
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
def test_unauthenticated(self, api_client, mailbox):
|
||||
"""An unauthenticated request is rejected."""
|
||||
url = reverse("mailboxes-detail", args=[mailbox.id])
|
||||
response = api_client.patch(url, data={"name": "Nope"}, format="json")
|
||||
assert response.status_code == status.HTTP_401_UNAUTHORIZED
|
||||
|
||||
def test_blank_name_rejected(self, api_client, user, mailbox):
|
||||
"""An empty name is rejected with a 400."""
|
||||
self._grant(mailbox, user, models.MailboxRoleChoices.ADMIN)
|
||||
api_client.force_authenticate(user=user)
|
||||
url = reverse("mailboxes-detail", args=[mailbox.id])
|
||||
response = api_client.patch(url, data={"name": ""}, format="json")
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
def test_whitespace_only_name_rejected(self, api_client, user, mailbox):
|
||||
"""A whitespace-only name is rejected with a 400 (no blank rename)."""
|
||||
self._grant(mailbox, user, models.MailboxRoleChoices.ADMIN)
|
||||
api_client.force_authenticate(user=user)
|
||||
url = reverse("mailboxes-detail", args=[mailbox.id])
|
||||
response = api_client.patch(url, data={"name": " "}, format="json")
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
def test_only_name_is_updated(self, api_client, user):
|
||||
"""Extra fields in the payload are ignored: only ``name`` is applied."""
|
||||
mailbox = factories.MailboxFactory(local_part="support", is_identity=False)
|
||||
self._grant(mailbox, user, models.MailboxRoleChoices.ADMIN)
|
||||
api_client.force_authenticate(user=user)
|
||||
|
||||
url = reverse("mailboxes-detail", args=[mailbox.id])
|
||||
response = api_client.patch(
|
||||
url,
|
||||
data={"name": "Renamed", "local_part": "hijacked", "is_identity": True},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["name"] == "Renamed"
|
||||
mailbox.refresh_from_db()
|
||||
assert mailbox.contact.name == "Renamed"
|
||||
# Fields outside the dedicated serializer stay untouched.
|
||||
assert mailbox.local_part == "support"
|
||||
assert mailbox.is_identity is False
|
||||
|
||||
def test_missing_name_is_noop(self, api_client, user):
|
||||
"""An empty body leaves the name untouched (partial PATCH no-op)."""
|
||||
contact = factories.ContactFactory(name="Old")
|
||||
mailbox = factories.MailboxFactory(contact=contact)
|
||||
contact.mailbox = mailbox
|
||||
contact.email = str(mailbox)
|
||||
contact.save()
|
||||
self._grant(mailbox, user, models.MailboxRoleChoices.ADMIN)
|
||||
api_client.force_authenticate(user=user)
|
||||
|
||||
url = reverse("mailboxes-detail", args=[mailbox.id])
|
||||
response = api_client.patch(url, data={}, format="json")
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["name"] == "Old"
|
||||
mailbox.refresh_from_db()
|
||||
assert mailbox.contact.name == "Old"
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Tests for ``MessageSerializer.get_attachments`` name handling."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from core.api.serializers import MessageSerializer
|
||||
|
||||
|
||||
def _serialize_parsed_attachments(parsed_attachments):
|
||||
"""Run ``get_attachments`` against a non-draft message whose parsed MIME
|
||||
exposes ``parsed_attachments``, bypassing DB and ``parse_email``."""
|
||||
instance = Mock()
|
||||
instance.id = "00000000-0000-0000-0000-000000000000"
|
||||
instance.has_attachments = True
|
||||
instance.is_draft = False
|
||||
instance.get_parsed_field.return_value = parsed_attachments
|
||||
return MessageSerializer().get_attachments(instance)
|
||||
|
||||
|
||||
def test_get_attachments_preserves_present_name():
|
||||
"""A MIME part that carries a ``filename`` keeps it verbatim."""
|
||||
result = _serialize_parsed_attachments(
|
||||
[{"name": "report.pdf", "size": 12, "type": "application/pdf"}]
|
||||
)
|
||||
assert [a["name"] for a in result] == ["report.pdf"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attachment",
|
||||
[
|
||||
pytest.param({"size": 1, "type": "text/plain"}, id="missing-name"),
|
||||
pytest.param({"name": None, "size": 1, "type": "text/plain"}, id="none-name"),
|
||||
pytest.param({"name": "", "size": 1, "type": "text/plain"}, id="empty-name"),
|
||||
],
|
||||
)
|
||||
def test_get_attachments_falls_back_to_unnamed(attachment):
|
||||
"""A MIME part with no usable ``filename`` falls back to the "unnamed"
|
||||
sentinel so consumers never receive a null/empty name (regression: a null
|
||||
name crashed the frontend calendar-invite download button)."""
|
||||
result = _serialize_parsed_attachments([attachment])
|
||||
assert result[0]["name"] == "unnamed"
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Test the render action of MailboxMessageTemplateViewSet."""
|
||||
|
||||
import uuid
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.urls import reverse
|
||||
|
||||
import pytest
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from core import enums, factories, models
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
# A template exercising mailbox/user placeholders (name, job_title) and the
|
||||
# message-level one (recipient_name) that only resolves when a draft is given.
|
||||
TEMPLATE_HTML = "<p>{name} - {job_title} - {recipient_name}</p>"
|
||||
TEMPLATE_TEXT = "{name} - {job_title} - {recipient_name}"
|
||||
|
||||
|
||||
@pytest.fixture(name="user")
|
||||
def fixture_user():
|
||||
"""Create a test user."""
|
||||
return factories.UserFactory(
|
||||
full_name="John Doe", custom_attributes={"job_title": "Adjointe"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="mailbox")
|
||||
def fixture_mailbox():
|
||||
"""Create a test mailbox."""
|
||||
return factories.MailboxFactory()
|
||||
|
||||
|
||||
def render_url(mailbox_id, template_id):
|
||||
"""Build the URL for the message template render endpoint."""
|
||||
return reverse(
|
||||
"mailbox-message-templates-render",
|
||||
kwargs={"mailbox_id": mailbox_id, "pk": template_id},
|
||||
)
|
||||
|
||||
|
||||
def _create_draft_with_recipient(mailbox, recipient_name):
|
||||
"""Create a draft owned by the mailbox with a single TO recipient."""
|
||||
sender_contact = factories.ContactFactory(
|
||||
name="Sender", email="sender@example.com", mailbox=mailbox
|
||||
)
|
||||
draft = factories.MessageFactory(
|
||||
sender=sender_contact, thread=factories.ThreadFactory(), is_draft=True
|
||||
)
|
||||
recipient = factories.ContactFactory(
|
||||
name=recipient_name, email="recipient@example.com", mailbox=mailbox
|
||||
)
|
||||
factories.MessageRecipientFactory(
|
||||
message=draft, contact=recipient, type=enums.MessageRecipientTypeChoices.TO
|
||||
)
|
||||
return draft
|
||||
|
||||
|
||||
class TestMessageTemplateRender:
|
||||
"""Test the render action under mailboxes/{id}/message-templates/{id}/render/."""
|
||||
|
||||
def test_message_template_render_unauthorized(self, mailbox):
|
||||
"""Unauthenticated users cannot render a template."""
|
||||
template = factories.MessageTemplateFactory(mailbox=mailbox)
|
||||
client = APIClient()
|
||||
response = client.get(render_url(mailbox.id, template.id))
|
||||
assert response.status_code == status.HTTP_401_UNAUTHORIZED
|
||||
|
||||
def test_message_template_render_no_access(self, user, mailbox):
|
||||
"""Users without mailbox access cannot render a template."""
|
||||
template = factories.MessageTemplateFactory(mailbox=mailbox)
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.get(render_url(mailbox.id, template.id))
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
@patch(
|
||||
"django.conf.settings.SCHEMA_CUSTOM_ATTRIBUTES_USER",
|
||||
{"properties": {"job_title": {"type": "string"}}},
|
||||
)
|
||||
def test_message_template_render_without_message_keeps_recipient_token(
|
||||
self, user, mailbox
|
||||
):
|
||||
"""Without a draft, mailbox/user placeholders resolve but recipient_name
|
||||
stays an unresolved token — a viewer role is enough."""
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox, user=user, role=models.MailboxRoleChoices.VIEWER
|
||||
)
|
||||
template = factories.MessageTemplateFactory(
|
||||
mailbox=mailbox, html_body=TEMPLATE_HTML, text_body=TEMPLATE_TEXT
|
||||
)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.get(render_url(mailbox.id, template.id))
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert "John Doe" in response.data["html_body"]
|
||||
assert "Adjointe" in response.data["html_body"]
|
||||
# recipient_name has no value without a draft: its token is left intact.
|
||||
assert "{recipient_name}" in response.data["html_body"]
|
||||
|
||||
@patch(
|
||||
"django.conf.settings.SCHEMA_CUSTOM_ATTRIBUTES_USER",
|
||||
{"properties": {"job_title": {"type": "string"}}},
|
||||
)
|
||||
def test_message_template_render_with_message_resolves_recipient(
|
||||
self, user, mailbox
|
||||
):
|
||||
"""With a draft from this mailbox, recipient_name is resolved."""
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox, user=user, role=models.MailboxRoleChoices.EDITOR
|
||||
)
|
||||
template = factories.MessageTemplateFactory(
|
||||
mailbox=mailbox, html_body=TEMPLATE_HTML, text_body=TEMPLATE_TEXT
|
||||
)
|
||||
draft = _create_draft_with_recipient(mailbox, "Jane Smith")
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.get(
|
||||
render_url(mailbox.id, template.id), {"message_id": str(draft.id)}
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert "Jane Smith" in response.data["html_body"]
|
||||
assert "{recipient_name}" not in response.data["html_body"]
|
||||
|
||||
@patch(
|
||||
"django.conf.settings.SCHEMA_CUSTOM_ATTRIBUTES_USER",
|
||||
{"properties": {"job_title": {"type": "string"}}},
|
||||
)
|
||||
def test_message_template_render_ignores_foreign_draft(self, user, mailbox):
|
||||
"""A draft owned by another mailbox is ignored: recipient_name is not
|
||||
resolved, preventing recipient probing across mailboxes."""
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox, user=user, role=models.MailboxRoleChoices.EDITOR
|
||||
)
|
||||
template = factories.MessageTemplateFactory(
|
||||
mailbox=mailbox, html_body=TEMPLATE_HTML, text_body=TEMPLATE_TEXT
|
||||
)
|
||||
foreign_draft = _create_draft_with_recipient(
|
||||
factories.MailboxFactory(), "Secret Recipient"
|
||||
)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.get(
|
||||
render_url(mailbox.id, template.id), {"message_id": str(foreign_draft.id)}
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert "Secret Recipient" not in response.data["html_body"]
|
||||
assert "{recipient_name}" in response.data["html_body"]
|
||||
|
||||
def test_message_template_render_maildomain_template(self, user, mailbox):
|
||||
"""A domain-level template renders through the mailbox endpoint."""
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox, user=user, role=models.MailboxRoleChoices.VIEWER
|
||||
)
|
||||
template = factories.MessageTemplateFactory(
|
||||
maildomain=mailbox.domain,
|
||||
html_body="<p>Domain signature</p>",
|
||||
text_body="Domain signature",
|
||||
)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.get(render_url(mailbox.id, template.id))
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["html_body"] == "<p>Domain signature</p>"
|
||||
|
||||
def test_message_template_render_nonexistent(self, user, mailbox):
|
||||
"""Rendering a nonexistent template returns 404."""
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox, user=user, role=models.MailboxRoleChoices.VIEWER
|
||||
)
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.get(render_url(mailbox.id, uuid.uuid4()))
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
def test_message_template_render_foreign_mailbox_template(self, user, mailbox):
|
||||
"""A template owned by another mailbox (not this mailbox nor its domain)
|
||||
is not reachable through this mailbox: it returns 404."""
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox, user=user, role=models.MailboxRoleChoices.VIEWER
|
||||
)
|
||||
foreign_template = factories.MessageTemplateFactory(
|
||||
mailbox=factories.MailboxFactory(),
|
||||
html_body="<p>Foreign</p>",
|
||||
text_body="Foreign",
|
||||
)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.get(render_url(mailbox.id, foreign_template.id))
|
||||
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
def test_message_template_render_escapes_html_in_html_body_only(
|
||||
self, user, mailbox
|
||||
):
|
||||
"""Resolved values are HTML-escaped in html_body to prevent injection,
|
||||
but kept raw in text_body."""
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox, user=user, role=models.MailboxRoleChoices.EDITOR
|
||||
)
|
||||
template = factories.MessageTemplateFactory(
|
||||
mailbox=mailbox, html_body=TEMPLATE_HTML, text_body=TEMPLATE_TEXT
|
||||
)
|
||||
draft = _create_draft_with_recipient(mailbox, "<script>alert(1)</script>")
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.get(
|
||||
render_url(mailbox.id, template.id), {"message_id": str(draft.id)}
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
# The raw markup must never appear in the html body.
|
||||
assert "<script>" not in response.data["html_body"]
|
||||
assert "<script>" in response.data["html_body"]
|
||||
# The text body keeps the value verbatim (no HTML context to escape).
|
||||
assert "<script>alert(1)</script>" in response.data["text_body"]
|
||||
|
||||
def test_message_template_render_invalid_message_id(self, user, mailbox):
|
||||
"""A malformed (non-UUID) message_id is rejected with a 400 rather than
|
||||
crashing the endpoint."""
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox, user=user, role=models.MailboxRoleChoices.EDITOR
|
||||
)
|
||||
template = factories.MessageTemplateFactory(
|
||||
mailbox=mailbox, html_body=TEMPLATE_HTML, text_body=TEMPLATE_TEXT
|
||||
)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.get(
|
||||
render_url(mailbox.id, template.id), {"message_id": "not-a-uuid"}
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
@@ -65,7 +65,12 @@ class TestApiDraftAndSendMessage:
|
||||
|
||||
@patch("core.mda.outbound.send_outbound_message")
|
||||
def test_draft_and_send_message_success(
|
||||
self, mock_send_outbound_message, mailbox, authenticated_user, send_url
|
||||
self,
|
||||
mock_send_outbound_message,
|
||||
mailbox,
|
||||
authenticated_user,
|
||||
send_url,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Test create draft message and then successfully send it via the service."""
|
||||
|
||||
@@ -151,15 +156,16 @@ class TestApiDraftAndSendMessage:
|
||||
assert draft_api_message["is_draft"] is True
|
||||
assert draft_api_message["bcc"][0]["contact"]["email"] == "jean@external.com"
|
||||
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message_id,
|
||||
"senderId": mailbox.id,
|
||||
"textBody": "test",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message_id,
|
||||
"senderId": mailbox.id,
|
||||
"textBody": "test",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert send_response.status_code == status.HTTP_200_OK
|
||||
|
||||
@@ -234,7 +240,12 @@ class TestApiDraftAndSendMessage:
|
||||
|
||||
@patch("core.mda.outbound.send_outbound_message")
|
||||
def test_draft_and_send_message_success_delegated_access(
|
||||
self, mock_send_outbound_message, mailbox, authenticated_user, send_url
|
||||
self,
|
||||
mock_send_outbound_message,
|
||||
mailbox,
|
||||
authenticated_user,
|
||||
send_url,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Test create draft message and then successfully send it via the service."""
|
||||
mock_send_outbound_message.side_effect = (
|
||||
@@ -332,14 +343,15 @@ class TestApiDraftAndSendMessage:
|
||||
assert draft_api_message["draftBody"] == draft_content
|
||||
assert draft_api_message["is_draft"] is True
|
||||
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message_id,
|
||||
"senderId": mailbox.id,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message_id,
|
||||
"senderId": mailbox.id,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert send_response.status_code == status.HTTP_200_OK
|
||||
|
||||
@@ -383,6 +395,7 @@ class TestApiDraftAndSendMessage:
|
||||
mailbox,
|
||||
authenticated_user,
|
||||
send_url,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Test sending a draft message when the delivery service fails."""
|
||||
|
||||
@@ -430,15 +443,16 @@ class TestApiDraftAndSendMessage:
|
||||
assert draft_response.status_code == status.HTTP_201_CREATED
|
||||
draft_message_id = draft_response.data["id"]
|
||||
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message_id,
|
||||
"senderId": str(mailbox.id),
|
||||
"textBody": "test",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message_id,
|
||||
"senderId": str(mailbox.id),
|
||||
"textBody": "test",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert send_response.status_code == status.HTTP_200_OK
|
||||
|
||||
@@ -787,7 +801,12 @@ class TestApiDraftAndSendMessage:
|
||||
|
||||
@patch("core.mda.outbound.send_outbound_message")
|
||||
def test_send_message_with_empty_subject(
|
||||
self, mock_send_outbound_message, mailbox, authenticated_user, send_url
|
||||
self,
|
||||
mock_send_outbound_message,
|
||||
mailbox,
|
||||
authenticated_user,
|
||||
send_url,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Test sending a message with empty subject (migration 0018)."""
|
||||
mock_send_outbound_message.side_effect = (
|
||||
@@ -825,14 +844,15 @@ class TestApiDraftAndSendMessage:
|
||||
draft_message_id = draft_response.data["id"]
|
||||
|
||||
# Send the message
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message_id,
|
||||
"senderId": mailbox.id,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message_id,
|
||||
"senderId": mailbox.id,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert send_response.status_code == status.HTTP_200_OK
|
||||
|
||||
@@ -1099,7 +1119,13 @@ class TestApiDraftAndSendMessage:
|
||||
class TestApiDraftAndSendReply:
|
||||
"""Test API draft and send reply endpoints."""
|
||||
|
||||
def test_draft_and_send_reply_success(self, mailbox, authenticated_user, send_url):
|
||||
def test_draft_and_send_reply_success(
|
||||
self,
|
||||
mailbox,
|
||||
authenticated_user,
|
||||
send_url,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Create draft reply to an existing message and then send it."""
|
||||
# Create a mailbox access on this mailbox for the authenticated user
|
||||
factories.MailboxAccessFactory(
|
||||
@@ -1150,15 +1176,16 @@ class TestApiDraftAndSendReply:
|
||||
assert draft_api_message["parent_id"] == str(message.id)
|
||||
|
||||
# Step 2: Send the draft reply
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message.id,
|
||||
"senderId": mailbox.id,
|
||||
"textBody": "test",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message.id,
|
||||
"senderId": mailbox.id,
|
||||
"textBody": "test",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
# Assert the send response is successful
|
||||
assert send_response.status_code == status.HTTP_200_OK
|
||||
@@ -1307,6 +1334,7 @@ class TestApiDraftAndSendReply:
|
||||
mailbox_role,
|
||||
draft_detail_url,
|
||||
send_url,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Test updating a draft message successfully."""
|
||||
# Create a mailbox access on this mailbox for the authenticated user
|
||||
@@ -1395,14 +1423,15 @@ class TestApiDraftAndSendReply:
|
||||
# assert thread.snippet == "updated content"[:100]
|
||||
|
||||
# Step 3: Send the updated draft message
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": updated_message.id,
|
||||
"senderId": mailbox.id,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": updated_message.id,
|
||||
"senderId": mailbox.id,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
sent_message = models.Message.objects.get(id=updated_message.id)
|
||||
assert sent_message.subject == updated_subject
|
||||
@@ -1510,7 +1539,58 @@ class TestApiDraftAndSendReply:
|
||||
# Assert the response is unauthorized
|
||||
assert response.status_code == status.HTTP_401_UNAUTHORIZED
|
||||
|
||||
def test_api_email_exchange_single_thread(self, send_url):
|
||||
def test_update_draft_ignores_body_message_id_for_authorization(
|
||||
self, mailbox, authenticated_user, draft_detail_url
|
||||
):
|
||||
"""The draft being updated is the one in the URL, not a body messageId.
|
||||
|
||||
A caller with edit rights on a draft they own must not edit a
|
||||
*different* draft (named in the URL) that their sender mailbox cannot
|
||||
access by passing the accessible draft's id in the body. The body id is
|
||||
never read for authorization, so the inaccessible URL draft is denied
|
||||
(the view scopes the lookup to an editable thread → 404).
|
||||
"""
|
||||
# Sender mailbox the user can edit.
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox,
|
||||
user=authenticated_user,
|
||||
role=enums.MailboxRoleChoices.EDITOR,
|
||||
)
|
||||
# A draft the sender mailbox CAN edit (the decoy passed in the body).
|
||||
own_access = factories.ThreadAccessFactory(
|
||||
mailbox=mailbox, role=enums.ThreadAccessRoleChoices.EDITOR
|
||||
)
|
||||
own_draft = factories.MessageFactory(thread=own_access.thread, is_draft=True)
|
||||
# A draft the sender mailbox CANNOT access (the real target, in the URL).
|
||||
other_mailbox = factories.MailboxFactory()
|
||||
victim_access = factories.ThreadAccessFactory(
|
||||
mailbox=other_mailbox, role=enums.ThreadAccessRoleChoices.EDITOR
|
||||
)
|
||||
victim_draft = factories.MessageFactory(
|
||||
thread=victim_access.thread, is_draft=True, subject="victim"
|
||||
)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=authenticated_user)
|
||||
response = client.put(
|
||||
draft_detail_url(victim_draft.id),
|
||||
{
|
||||
"senderId": mailbox.id,
|
||||
"messageId": own_draft.id, # decoy — must be ignored
|
||||
"subject": "hijacked",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
victim_draft.refresh_from_db()
|
||||
assert victim_draft.subject == "victim"
|
||||
|
||||
def test_api_email_exchange_single_thread(
|
||||
self,
|
||||
send_url,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Test a multi-step API email exchange results in one thread per mailbox."""
|
||||
# Setup Users and Mailboxes
|
||||
user1 = factories.UserFactory(email="user1@exchange.api")
|
||||
@@ -1543,15 +1623,16 @@ class TestApiDraftAndSendReply:
|
||||
assert draft1_response.status_code == status.HTTP_201_CREATED
|
||||
message1_id = draft1_response.data["id"]
|
||||
|
||||
send1_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": message1_id,
|
||||
"senderId": str(mailbox1.id),
|
||||
"textBody": "Hello User Two!",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
send1_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": message1_id,
|
||||
"senderId": str(mailbox1.id),
|
||||
"textBody": "Hello User Two!",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
assert send1_response.status_code == status.HTTP_200_OK
|
||||
|
||||
# Message should be marked as sent immediately for local delivery
|
||||
@@ -1595,15 +1676,16 @@ class TestApiDraftAndSendReply:
|
||||
assert draft2_response.status_code == status.HTTP_201_CREATED
|
||||
message2_id = draft2_response.data["id"]
|
||||
|
||||
send2_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": message2_id,
|
||||
"senderId": str(mailbox2.id),
|
||||
"textBody": "Hi User One, thanks!",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
send2_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": message2_id,
|
||||
"senderId": str(mailbox2.id),
|
||||
"textBody": "Hi User One, thanks!",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
assert send2_response.status_code == status.HTTP_200_OK
|
||||
|
||||
# Mark message as sent (local delivery)
|
||||
@@ -1651,11 +1733,12 @@ class TestApiDraftAndSendReply:
|
||||
assert draft3_response.status_code == status.HTTP_201_CREATED
|
||||
message3_id = draft3_response.data["id"]
|
||||
|
||||
send3_response = client.post(
|
||||
send_url,
|
||||
{"messageId": message3_id, "senderId": str(mailbox1.id)},
|
||||
format="json",
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
send3_response = client.post(
|
||||
send_url,
|
||||
{"messageId": message3_id, "senderId": str(mailbox1.id)},
|
||||
format="json",
|
||||
)
|
||||
assert send3_response.status_code == status.HTTP_200_OK
|
||||
|
||||
assert models.Thread.objects.count() == 2 # Still only 2 threads
|
||||
@@ -1687,7 +1770,12 @@ class TestApiDraftAndSendReply:
|
||||
assert models.Thread.objects.filter(accesses__mailbox=mailbox2).count() == 1
|
||||
|
||||
def test_send_message_with_user_having_role_on_two_mailboxes_on_same_thread(
|
||||
self, mailbox, mailbox2, authenticated_user, send_url
|
||||
self,
|
||||
mailbox,
|
||||
mailbox2,
|
||||
authenticated_user,
|
||||
send_url,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""
|
||||
Test that sending a message succeeds when a user has access to two mailboxes
|
||||
@@ -1753,15 +1841,16 @@ class TestApiDraftAndSendReply:
|
||||
assert draft_api_message["parent_id"] == str(message.id)
|
||||
|
||||
# Step 2: Send the draft reply
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message.id,
|
||||
"senderId": mailbox.id,
|
||||
"textBody": "test",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
send_response = client.post(
|
||||
send_url,
|
||||
{
|
||||
"messageId": draft_message.id,
|
||||
"senderId": mailbox.id,
|
||||
"textBody": "test",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
# Assert the send response is successful
|
||||
assert send_response.status_code == status.HTTP_200_OK
|
||||
|
||||
@@ -22,7 +22,7 @@ pytestmark = pytest.mark.django_db
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _mock_ssrf_dns():
|
||||
"""Short-circuit SSRF DNS validation for IMAP import tests.
|
||||
"""Short-circuit SSRF hostname validation for IMAP import tests.
|
||||
|
||||
The IMAP endpoint validates the server hostname via
|
||||
``core.services.ssrf.validate_hostname``; tests use unresolvable fixtures
|
||||
@@ -332,7 +332,7 @@ def test_api_import_imap(api_client, user, mailbox):
|
||||
"""Test import of IMAP messages."""
|
||||
mailbox.accesses.create(user=user, role=MailboxRoleChoices.ADMIN)
|
||||
# Mock IMAP connection and responses
|
||||
with patch("imaplib.IMAP4_SSL") as mock_imap:
|
||||
with patch("core.services.importer.imap._IPPinnedIMAP4SSL") as mock_imap:
|
||||
mock_imap_instance = mock_imap.return_value
|
||||
|
||||
# Mock login
|
||||
@@ -706,7 +706,7 @@ def test_api_import_duplicate_imap_messages(api_client, user, mailbox):
|
||||
assert Thread.objects.count() == 0
|
||||
|
||||
# Mock IMAP connection and responses
|
||||
with patch("imaplib.IMAP4_SSL") as mock_imap:
|
||||
with patch("core.services.importer.imap._IPPinnedIMAP4SSL") as mock_imap:
|
||||
mock_imap_instance = mock_imap.return_value
|
||||
|
||||
# Mock login
|
||||
@@ -781,7 +781,7 @@ def test_api_import_duplicate_imap_messages_different_mailboxes(
|
||||
mailbox2 = factories.MailboxFactory()
|
||||
mailbox2.accesses.create(user=user, role=MailboxRoleChoices.ADMIN)
|
||||
# Mock IMAP connection and responses
|
||||
with patch("imaplib.IMAP4_SSL") as mock_imap:
|
||||
with patch("core.services.importer.imap._IPPinnedIMAP4SSL") as mock_imap:
|
||||
mock_imap_instance = mock_imap.return_value
|
||||
|
||||
# Mock login
|
||||
@@ -869,7 +869,7 @@ Date: Mon, 26 May 2025 10:00:00 +0000
|
||||
This is a draft message."""
|
||||
|
||||
# Mock IMAP connection and responses
|
||||
with patch("imaplib.IMAP4_SSL") as mock_imap:
|
||||
with patch("core.services.importer.imap._IPPinnedIMAP4SSL") as mock_imap:
|
||||
mock_imap_instance = mock_imap.return_value
|
||||
|
||||
# Mock login
|
||||
@@ -935,7 +935,7 @@ Date: Mon, 26 May 2025 10:00:00 +0000
|
||||
This is a regular message."""
|
||||
|
||||
# Mock IMAP connection and responses
|
||||
with patch("imaplib.IMAP4_SSL") as mock_imap:
|
||||
with patch("core.services.importer.imap._IPPinnedIMAP4SSL") as mock_imap:
|
||||
mock_imap_instance = mock_imap.return_value
|
||||
|
||||
# Mock login
|
||||
|
||||
@@ -9,6 +9,45 @@ from rest_framework.test import APIClient
|
||||
|
||||
from core.factories import UserFactory
|
||||
|
||||
# Custom attributes schema providing x-i18n labels, shared by the i18n tests.
|
||||
SCHEMA_WITH_I18N = {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/suitenumerique/messages/schemas/custom-fields/user",
|
||||
"type": "object",
|
||||
"title": "User custom fields",
|
||||
"additionalProperties": False,
|
||||
"properties": {
|
||||
"job_title": {
|
||||
"type": "string",
|
||||
"title": "Job title",
|
||||
"default": "",
|
||||
"description": "The job name of the user",
|
||||
"minLength": 3,
|
||||
"x-i18n": {
|
||||
"title": {"fr": "Fonction", "en": "Job title"},
|
||||
"description": {
|
||||
"fr": "Le nom de la fonction de l'utilisateur",
|
||||
"en": "The job name of the user",
|
||||
},
|
||||
},
|
||||
},
|
||||
"is_elected": {
|
||||
"type": "boolean",
|
||||
"title": "Is elected",
|
||||
"default": False,
|
||||
"description": "Whether the user is elected",
|
||||
"x-i18n": {
|
||||
"title": {"fr": "Est élu", "en": "Is elected"},
|
||||
"description": {
|
||||
"fr": "Indique si l'utilisateur est élu",
|
||||
"en": "Indicates if the user is elected",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(name="user")
|
||||
def fixture_user():
|
||||
@@ -64,62 +103,36 @@ class TestPlaceholderView:
|
||||
}
|
||||
)
|
||||
def test_get_fields_structure(self, api_client):
|
||||
"""Test that the endpoint returns field structure with slugs and labels."""
|
||||
"""Built-in fields are empty; custom fields expose their schema title."""
|
||||
url = reverse("placeholders")
|
||||
response = api_client.get(url)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()
|
||||
assert data["name"] == "Name"
|
||||
assert data["job_title"] == "Job title"
|
||||
assert data["is_elected"] == "Is elected"
|
||||
# Built-in fields carry no label (localized client-side).
|
||||
assert data["name"] == {}
|
||||
assert data["recipient_name"] == {}
|
||||
assert data["user_name"] == {}
|
||||
# Custom fields without x-i18n expose their schema title only.
|
||||
assert data["job_title"] == {"title": "Job title"}
|
||||
# Non-string custom fields (e.g. the boolean "is_elected") are excluded.
|
||||
assert "is_elected" not in data
|
||||
|
||||
@override_settings(
|
||||
SCHEMA_CUSTOM_ATTRIBUTES_USER={
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/suitenumerique/messages/schemas/custom-fields/user",
|
||||
"type": "object",
|
||||
"title": "User custom fields",
|
||||
"additionalProperties": False,
|
||||
"properties": {
|
||||
"job_title": {
|
||||
"type": "string",
|
||||
"title": "Job title",
|
||||
"default": "",
|
||||
"description": "The job name of the user",
|
||||
"minLength": 3,
|
||||
"x-i18n": {
|
||||
"title": {"fr": "Fonction", "en": "Job title"},
|
||||
"description": {
|
||||
"fr": "Le nom de la fonction de l'utilisateur",
|
||||
"en": "The job name of the user",
|
||||
},
|
||||
},
|
||||
},
|
||||
"is_elected": {
|
||||
"type": "boolean",
|
||||
"title": "Is elected",
|
||||
"default": False,
|
||||
"description": "Whether the user is elected",
|
||||
"x-i18n": {
|
||||
"title": {"fr": "Est élu", "en": "Is elected"},
|
||||
"description": {
|
||||
"fr": "Indique si l'utilisateur est élu",
|
||||
"en": "Indicates if the user is elected",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": [],
|
||||
}
|
||||
)
|
||||
def test_i18n_schema_uses_default_language(self, api_client):
|
||||
"""Test that x-i18n schema labels always use the default language."""
|
||||
@override_settings(SCHEMA_CUSTOM_ATTRIBUTES_USER=SCHEMA_WITH_I18N)
|
||||
def test_returns_x_i18n_translations_for_custom_fields(self, api_client):
|
||||
"""Custom fields expose their x-i18n title translations for the frontend."""
|
||||
url = reverse("placeholders")
|
||||
# Accept-Language header is ignored; backend always uses LANGUAGE_CODE
|
||||
response = api_client.get(url, HTTP_ACCEPT_LANGUAGE="fr-fr")
|
||||
response = api_client.get(url)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()
|
||||
assert data["job_title"] == "Job title"
|
||||
assert data["is_elected"] == "Is elected"
|
||||
assert data["name"] == "Name"
|
||||
# Built-in fields remain unlabeled regardless of the schema.
|
||||
assert data["name"] == {}
|
||||
assert data["recipient_name"] == {}
|
||||
assert data["user_name"] == {}
|
||||
# String custom fields ship every available translation; frontend picks one.
|
||||
assert data["job_title"] == {
|
||||
"title": "Job title",
|
||||
"i18n": {"fr": "Fonction", "en": "Job title"},
|
||||
}
|
||||
# Non-string custom fields (e.g. the boolean "is_elected") are excluded.
|
||||
assert "is_elected" not in data
|
||||
|
||||
@@ -126,6 +126,8 @@ class TestResolvePlaceholder:
|
||||
response = client.get(resolve_url(draft.id))
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["name"] == "Mairie de Brigny"
|
||||
# user_name resolves to the authenticated user, distinct from the mailbox.
|
||||
assert response.data["user_name"] == "John Doe"
|
||||
assert response.data["job_title"] == "Adjointe"
|
||||
|
||||
@patch(
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from django.test import override_settings
|
||||
@@ -132,7 +133,9 @@ class TestSendMessageAPIView:
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["task_id"] == "task-123"
|
||||
assert uuid.UUID(
|
||||
response.data["task_id"]
|
||||
) # view-generated id, dispatched on commit
|
||||
|
||||
message = models.Message.objects.get(id=draft_message.id)
|
||||
content = message.blob.get_content().decode()
|
||||
@@ -175,7 +178,9 @@ class TestSendMessageAPIView:
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["task_id"] == "task-123"
|
||||
assert uuid.UUID(
|
||||
response.data["task_id"]
|
||||
) # view-generated id, dispatched on commit
|
||||
|
||||
@override_settings(SCHEMA_CUSTOM_ATTRIBUTES_USER=SCHEMA_CUSTOM_ATTRIBUTES)
|
||||
def test_api_send_message_with_text_body_only(
|
||||
@@ -212,7 +217,9 @@ class TestSendMessageAPIView:
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["task_id"] == "task-123"
|
||||
assert uuid.UUID(
|
||||
response.data["task_id"]
|
||||
) # view-generated id, dispatched on commit
|
||||
|
||||
message = models.Message.objects.get(id=draft_message.id)
|
||||
content = message.blob.get_content().decode()
|
||||
@@ -257,7 +264,9 @@ class TestSendMessageAPIView:
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["task_id"] == "task-123"
|
||||
assert uuid.UUID(
|
||||
response.data["task_id"]
|
||||
) # view-generated id, dispatched on commit
|
||||
|
||||
message = models.Message.objects.get(id=draft_message.id)
|
||||
content = message.blob.get_content().decode()
|
||||
@@ -312,7 +321,9 @@ class TestSendMessageAPIView:
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["task_id"] == "task-123"
|
||||
assert uuid.UUID(
|
||||
response.data["task_id"]
|
||||
) # view-generated id, dispatched on commit
|
||||
|
||||
message = models.Message.objects.get(id=draft_message.id)
|
||||
content = message.blob.get_content().decode()
|
||||
@@ -329,6 +340,7 @@ class TestSendMessageAPIView:
|
||||
mailbox,
|
||||
draft_message,
|
||||
signature_template,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Test sending a message with archive=True passes the parameter to the task."""
|
||||
# Authenticate user
|
||||
@@ -337,31 +349,38 @@ class TestSendMessageAPIView:
|
||||
|
||||
# Mock the send_message_task
|
||||
with patch("core.api.viewsets.send.send_message_task") as mock_task:
|
||||
mock_task_instance = MagicMock()
|
||||
mock_task_instance.id = "task-123"
|
||||
mock_task.delay.return_value = mock_task_instance
|
||||
|
||||
# Send request with HTML body only
|
||||
response = client.post(
|
||||
reverse("send-message"),
|
||||
format="json",
|
||||
data={
|
||||
"messageId": str(draft_message.id),
|
||||
"senderId": str(mailbox.id),
|
||||
"htmlBody": "<p>Hello world!</p>",
|
||||
"archive": True,
|
||||
},
|
||||
)
|
||||
# Send request with HTML body only. The delivery task is dispatched
|
||||
# via transaction.on_commit, so capture and run the callbacks.
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
response = client.post(
|
||||
reverse("send-message"),
|
||||
format="json",
|
||||
data={
|
||||
"messageId": str(draft_message.id),
|
||||
"senderId": str(mailbox.id),
|
||||
"htmlBody": "<p>Hello world!</p>",
|
||||
"archive": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["task_id"] == "task-123"
|
||||
assert uuid.UUID(
|
||||
response.data["task_id"]
|
||||
) # view-generated id, dispatched on commit
|
||||
|
||||
mock_task.delay.assert_called_once_with(
|
||||
str(draft_message.id), must_archive=True
|
||||
mock_task.apply_async.assert_called_once_with(
|
||||
args=[str(draft_message.id)],
|
||||
kwargs={"must_archive": True},
|
||||
task_id=response.data["task_id"],
|
||||
)
|
||||
|
||||
def test_api_send_message_with_archive_false(
|
||||
self, user, mailbox_access, mailbox, draft_message
|
||||
self,
|
||||
user,
|
||||
mailbox_access,
|
||||
mailbox,
|
||||
draft_message,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Test sending a message with archive=False passes the parameter to the task."""
|
||||
# Authenticate user
|
||||
@@ -370,33 +389,39 @@ class TestSendMessageAPIView:
|
||||
|
||||
# Mock the send_message_task
|
||||
with patch("core.api.viewsets.send.send_message_task") as mock_task:
|
||||
mock_task_instance = MagicMock()
|
||||
mock_task_instance.id = "task-123"
|
||||
mock_task.delay.return_value = mock_task_instance
|
||||
|
||||
# Send request with archive=False
|
||||
response = client.post(
|
||||
reverse("send-message"),
|
||||
format="json",
|
||||
data={
|
||||
"messageId": str(draft_message.id),
|
||||
"senderId": str(mailbox.id),
|
||||
"textBody": "Hello world!",
|
||||
"htmlBody": "<p>Hello world!</p>",
|
||||
"archive": False,
|
||||
},
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
response = client.post(
|
||||
reverse("send-message"),
|
||||
format="json",
|
||||
data={
|
||||
"messageId": str(draft_message.id),
|
||||
"senderId": str(mailbox.id),
|
||||
"textBody": "Hello world!",
|
||||
"htmlBody": "<p>Hello world!</p>",
|
||||
"archive": False,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["task_id"] == "task-123"
|
||||
assert uuid.UUID(
|
||||
response.data["task_id"]
|
||||
) # view-generated id, dispatched on commit
|
||||
|
||||
# Verify the task was called with must_archive=False
|
||||
mock_task.delay.assert_called_once_with(
|
||||
str(draft_message.id), must_archive=False
|
||||
# Verify the task was dispatched with must_archive=False
|
||||
mock_task.apply_async.assert_called_once_with(
|
||||
args=[str(draft_message.id)],
|
||||
kwargs={"must_archive": False},
|
||||
task_id=response.data["task_id"],
|
||||
)
|
||||
|
||||
def test_api_send_message_without_archive_parameter(
|
||||
self, user, mailbox_access, mailbox, draft_message
|
||||
self,
|
||||
user,
|
||||
mailbox_access,
|
||||
mailbox,
|
||||
draft_message,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Test sending a message without archive parameter defaults to False."""
|
||||
# Authenticate user
|
||||
@@ -405,26 +430,123 @@ class TestSendMessageAPIView:
|
||||
|
||||
# Mock the send_message_task
|
||||
with patch("core.api.viewsets.send.send_message_task") as mock_task:
|
||||
mock_task_instance = MagicMock()
|
||||
mock_task_instance.id = "task-123"
|
||||
mock_task.delay.return_value = mock_task_instance
|
||||
|
||||
# Send request without archive parameter
|
||||
response = client.post(
|
||||
reverse("send-message"),
|
||||
format="json",
|
||||
data={
|
||||
"messageId": str(draft_message.id),
|
||||
"senderId": str(mailbox.id),
|
||||
"textBody": "Hello world!",
|
||||
"htmlBody": "<p>Hello world!</p>",
|
||||
},
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
response = client.post(
|
||||
reverse("send-message"),
|
||||
format="json",
|
||||
data={
|
||||
"messageId": str(draft_message.id),
|
||||
"senderId": str(mailbox.id),
|
||||
"textBody": "Hello world!",
|
||||
"htmlBody": "<p>Hello world!</p>",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.data["task_id"] == "task-123"
|
||||
assert uuid.UUID(
|
||||
response.data["task_id"]
|
||||
) # view-generated id, dispatched on commit
|
||||
|
||||
# Verify the task was called with must_archive=False (default)
|
||||
mock_task.delay.assert_called_once_with(
|
||||
str(draft_message.id), must_archive=False
|
||||
# Verify the task was dispatched with must_archive=False (default)
|
||||
mock_task.apply_async.assert_called_once_with(
|
||||
args=[str(draft_message.id)],
|
||||
kwargs={"must_archive": False},
|
||||
task_id=response.data["task_id"],
|
||||
)
|
||||
|
||||
|
||||
class TestSendMessageSecurity:
|
||||
"""Security regressions for SendMessageView."""
|
||||
|
||||
def test_cannot_send_as_mailbox_user_only_views(
|
||||
self, user, mailbox, draft_message, django_capture_on_commit_callbacks
|
||||
):
|
||||
"""A VIEWER on the sender mailbox cannot send as it by leaning on a
|
||||
SENDER role held on a *different* mailbox that shares the thread.
|
||||
|
||||
Setup:
|
||||
- ``mailbox`` (the senderId, "B"): the draft lives here, user is VIEWER.
|
||||
- ``other_mailbox`` ("A"): user is SENDER, and it also has EDITOR
|
||||
ThreadAccess on the same thread.
|
||||
|
||||
The old object-level check passed as long as the user could SEND
|
||||
through *any* EDITOR mailbox on the thread — so A's SENDER role would
|
||||
wrongly authorise sending as B. The fix re-checks the role on the
|
||||
specific senderId, so this must be 403.
|
||||
"""
|
||||
# User is only a VIEWER on the sender mailbox B.
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox, user=user, role=enums.MailboxRoleChoices.VIEWER
|
||||
)
|
||||
|
||||
# User is SENDER on a different mailbox A that also edits the thread.
|
||||
other_mailbox = factories.MailboxFactory()
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=other_mailbox, user=user, role=enums.MailboxRoleChoices.SENDER
|
||||
)
|
||||
factories.ThreadAccessFactory(
|
||||
mailbox=other_mailbox,
|
||||
thread=draft_message.thread,
|
||||
role=enums.ThreadAccessRoleChoices.EDITOR,
|
||||
)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
|
||||
with patch("core.api.viewsets.send.send_message_task") as mock_task:
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
response = client.post(
|
||||
reverse("send-message"),
|
||||
format="json",
|
||||
data={
|
||||
"messageId": str(draft_message.id),
|
||||
"senderId": str(mailbox.id), # send AS B
|
||||
"textBody": "Hello world!",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
# Nothing was dispatched.
|
||||
mock_task.apply_async.assert_not_called()
|
||||
mock_task.delay.assert_not_called()
|
||||
|
||||
def test_send_task_dispatched_only_after_commit(
|
||||
self,
|
||||
user,
|
||||
mailbox_access,
|
||||
mailbox,
|
||||
draft_message,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""The delivery task is registered on transaction.on_commit, not fired
|
||||
inline — so a rolled-back send never leaks a task to the broker.
|
||||
"""
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
|
||||
with patch("core.api.viewsets.send.send_message_task") as mock_task:
|
||||
# execute=False: capture the on_commit callbacks without running them.
|
||||
with django_capture_on_commit_callbacks(execute=False) as callbacks:
|
||||
response = client.post(
|
||||
reverse("send-message"),
|
||||
format="json",
|
||||
data={
|
||||
"messageId": str(draft_message.id),
|
||||
"senderId": str(mailbox.id),
|
||||
"textBody": "Hello world!",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
# Inside the request/transaction the task must NOT be dispatched.
|
||||
mock_task.apply_async.assert_not_called()
|
||||
|
||||
# The send dispatch was deferred to commit (other unrelated
|
||||
# on_commit hooks, e.g. search reindex, may also be present).
|
||||
send_callbacks = [
|
||||
cb
|
||||
for cb in callbacks
|
||||
if "SendMessageView" in getattr(cb, "__qualname__", "")
|
||||
]
|
||||
assert len(send_callbacks) == 1
|
||||
|
||||
@@ -343,6 +343,33 @@ class TestSubmitValidation:
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_bcc_header_returns_400(self, client, auth_header, mailbox):
|
||||
"""A Bcc header in the submitted MIME is rejected: blind recipients
|
||||
belong in the X-Rcpt-To envelope, not in the (signed, delivered)
|
||||
headers."""
|
||||
mime_with_bcc = (
|
||||
b"From: contact@company.com\r\n"
|
||||
b"To: attendee@example.com\r\n"
|
||||
b"Bcc: secret@example.com\r\n"
|
||||
b"Subject: With Bcc\r\n"
|
||||
b"Message-ID: <bcc-reject@company.com>\r\n"
|
||||
b"Date: Mon, 30 Mar 2026 10:00:00 +0000\r\n"
|
||||
b"MIME-Version: 1.0\r\n"
|
||||
b"Content-Type: text/plain; charset=utf-8\r\n"
|
||||
b"\r\n"
|
||||
b"Hello world\r\n"
|
||||
)
|
||||
response = client.post(
|
||||
SUBMIT_URL,
|
||||
data=mime_with_bcc,
|
||||
content_type="message/rfc822",
|
||||
HTTP_X_MAIL_FROM=str(mailbox.id),
|
||||
HTTP_X_RCPT_TO="attendee@example.com",
|
||||
**auth_header,
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "Bcc" in response.json()["detail"]
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Message creation + DKIM signing + async dispatch
|
||||
@@ -364,19 +391,29 @@ class TestSubmitDispatch:
|
||||
@patch(PREPARE_MOCK, return_value=True)
|
||||
@patch(CREATE_MSG_MOCK)
|
||||
def test_accepted(
|
||||
self, mock_create, mock_prepare, mock_task, client, auth_header, mailbox
|
||||
self,
|
||||
mock_create,
|
||||
mock_prepare,
|
||||
mock_task,
|
||||
client,
|
||||
auth_header,
|
||||
mailbox,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
fake_message = self._fake_message()
|
||||
mock_create.return_value = fake_message
|
||||
|
||||
response = client.post(
|
||||
SUBMIT_URL,
|
||||
data=MINIMAL_MIME,
|
||||
content_type="message/rfc822",
|
||||
HTTP_X_MAIL_FROM=str(mailbox.id),
|
||||
HTTP_X_RCPT_TO="attendee@example.com",
|
||||
**auth_header,
|
||||
)
|
||||
# The delivery task is dispatched via transaction.on_commit, so capture
|
||||
# and run the callbacks to observe the dispatch.
|
||||
with django_capture_on_commit_callbacks(execute=True) as callbacks:
|
||||
response = client.post(
|
||||
SUBMIT_URL,
|
||||
data=MINIMAL_MIME,
|
||||
content_type="message/rfc822",
|
||||
HTTP_X_MAIL_FROM=str(mailbox.id),
|
||||
HTTP_X_RCPT_TO="attendee@example.com",
|
||||
**auth_header,
|
||||
)
|
||||
|
||||
assert response.status_code == 202
|
||||
data = response.json()
|
||||
@@ -392,7 +429,8 @@ class TestSubmitDispatch:
|
||||
mock_prepare.assert_called_once()
|
||||
assert mock_prepare.call_args[1]["raw_mime"] == MINIMAL_MIME
|
||||
|
||||
# Async task dispatched
|
||||
# Async task dispatched, and only after the transaction committed.
|
||||
assert len(callbacks) == 1
|
||||
mock_task.delay.assert_called_once_with(str(fake_message.id))
|
||||
|
||||
@patch(TASK_MOCK)
|
||||
@@ -450,20 +488,28 @@ class TestSubmitIntegration:
|
||||
signing, blob storage) and only mock the final async SMTP task."""
|
||||
|
||||
@patch(TASK_MOCK)
|
||||
def test_full_pipeline(self, mock_task, client, auth_header, mailbox):
|
||||
def test_full_pipeline(
|
||||
self,
|
||||
mock_task,
|
||||
client,
|
||||
auth_header,
|
||||
mailbox,
|
||||
django_capture_on_commit_callbacks,
|
||||
):
|
||||
"""Submit creates a Message with thread, recipients, blob, and dispatches delivery."""
|
||||
mailbox_email = str(mailbox)
|
||||
# X-Rcpt-To matches the To: header in MINIMAL_MIME (attendee@example.com)
|
||||
rcpt_to = "attendee@example.com"
|
||||
|
||||
response = client.post(
|
||||
SUBMIT_URL,
|
||||
data=MINIMAL_MIME,
|
||||
content_type="message/rfc822",
|
||||
HTTP_X_MAIL_FROM=str(mailbox.id),
|
||||
HTTP_X_RCPT_TO=rcpt_to,
|
||||
**auth_header,
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
response = client.post(
|
||||
SUBMIT_URL,
|
||||
data=MINIMAL_MIME,
|
||||
content_type="message/rfc822",
|
||||
HTTP_X_MAIL_FROM=str(mailbox.id),
|
||||
HTTP_X_RCPT_TO=rcpt_to,
|
||||
**auth_header,
|
||||
)
|
||||
|
||||
assert response.status_code == 202
|
||||
data = response.json()
|
||||
@@ -575,6 +621,49 @@ class TestSubmitIntegration:
|
||||
type=MessageRecipientTypeChoices.BCC,
|
||||
).exists()
|
||||
|
||||
@patch(TASK_MOCK)
|
||||
def test_to_less_submission_gets_undisclosed_recipients(
|
||||
self, mock_task, client, auth_header, mailbox
|
||||
):
|
||||
"""A submission with no To/Cc header (every recipient travels via
|
||||
X-Rcpt-To) gets the empty-group placeholder in the stored blob, so it
|
||||
isn't flagged for a missing To."""
|
||||
mailbox_email = str(mailbox)
|
||||
# No To/Cc header at all.
|
||||
mime = (
|
||||
f"From: {mailbox_email}\r\n"
|
||||
f"Subject: No To header\r\n"
|
||||
f"Message-ID: <noto@example.com>\r\n"
|
||||
f"Date: Mon, 30 Mar 2026 10:00:00 +0000\r\n"
|
||||
f"MIME-Version: 1.0\r\n"
|
||||
f"Content-Type: text/plain\r\n"
|
||||
f"\r\n"
|
||||
f"body\r\n"
|
||||
).encode()
|
||||
|
||||
response = client.post(
|
||||
SUBMIT_URL,
|
||||
data=mime,
|
||||
content_type="message/rfc822",
|
||||
HTTP_X_MAIL_FROM=str(mailbox.id),
|
||||
HTTP_X_RCPT_TO="hidden@example.com",
|
||||
**auth_header,
|
||||
)
|
||||
|
||||
assert response.status_code == 202, response.content
|
||||
from core.enums import MessageRecipientTypeChoices
|
||||
from core.models import Message
|
||||
|
||||
message = Message.objects.get(id=response.json()["message_id"])
|
||||
# The envelope-only recipient is stored as BCC, never in the MIME.
|
||||
assert message.recipients.filter(
|
||||
contact__email="hidden@example.com",
|
||||
type=MessageRecipientTypeChoices.BCC,
|
||||
).exists()
|
||||
content = message.blob.get_content()
|
||||
assert b"To: undisclosed-recipients:;" in content
|
||||
assert b"hidden@example.com" not in content
|
||||
|
||||
@patch(TASK_MOCK)
|
||||
def test_cc_recipients_created(self, mock_task, client, auth_header, mailbox):
|
||||
"""Cc recipients from MIME headers are created as MessageRecipient rows."""
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Tests for the threads bulk-delete endpoint (permanent message deletion)."""
|
||||
|
||||
from django.urls import reverse
|
||||
|
||||
import pytest
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from core import enums, factories, models
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
BULK_DELETE_URL = reverse("threads-bulk-delete")
|
||||
|
||||
|
||||
def _editable_thread(user, mailbox=None):
|
||||
"""Create a thread the ``user`` can fully edit, returning (thread, mailbox).
|
||||
|
||||
Full edit rights require an EDITOR ThreadAccess backed by a MailboxAccess
|
||||
whose role is in ``MAILBOX_ROLES_CAN_EDIT`` on the same mailbox.
|
||||
"""
|
||||
mailbox = mailbox or factories.MailboxFactory()
|
||||
if not models.MailboxAccess.objects.filter(mailbox=mailbox, user=user).exists():
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox,
|
||||
user=user,
|
||||
role=enums.MailboxRoleChoices.EDITOR,
|
||||
)
|
||||
thread = factories.ThreadFactory()
|
||||
factories.ThreadAccessFactory(
|
||||
mailbox=mailbox,
|
||||
thread=thread,
|
||||
role=enums.ThreadAccessRoleChoices.EDITOR,
|
||||
)
|
||||
return thread, mailbox
|
||||
|
||||
|
||||
def test_api_bulk_delete_anonymous():
|
||||
"""An anonymous user cannot delete anything."""
|
||||
thread = factories.ThreadFactory()
|
||||
response = APIClient().post(
|
||||
BULK_DELETE_URL,
|
||||
{"scope": "draft", "thread_ids": [str(thread.id)]},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == status.HTTP_401_UNAUTHORIZED
|
||||
|
||||
|
||||
def test_api_bulk_delete_missing_scope():
|
||||
"""A request without a scope is rejected."""
|
||||
user = factories.UserFactory()
|
||||
thread, _ = _editable_thread(user)
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.post(
|
||||
BULK_DELETE_URL, {"thread_ids": [str(thread.id)]}, format="json"
|
||||
)
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
|
||||
def test_api_bulk_delete_invalid_scope():
|
||||
"""An unknown scope is rejected."""
|
||||
user = factories.UserFactory()
|
||||
thread, _ = _editable_thread(user)
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.post(
|
||||
BULK_DELETE_URL,
|
||||
{"scope": "archived", "thread_ids": [str(thread.id)]},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
|
||||
def test_api_bulk_delete_trashed_scope_rejected():
|
||||
"""The 'trashed' scope is intentionally not supported: deleting trashed
|
||||
messages is never exposed, so the request is rejected and nothing removed."""
|
||||
user = factories.UserFactory()
|
||||
thread, _ = _editable_thread(user)
|
||||
trashed = factories.MessageFactory(thread=thread, is_trashed=True)
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.post(
|
||||
BULK_DELETE_URL,
|
||||
{"scope": "trashed", "thread_ids": [str(thread.id)]},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert models.Message.objects.filter(id=trashed.id).exists()
|
||||
|
||||
|
||||
def test_api_bulk_delete_no_targets():
|
||||
"""A request without thread_ids nor message_ids is rejected."""
|
||||
user = factories.UserFactory()
|
||||
_editable_thread(user)
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.post(BULK_DELETE_URL, {"scope": "draft"}, format="json")
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
|
||||
def test_api_bulk_delete_drafts_requires_edit_rights():
|
||||
"""A viewer cannot delete drafts: nothing is removed."""
|
||||
user = factories.UserFactory()
|
||||
mailbox = factories.MailboxFactory()
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox, user=user, role=enums.MailboxRoleChoices.VIEWER
|
||||
)
|
||||
thread = factories.ThreadFactory()
|
||||
factories.ThreadAccessFactory(
|
||||
mailbox=mailbox, thread=thread, role=enums.ThreadAccessRoleChoices.VIEWER
|
||||
)
|
||||
draft = factories.MessageFactory(thread=thread, is_draft=True)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.post(
|
||||
BULK_DELETE_URL,
|
||||
{"scope": "draft", "thread_ids": [str(thread.id)]},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json()["deleted_count"] == 0
|
||||
assert models.Message.objects.filter(id=draft.id).exists()
|
||||
|
||||
|
||||
def test_api_bulk_delete_drafts_only_thread_is_removed():
|
||||
"""Deleting the sole draft of a thread removes the thread too."""
|
||||
user = factories.UserFactory()
|
||||
thread, _ = _editable_thread(user)
|
||||
draft = factories.MessageFactory(thread=thread, is_draft=True)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.post(
|
||||
BULK_DELETE_URL,
|
||||
{"scope": "draft", "thread_ids": [str(thread.id)]},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json() == {"success": True, "deleted_count": 1}
|
||||
assert not models.Message.objects.filter(id=draft.id).exists()
|
||||
assert not models.Thread.objects.filter(id=thread.id).exists()
|
||||
|
||||
|
||||
def test_api_bulk_delete_drafts_keeps_real_messages_of_reply_draft():
|
||||
"""Deleting a reply draft keeps the thread and its real messages."""
|
||||
user = factories.UserFactory()
|
||||
thread, _ = _editable_thread(user)
|
||||
real_message = factories.MessageFactory(thread=thread, is_draft=False)
|
||||
draft = factories.MessageFactory(thread=thread, is_draft=True, parent=real_message)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.post(
|
||||
BULK_DELETE_URL,
|
||||
{"scope": "draft", "thread_ids": [str(thread.id)]},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json()["deleted_count"] == 1
|
||||
assert not models.Message.objects.filter(id=draft.id).exists()
|
||||
assert models.Message.objects.filter(id=real_message.id).exists()
|
||||
|
||||
thread.refresh_from_db()
|
||||
assert thread.has_draft is False
|
||||
assert thread.has_messages is True
|
||||
|
||||
|
||||
def test_api_bulk_delete_drafts_scoped_by_message_ids():
|
||||
"""``message_ids`` restricts deletion to the targeted drafts."""
|
||||
user = factories.UserFactory()
|
||||
mailbox = factories.MailboxFactory()
|
||||
thread_a, _ = _editable_thread(user, mailbox=mailbox)
|
||||
thread_b, _ = _editable_thread(user, mailbox=mailbox)
|
||||
draft_a = factories.MessageFactory(thread=thread_a, is_draft=True)
|
||||
draft_b = factories.MessageFactory(thread=thread_b, is_draft=True)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.post(
|
||||
BULK_DELETE_URL,
|
||||
{"scope": "draft", "message_ids": [str(draft_a.id)]},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json()["deleted_count"] == 1
|
||||
assert not models.Message.objects.filter(id=draft_a.id).exists()
|
||||
assert models.Message.objects.filter(id=draft_b.id).exists()
|
||||
|
||||
|
||||
def test_api_bulk_delete_drafts_ignores_non_draft_messages():
|
||||
"""The ``draft`` scope never touches trashed or active messages."""
|
||||
user = factories.UserFactory()
|
||||
thread, _ = _editable_thread(user)
|
||||
draft = factories.MessageFactory(thread=thread, is_draft=True)
|
||||
trashed = factories.MessageFactory(thread=thread, is_trashed=True)
|
||||
active = factories.MessageFactory(thread=thread, is_draft=False)
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(user=user)
|
||||
response = client.post(
|
||||
BULK_DELETE_URL,
|
||||
{"scope": "draft", "thread_ids": [str(thread.id)]},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json()["deleted_count"] == 1
|
||||
assert not models.Message.objects.filter(id=draft.id).exists()
|
||||
assert models.Message.objects.filter(id=trashed.id).exists()
|
||||
assert models.Message.objects.filter(id=active.id).exists()
|
||||
@@ -5,7 +5,7 @@ from datetime import timedelta
|
||||
from unittest import mock
|
||||
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from django.test.utils import CaptureQueriesContext, override_settings
|
||||
from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
|
||||
@@ -1801,6 +1801,50 @@ class TestThreadListAPI:
|
||||
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
@override_settings(OPENSEARCH_HOSTS=["http://opensearch:9200"])
|
||||
def test_search_without_mailbox_id_scopes_to_accessible_mailboxes(
|
||||
self, api_client, url
|
||||
):
|
||||
"""A search with no mailbox_id must be scoped to the user's own
|
||||
mailboxes — never run cluster-wide (which would leak hit-totals /
|
||||
content-existence across every mailbox)."""
|
||||
user = UserFactory()
|
||||
api_client.force_authenticate(user=user)
|
||||
|
||||
mbx_a = MailboxFactory(users_read=[user])
|
||||
mbx_b = MailboxFactory(users_read=[user])
|
||||
# A mailbox the user cannot access — must not be in the search scope.
|
||||
MailboxFactory(users_read=[UserFactory()])
|
||||
|
||||
with mock.patch("core.api.viewsets.thread.search_threads") as mock_search:
|
||||
mock_search.return_value = {"threads": [], "total": 0}
|
||||
response = api_client.get(url, {"search": "test query"})
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
passed_mailbox_ids = mock_search.call_args.kwargs["mailbox_ids"]
|
||||
# Scoped to exactly the user's accessible mailboxes, and never None.
|
||||
assert passed_mailbox_ids is not None
|
||||
assert set(passed_mailbox_ids) == {str(mbx_a.id), str(mbx_b.id)}
|
||||
|
||||
@override_settings(OPENSEARCH_HOSTS=["http://opensearch:9200"])
|
||||
def test_search_without_mailbox_id_and_no_access_passes_empty_scope(
|
||||
self, api_client, url
|
||||
):
|
||||
"""A user with no mailbox access searches an empty scope (not the whole
|
||||
cluster). The viewset passes [] — which search_threads treats as
|
||||
'no results' rather than 'all mailboxes'."""
|
||||
user = UserFactory()
|
||||
api_client.force_authenticate(user=user)
|
||||
# Some other user's mailbox exists, but ours has none.
|
||||
MailboxFactory(users_read=[UserFactory()])
|
||||
|
||||
with mock.patch("core.api.viewsets.thread.search_threads") as mock_search:
|
||||
mock_search.return_value = {"threads": [], "total": 0}
|
||||
response = api_client.get(url, {"search": "test query"})
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert mock_search.call_args.kwargs["mailbox_ids"] == []
|
||||
|
||||
|
||||
class TestThreadListEventsCount:
|
||||
"""Test that ThreadSerializer exposes events_count on the list endpoint.
|
||||
|
||||
@@ -167,6 +167,7 @@ class TestUsersGetMe:
|
||||
assert "full_name" in data
|
||||
|
||||
|
||||
# pylint: disable=too-many-public-methods
|
||||
class TestAdminUsersList:
|
||||
"""Test suite for the admin user list endpoint API."""
|
||||
|
||||
@@ -248,6 +249,42 @@ class TestAdminUsersList:
|
||||
for user in response.data:
|
||||
assert user["email"] in [admin_user.email, mailbox_user.email]
|
||||
|
||||
def test_api_admin_users_list_allowed_mailbox_admin(self, api_client):
|
||||
"""A mailbox admin (not a domain admin) can search users within the
|
||||
domain of a mailbox they administer."""
|
||||
domain = factories.MailDomainFactory(name="sardine.local")
|
||||
mailbox = factories.MailboxFactory(domain=domain)
|
||||
admin_user = factories.UserFactory(email="admin@sardine.local")
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox,
|
||||
user=admin_user,
|
||||
role=enums.MailboxRoleChoices.ADMIN,
|
||||
)
|
||||
|
||||
url = reverse("users-list")
|
||||
api_client.force_authenticate(user=admin_user)
|
||||
response = api_client.get(url, {"maildomain_pk": domain.id, "q": "sardine"})
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert [user["email"] for user in response.data] == [admin_user.email]
|
||||
|
||||
def test_api_admin_users_list_mailbox_non_admin_forbidden(self, api_client):
|
||||
"""A non-admin mailbox member cannot search users in the domain."""
|
||||
domain = factories.MailDomainFactory(name="sardine.local")
|
||||
mailbox = factories.MailboxFactory(domain=domain)
|
||||
member = factories.UserFactory(email="member@sardine.local")
|
||||
factories.MailboxAccessFactory(
|
||||
mailbox=mailbox,
|
||||
user=member,
|
||||
role=enums.MailboxRoleChoices.EDITOR,
|
||||
)
|
||||
|
||||
url = reverse("users-list")
|
||||
api_client.force_authenticate(user=member)
|
||||
response = api_client.get(url, {"maildomain_pk": domain.id, "q": "sardine"})
|
||||
|
||||
assert response.status_code == status.HTTP_403_FORBIDDEN
|
||||
|
||||
def test_api_admin_users_list_search_query_mandatory(self, api_client):
|
||||
"""
|
||||
Test list users endpoint returns an empty list if no search query is provided.
|
||||
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _mock_ssrf_dns():
|
||||
"""Short-circuit SSRF DNS validation for IMAP tests.
|
||||
"""Short-circuit SSRF hostname validation for IMAP tests.
|
||||
|
||||
The IMAP import path validates the server hostname via
|
||||
``core.services.ssrf.validate_hostname``. Test fixtures use unresolvable
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Tests for IMAP connection manager and security features."""
|
||||
|
||||
# pylint: disable=redefined-outer-name,invalid-name
|
||||
# pylint: disable=redefined-outer-name,invalid-name,protected-access
|
||||
|
||||
import imaplib
|
||||
import ssl
|
||||
@@ -8,17 +8,80 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.services.importer.imap import IMAPConnectionManager, IMAPSecurityError
|
||||
from core.services.importer.imap import (
|
||||
IMAPConnectionManager,
|
||||
IMAPSecurityError,
|
||||
_IPPinnedIMAP4,
|
||||
_IPPinnedIMAP4SSL,
|
||||
_validate_imap_host,
|
||||
)
|
||||
from core.services.ssrf import SSRFValidationError
|
||||
|
||||
# Store reference to the real error class before any patching
|
||||
# This is needed because patching imaplib.IMAP4 affects the module globally
|
||||
IMAP4_ERROR = imaplib.IMAP4.error
|
||||
|
||||
|
||||
class TestIMAPSSRFPinning:
|
||||
"""The IMAP importer connects to the validated IP, not a re-resolved
|
||||
hostname — closing the DNS-rebinding (TOCTOU) SSRF window."""
|
||||
|
||||
def test_validate_imap_host_returns_first_validated_ip(self):
|
||||
"""The first validated IP is the address pinned for the connection."""
|
||||
with patch(
|
||||
"core.services.importer.imap.validate_hostname",
|
||||
return_value=["203.0.113.5", "203.0.113.6"],
|
||||
):
|
||||
assert _validate_imap_host("imap.example.com") == "203.0.113.5"
|
||||
|
||||
def test_validate_imap_host_rejects_blocked_address(self):
|
||||
"""A host resolving to a blocked address raises ValueError."""
|
||||
with patch(
|
||||
"core.services.importer.imap.validate_hostname",
|
||||
side_effect=SSRFValidationError("resolves to private IP address"),
|
||||
):
|
||||
with pytest.raises(ValueError, match="not allowed"):
|
||||
_validate_imap_host("internal.evil.test")
|
||||
|
||||
def test_pinned_imap4_dials_validated_ip(self):
|
||||
"""Plain IMAP4 connects to the pinned IP, never re-resolving the host."""
|
||||
inst = _IPPinnedIMAP4.__new__(_IPPinnedIMAP4)
|
||||
inst._connect_ip = "203.0.113.5"
|
||||
inst.port = 143
|
||||
fake_sock = MagicMock()
|
||||
with patch(
|
||||
"core.services.importer.imap.socket.create_connection",
|
||||
return_value=fake_sock,
|
||||
) as mock_conn:
|
||||
result = inst._create_socket(30)
|
||||
mock_conn.assert_called_once_with(("203.0.113.5", 143), 30)
|
||||
assert result is fake_sock
|
||||
|
||||
def test_pinned_imap4ssl_pins_ip_and_verifies_hostname(self):
|
||||
"""SSL: dial the pinned IP but verify the cert against the hostname."""
|
||||
inst = _IPPinnedIMAP4SSL.__new__(_IPPinnedIMAP4SSL)
|
||||
inst._connect_ip = "203.0.113.5"
|
||||
inst.port = 993
|
||||
inst.host = "imap.example.com"
|
||||
inst.ssl_context = MagicMock()
|
||||
raw_sock, wrapped = MagicMock(), MagicMock()
|
||||
inst.ssl_context.wrap_socket.return_value = wrapped
|
||||
with patch(
|
||||
"core.services.importer.imap.socket.create_connection",
|
||||
return_value=raw_sock,
|
||||
) as mock_conn:
|
||||
result = inst._create_socket(30)
|
||||
mock_conn.assert_called_once_with(("203.0.113.5", 993), 30)
|
||||
inst.ssl_context.wrap_socket.assert_called_once_with(
|
||||
raw_sock, server_hostname="imap.example.com"
|
||||
)
|
||||
assert result is wrapped
|
||||
|
||||
|
||||
class TestIMAPConnectionManagerSSLDirect:
|
||||
"""Tests for SSL direct connections (typically port 993)."""
|
||||
|
||||
@patch("core.services.importer.imap.imaplib.IMAP4_SSL")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4SSL")
|
||||
def test_ssl_direct_success(self, mock_imap4_ssl):
|
||||
"""Test successful SSL direct connection on port 993."""
|
||||
mock_conn = MagicMock()
|
||||
@@ -35,7 +98,7 @@ class TestIMAPConnectionManagerSSLDirect:
|
||||
mock_imap4_ssl.assert_called_once()
|
||||
mock_conn.login.assert_called_once_with("user@example.com", "password")
|
||||
|
||||
@patch("core.services.importer.imap.imaplib.IMAP4_SSL")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4SSL")
|
||||
def test_ssl_direct_handshake_failure(self, mock_imap4_ssl):
|
||||
"""Test SSL handshake failure raises IMAPSecurityError."""
|
||||
mock_imap4_ssl.side_effect = ssl.SSLError("handshake failed")
|
||||
@@ -57,7 +120,7 @@ class TestIMAPConnectionManagerSSLDirect:
|
||||
class TestIMAPConnectionManagerSTARTTLS:
|
||||
"""Tests for STARTTLS connections (typically port 143 with use_ssl=True)."""
|
||||
|
||||
@patch("core.services.importer.imap.imaplib.IMAP4")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4")
|
||||
def test_starttls_success(self, mock_imap4):
|
||||
"""Test successful STARTTLS upgrade on port 143."""
|
||||
mock_conn = MagicMock()
|
||||
@@ -77,7 +140,7 @@ class TestIMAPConnectionManagerSTARTTLS:
|
||||
mock_conn.starttls.assert_called_once()
|
||||
mock_conn.login.assert_called_once_with("user@example.com", "password")
|
||||
|
||||
@patch("core.services.importer.imap.imaplib.IMAP4")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4")
|
||||
def test_starttls_not_supported(self, mock_imap4):
|
||||
"""Test STARTTLS not supported raises IMAPSecurityError."""
|
||||
mock_conn = MagicMock()
|
||||
@@ -98,7 +161,7 @@ class TestIMAPConnectionManagerSTARTTLS:
|
||||
assert "does not support STARTTLS" in str(exc_info.value)
|
||||
mock_conn.logout.assert_called_once()
|
||||
|
||||
@patch("core.services.importer.imap.imaplib.IMAP4")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4")
|
||||
def test_starttls_negotiation_failure(self, mock_imap4):
|
||||
"""Test STARTTLS negotiation failure raises IMAPSecurityError."""
|
||||
mock_conn = MagicMock()
|
||||
@@ -119,7 +182,7 @@ class TestIMAPConnectionManagerSTARTTLS:
|
||||
assert "STARTTLS failed" in str(exc_info.value)
|
||||
mock_conn.logout.assert_called_once()
|
||||
|
||||
@patch("core.services.importer.imap.imaplib.IMAP4")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4")
|
||||
def test_starttls_capability_empty_response(self, mock_imap4):
|
||||
"""Test STARTTLS with empty capability response raises IMAPSecurityError."""
|
||||
mock_conn = MagicMock()
|
||||
@@ -139,7 +202,7 @@ class TestIMAPConnectionManagerSTARTTLS:
|
||||
|
||||
assert "does not support STARTTLS" in str(exc_info.value)
|
||||
|
||||
@patch("core.services.importer.imap.imaplib.IMAP4")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4")
|
||||
def test_starttls_capability_none_response(self, mock_imap4):
|
||||
"""Test STARTTLS with None capability response raises IMAPSecurityError."""
|
||||
mock_conn = MagicMock()
|
||||
@@ -163,7 +226,7 @@ class TestIMAPConnectionManagerSTARTTLS:
|
||||
class TestIMAPConnectionManagerUnencrypted:
|
||||
"""Tests for unencrypted connections (use_ssl=False)."""
|
||||
|
||||
@patch("core.services.importer.imap.imaplib.IMAP4")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4")
|
||||
def test_unencrypted_connection(self, mock_imap4):
|
||||
"""Test unencrypted connection when use_ssl=False."""
|
||||
mock_conn = MagicMock()
|
||||
@@ -185,7 +248,7 @@ class TestIMAPConnectionManagerUnencrypted:
|
||||
class TestIMAPConnectionManagerAuthentication:
|
||||
"""Tests for authentication handling."""
|
||||
|
||||
@patch("core.services.importer.imap.imaplib.IMAP4_SSL")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4SSL")
|
||||
def test_authentication_failure_cleanup(self, mock_imap4_ssl):
|
||||
"""Test connection is cleaned up after authentication failure."""
|
||||
mock_conn = MagicMock()
|
||||
@@ -205,7 +268,7 @@ class TestIMAPConnectionManagerAuthentication:
|
||||
# Connection should be cleaned up via logout
|
||||
mock_conn.logout.assert_called_once()
|
||||
|
||||
@patch("core.services.importer.imap.imaplib.IMAP4")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4")
|
||||
def test_authentication_failure_after_starttls(self, mock_imap4):
|
||||
"""Test auth failure after successful STARTTLS still cleans up."""
|
||||
mock_conn = MagicMock()
|
||||
|
||||
@@ -173,7 +173,7 @@ def test_imap_import_form_view(admin_client, mailbox):
|
||||
mock_task.assert_called_once()
|
||||
|
||||
|
||||
@patch("imaplib.IMAP4_SSL")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4SSL")
|
||||
@patch.object(celery_app.backend, "store_result")
|
||||
def test_imap_import_task_success(
|
||||
mock_store_result, mock_imap4_ssl, mailbox, mock_imap_connection, sample_email
|
||||
@@ -267,7 +267,7 @@ def test_imap_import_task_login_failure(mailbox):
|
||||
# Mock IMAP connection to raise an error on login
|
||||
with (
|
||||
patch.object(import_imap_messages_task, "update_state", mock_task.update_state),
|
||||
patch("core.services.importer.imap.imaplib.IMAP4_SSL") as mock_imap,
|
||||
patch("core.services.importer.imap._IPPinnedIMAP4SSL") as mock_imap,
|
||||
):
|
||||
mock_imap_instance = MagicMock()
|
||||
mock_imap.return_value = mock_imap_instance
|
||||
@@ -299,7 +299,7 @@ def test_imap_import_task_login_failure(mailbox):
|
||||
assert Message.objects.count() == 0
|
||||
|
||||
|
||||
@patch("imaplib.IMAP4_SSL")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4SSL")
|
||||
@patch.object(celery_app.backend, "store_result")
|
||||
def test_imap_import_task_message_fetch_failure(
|
||||
mock_store_result, mock_imap4_ssl, mailbox
|
||||
@@ -374,7 +374,7 @@ def test_imap_import_task_message_fetch_failure(
|
||||
|
||||
|
||||
@patch("core.mda.inbound.logger")
|
||||
@patch("imaplib.IMAP4_SSL")
|
||||
@patch("core.services.importer.imap._IPPinnedIMAP4SSL")
|
||||
@patch.object(celery_app.backend, "store_result")
|
||||
def test_imap_import_task_duplicate_recipients(
|
||||
mock_store_result,
|
||||
|
||||
@@ -453,6 +453,79 @@ def test_import_file_invalid_file(admin_user, mailbox, mock_request):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_import_file_mbox_misclassified_by_libmagic(admin_user, mailbox, mock_request):
|
||||
"""Regression: an mbox file must be recognized even when libmagic
|
||||
misclassifies it (e.g. returns text/html because the first message body
|
||||
contains HTML strong enough to outrank the ``From `` envelope signature).
|
||||
|
||||
Observed against libmagic 5.41 (Ubuntu 22.04, the libmagic Scalingo's
|
||||
scalingo-22 stack ships) on a real multipart/alternative Zimbra-exported
|
||||
mbox: libmagic 5.41 returns text/html and the upload is rejected. Debian
|
||||
13's libmagic 5.46 — used in the dev container and the distroless image
|
||||
built from src/backend/Dockerfile — has tuned scoring so the same bytes
|
||||
classify as application/mbox, which is why this test mocks
|
||||
``magic.from_buffer`` rather than relying on real bytes (a real-bytes
|
||||
fixture passes trivially on the dev libmagic regardless of the fix).
|
||||
|
||||
RFC 4155 requires every mbox file to start with a ``From `` envelope line
|
||||
at offset 0; we trust that signature ahead of libmagic.
|
||||
"""
|
||||
mbox_content = (
|
||||
b"From sender@example.com Mon Jun 01 00:00:00 2026\r\n"
|
||||
b"From: sender@example.com\r\n"
|
||||
b"To: jean.recipient@example.com\r\n"
|
||||
b"Subject: HTML body trips libmagic\r\n"
|
||||
b"MIME-Version: 1.0\r\n"
|
||||
b"Content-Type: text/html; charset=utf-8\r\n"
|
||||
b"\r\n"
|
||||
b"<!DOCTYPE html><html><body><h1>Hello</h1></body></html>\r\n"
|
||||
)
|
||||
uploaded = SimpleUploadedFile(
|
||||
# Deliberately no .mbox extension so the extension fallback can't rescue us.
|
||||
"ambiguous-upload",
|
||||
mbox_content,
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
file_key = get_file_key(admin_user.id, uploaded.name)
|
||||
storage = storages["message-imports"]
|
||||
s3_client = storage.connection.meta.client
|
||||
s3_client.put_object(
|
||||
Bucket=storage.bucket_name,
|
||||
Key=file_key,
|
||||
Body=mbox_content,
|
||||
ContentType=uploaded.content_type,
|
||||
)
|
||||
|
||||
try:
|
||||
with (
|
||||
patch(
|
||||
"core.services.importer.service.magic.from_buffer",
|
||||
return_value="text/html",
|
||||
) as mock_magic,
|
||||
patch(
|
||||
"core.services.importer.mbox_tasks.process_mbox_file_task.delay"
|
||||
) as mock_task,
|
||||
):
|
||||
mock_task.return_value.id = "fake-task-id"
|
||||
success, response_data = ImportService.import_file(
|
||||
file_key=file_key,
|
||||
recipient=mailbox,
|
||||
user=admin_user,
|
||||
request=mock_request,
|
||||
filename=uploaded.name,
|
||||
)
|
||||
|
||||
assert success is True, response_data
|
||||
assert response_data["type"] == "mbox"
|
||||
mock_task.assert_called_once()
|
||||
# The RFC 4155 ``From `` envelope at offset 0 must short-circuit
|
||||
# detection — libmagic is never consulted on this branch.
|
||||
mock_magic.assert_not_called()
|
||||
finally:
|
||||
s3_client.delete_object(Bucket=storage.bucket_name, Key=file_key)
|
||||
|
||||
|
||||
def test_import_imap_by_superuser(admin_user, mailbox, mock_request):
|
||||
"""Test successful IMAP import."""
|
||||
with patch(
|
||||
@@ -559,7 +632,7 @@ def test_import_imap_messages_by_superuser(admin_user, mailbox, mock_request):
|
||||
"""Test importing messages from IMAP server by superuser."""
|
||||
|
||||
# Mock IMAP connection and responses
|
||||
with patch("imaplib.IMAP4_SSL") as mock_imap:
|
||||
with patch("core.services.importer.imap._IPPinnedIMAP4SSL") as mock_imap:
|
||||
mock_imap_instance = mock_imap.return_value
|
||||
|
||||
# Mock login
|
||||
@@ -646,7 +719,7 @@ def test_import_imap_messages_user_with_access(user, mailbox, mock_request):
|
||||
mailbox.accesses.create(user=user, role=MailboxRoleChoices.ADMIN)
|
||||
|
||||
# Mock IMAP connection and responses
|
||||
with patch("imaplib.IMAP4_SSL") as mock_imap:
|
||||
with patch("core.services.importer.imap._IPPinnedIMAP4SSL") as mock_imap:
|
||||
mock_imap_instance = mock_imap.return_value
|
||||
|
||||
# Mock login
|
||||
@@ -743,7 +816,7 @@ def test_import_messages_do_not_trigger_ai_features(
|
||||
mailbox.accesses.create(user=user, role=MailboxRoleChoices.ADMIN)
|
||||
|
||||
# Mock IMAP connection and responses
|
||||
with patch("imaplib.IMAP4_SSL") as mock_imap:
|
||||
with patch("core.services.importer.imap._IPPinnedIMAP4SSL") as mock_imap:
|
||||
mock_imap_instance = mock_imap.return_value
|
||||
|
||||
# Mock login
|
||||
|
||||
@@ -4,12 +4,14 @@
|
||||
# pylint: disable=too-many-lines, too-many-arguments, broad-exception-caught
|
||||
|
||||
import email
|
||||
import email.policy
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
from django.core.files.storage import storages
|
||||
|
||||
import pytest
|
||||
from jmap_email import parse_email
|
||||
|
||||
from core.models import Mailbox, MailDomain, Message
|
||||
from core.services.importer.pst import (
|
||||
@@ -261,6 +263,7 @@ def _make_folder(
|
||||
# --- reconstruct_eml tests ---
|
||||
|
||||
|
||||
# pylint: disable=too-many-public-methods
|
||||
class TestReconstructEml:
|
||||
"""Tests for EML reconstruction from pypff messages."""
|
||||
|
||||
@@ -296,7 +299,7 @@ class TestReconstructEml:
|
||||
assert "Hello world" in text_parts[0].get_payload(decode=True).decode()
|
||||
|
||||
def test_reconstruct_preserves_rfc5322_date(self):
|
||||
"""Test that RFC5322 date from transport headers is preserved correctly."""
|
||||
"""Test that RFC 5322 date from transport headers is preserved correctly."""
|
||||
transport = (
|
||||
"From: sender@example.com\r\nDate: Mon, 26 May 2025 10:00:00 +0000\r\n"
|
||||
)
|
||||
@@ -588,6 +591,106 @@ class TestReconstructEml:
|
||||
assert len(att_parts) == 1
|
||||
assert att_parts[0].get_content_type() == "application/octet-stream"
|
||||
|
||||
def test_reconstruct_delivery_status_attachment_does_not_crash(self):
|
||||
"""A DSN delivery-status part imports without crashing the composer.
|
||||
|
||||
Regression: bounce/read-receipt reports in a PST carry the
|
||||
delivery-status body as a flat-bytes MAPI attachment. Fed to the strict
|
||||
composer as message/delivery-status, email.generator walked the base64
|
||||
string character by character and raised "'str' object has no attribute
|
||||
'policy'", dropping the whole message. The composer now relabels the
|
||||
part to text/plain (content preserved, readable), so reconstructing the
|
||||
report no longer crashes.
|
||||
"""
|
||||
dsn = (
|
||||
b"Reporting-MTA: dns; mx.example.com\r\n"
|
||||
b"Final-Recipient: rfc822; nobody@example.com\r\n"
|
||||
b"Action: failed\r\nStatus: 5.1.1\r\n"
|
||||
)
|
||||
att = _make_attachment(
|
||||
data=dsn,
|
||||
long_filename="details.txt",
|
||||
mime_type="message/delivery-status",
|
||||
)
|
||||
msg = _make_message(
|
||||
transport_headers="From: daemon@example.com\r\n",
|
||||
plain_text_body="Delivery failed",
|
||||
num_attachments=1,
|
||||
attachments=[att],
|
||||
)
|
||||
eml_bytes = reconstruct_eml(msg)
|
||||
parsed = email.message_from_bytes(eml_bytes, policy=email.policy.default)
|
||||
|
||||
part = next(p for p in parsed.walk() if p.get_filename() == "details.txt")
|
||||
# Relabelled to a safe leaf type; content preserved verbatim.
|
||||
assert part.get_content_type() == "text/plain"
|
||||
assert part.get_payload(decode=True) == dsn
|
||||
# The original message/delivery-status type is never emitted.
|
||||
assert b"message/delivery-status" not in eml_bytes
|
||||
|
||||
def test_reconstruct_skips_empty_attachment(self):
|
||||
"""Blank/whitespace-only attachments (e.g. empty DSN parts) are dropped.
|
||||
|
||||
libpff surfaces empty diagnostic report parts as attachments; importing
|
||||
them produces 0-byte attachments that render as broken in the UI while
|
||||
carrying no information.
|
||||
"""
|
||||
empty = _make_attachment(
|
||||
data=b"\r\n",
|
||||
long_filename="empty.txt",
|
||||
mime_type="text/rfc822-headers",
|
||||
)
|
||||
msg = _make_message(
|
||||
transport_headers="From: a@b.com\r\n",
|
||||
plain_text_body="body",
|
||||
num_attachments=1,
|
||||
attachments=[empty],
|
||||
)
|
||||
eml_bytes = reconstruct_eml(msg)
|
||||
parsed = email.message_from_bytes(eml_bytes)
|
||||
|
||||
att_parts = [
|
||||
p for p in parsed.walk() if p.get_content_disposition() == "attachment"
|
||||
]
|
||||
assert att_parts == []
|
||||
|
||||
def test_reconstruct_rfc822_headers_attachment_is_displayable(self):
|
||||
"""text/rfc822-headers is relabelled so it doesn't show as 0 bytes.
|
||||
|
||||
Regression: the original-headers part of a DSN composes fine, but
|
||||
on re-parse the display parser previously dropped the body of
|
||||
text/rfc822-headers, so the UI showed a 0-byte attachment. The
|
||||
importer normalizes the label to text/plain, which round-trips
|
||||
with its content intact.
|
||||
"""
|
||||
|
||||
headers = (
|
||||
b"Return-Path: <sender@example.com>\r\n"
|
||||
b"From: Sender <sender@example.com>\r\n"
|
||||
b"To: Recipient <rcpt@example.com>\r\n"
|
||||
b"Subject: original\r\n"
|
||||
)
|
||||
att = _make_attachment(
|
||||
data=headers,
|
||||
long_filename="Message Headers.txt",
|
||||
mime_type="text/rfc822-headers",
|
||||
)
|
||||
msg = _make_message(
|
||||
transport_headers="From: daemon@example.com\r\n",
|
||||
plain_text_body="Delivered",
|
||||
num_attachments=1,
|
||||
attachments=[att],
|
||||
)
|
||||
eml_bytes = reconstruct_eml(msg)
|
||||
|
||||
# Re-parse the stored .eml the way the message view does.
|
||||
parsed = parse_email(eml_bytes)
|
||||
headers_att = next(
|
||||
a for a in parsed["attachments"] if a["name"] == "Message Headers.txt"
|
||||
)
|
||||
assert headers_att["type"] == "text/plain"
|
||||
assert headers_att["size"] == len(headers)
|
||||
|
||||
def test_reconstruct_empty_message(self):
|
||||
"""Test EML reconstruction with no body."""
|
||||
msg = _make_message(
|
||||
@@ -941,11 +1044,11 @@ class TestDisplayRecipients:
|
||||
assert not _parse_display_recipients(" ")
|
||||
|
||||
def test_parse_recovers_bare_email_when_parser_returns_empty_addr(self):
|
||||
"""If parse_email_address returns no usable address but the original
|
||||
"""If parse_address returns no usable address but the original
|
||||
token contains '@', the token itself is salvaged as the email — guards
|
||||
against flanker quirks where the address ends up in the name slot."""
|
||||
against parser quirks where the address ends up in the name slot."""
|
||||
with patch(
|
||||
"core.services.importer.pst.parse_email_address",
|
||||
"core.services.importer.pst.parse_address",
|
||||
return_value=("", ""),
|
||||
):
|
||||
result = _parse_display_recipients("salvage@example.com")
|
||||
@@ -1227,6 +1330,46 @@ class TestFolderIdentification:
|
||||
count = count_pst_messages(pst, {})
|
||||
assert count == 1
|
||||
|
||||
def test_process_imap_folder(self):
|
||||
"""IMAP-archived PSTs tag mail folders IPF.Imap — they must be counted."""
|
||||
msg = _make_message(delivery_time=datetime(2025, 1, 1, tzinfo=timezone.utc))
|
||||
folder = _make_folder(
|
||||
name="Boîte de réception",
|
||||
messages=[msg],
|
||||
container_class="IPF.Imap",
|
||||
)
|
||||
root = _make_folder(name="Root", subfolders=[folder])
|
||||
|
||||
pst = Mock()
|
||||
pst.get_root_folder.return_value = root
|
||||
pst.get_message_store.return_value = Mock(number_of_record_sets=0)
|
||||
|
||||
count = count_pst_messages(pst, {})
|
||||
assert count == 1
|
||||
|
||||
def test_walk_yields_messages_from_imap_folder(self):
|
||||
"""walk_pst_messages (the import path) must yield IPF.Imap messages."""
|
||||
msg = _make_message(
|
||||
subject="IMAP message",
|
||||
transport_headers="From: a@example.com\r\nTo: b@example.com\r\n",
|
||||
delivery_time=datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
folder = _make_folder(
|
||||
name="Boîte de réception",
|
||||
messages=[msg],
|
||||
container_class="IPF.Imap",
|
||||
)
|
||||
root = _make_folder(name="Root", subfolders=[folder])
|
||||
|
||||
pst = Mock()
|
||||
pst.get_root_folder.return_value = root
|
||||
pst.get_message_store.return_value = Mock(number_of_record_sets=0)
|
||||
pst.get_name_to_id_map.side_effect = Exception("no named props")
|
||||
|
||||
results = list(walk_pst_messages(pst, {}))
|
||||
assert len(results) == 1
|
||||
assert results[0][4] is not None # eml_bytes reconstructed
|
||||
|
||||
def test_sent_folder_identification_via_entry_id(self):
|
||||
"""Test identifying Sent Items via message store folder identifier."""
|
||||
special_map = {100: FOLDER_TYPE_SENT}
|
||||
|
||||
@@ -86,78 +86,102 @@ class TestIsAutoReplyMessage:
|
||||
|
||||
def test_normal_message_passes(self):
|
||||
"""Normal message is not detected as auto-reply."""
|
||||
headers = {"From": "user@example.com", "Subject": "Hello"}
|
||||
assert _is_auto_reply_message(headers) is False
|
||||
parsed_email = {
|
||||
"headers": [
|
||||
{"name": "From", "value": "user@example.com"},
|
||||
{"name": "Subject", "value": "Hello"},
|
||||
]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is False
|
||||
|
||||
def test_empty_headers(self):
|
||||
"""Empty or None headers are not detected as auto-reply."""
|
||||
"""Empty or missing headers are not detected as auto-reply."""
|
||||
assert _is_auto_reply_message({}) is False
|
||||
assert _is_auto_reply_message(None) is False
|
||||
assert _is_auto_reply_message({"headers": []}) is False
|
||||
|
||||
def test_auto_submitted_auto_replied(self):
|
||||
"""Auto-Submitted: auto-replied is detected."""
|
||||
headers = {"Auto-Submitted": "auto-replied"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [{"name": "Auto-Submitted", "value": "auto-replied"}]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_auto_submitted_auto_generated(self):
|
||||
"""Auto-Submitted: auto-generated is detected."""
|
||||
headers = {"Auto-Submitted": "auto-generated"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [{"name": "Auto-Submitted", "value": "auto-generated"}]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_auto_submitted_no_passes(self):
|
||||
"""Auto-Submitted: no is not detected as auto-reply."""
|
||||
headers = {"Auto-Submitted": "no"}
|
||||
assert _is_auto_reply_message(headers) is False
|
||||
parsed_email = {"headers": [{"name": "Auto-Submitted", "value": "no"}]}
|
||||
assert _is_auto_reply_message(parsed_email) is False
|
||||
|
||||
def test_precedence_bulk(self):
|
||||
"""Precedence: bulk is detected."""
|
||||
headers = {"Precedence": "bulk"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {"headers": [{"name": "Precedence", "value": "bulk"}]}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_precedence_list(self):
|
||||
"""Precedence: list is detected."""
|
||||
headers = {"Precedence": "list"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {"headers": [{"name": "Precedence", "value": "list"}]}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_precedence_junk(self):
|
||||
"""Precedence: junk is detected."""
|
||||
headers = {"Precedence": "junk"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {"headers": [{"name": "Precedence", "value": "junk"}]}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_list_id_header(self):
|
||||
"""List-Id header is detected."""
|
||||
headers = {"List-Id": "<list.example.com>"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {"headers": [{"name": "List-Id", "value": "<list.example.com>"}]}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_list_unsubscribe_header(self):
|
||||
"""List-Unsubscribe header is detected."""
|
||||
headers = {"List-Unsubscribe": "<mailto:unsub@example.com>"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [
|
||||
{"name": "List-Unsubscribe", "value": "<mailto:unsub@example.com>"}
|
||||
]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_x_auto_response_suppress(self):
|
||||
"""X-Auto-Response-Suppress header is detected."""
|
||||
headers = {"X-Auto-Response-Suppress": "All"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [{"name": "X-Auto-Response-Suppress", "value": "All"}]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_x_autoreply(self):
|
||||
"""X-Autoreply header is detected."""
|
||||
headers = {"X-Autoreply": "yes"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {"headers": [{"name": "X-Autoreply", "value": "yes"}]}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_x_autorespond(self):
|
||||
"""X-Autorespond header is detected."""
|
||||
headers = {"X-Autorespond": "yes"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {"headers": [{"name": "X-Autorespond", "value": "yes"}]}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_auto_submitted_with_parameters(self):
|
||||
"""Auto-Submitted with RFC 3834 parameters after semicolon is detected."""
|
||||
headers = {"Auto-Submitted": 'auto-replied; owner-email="user@example.com"'}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [
|
||||
{
|
||||
"name": "Auto-Submitted",
|
||||
"value": 'auto-replied; owner-email="user@example.com"',
|
||||
}
|
||||
]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_auto_submitted_no_with_parameters(self):
|
||||
"""Auto-Submitted: no with parameters is not detected."""
|
||||
headers = {"Auto-Submitted": "no; some-param=value"}
|
||||
assert _is_auto_reply_message(headers) is False
|
||||
parsed_email = {
|
||||
"headers": [{"name": "Auto-Submitted", "value": "no; some-param=value"}]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is False
|
||||
|
||||
|
||||
class TestIsAutoReplyMessageExtended:
|
||||
@@ -165,48 +189,62 @@ class TestIsAutoReplyMessageExtended:
|
||||
|
||||
def test_return_path_null(self):
|
||||
"""Return-Path: <> (null sender) is detected."""
|
||||
headers = {"Return-Path": "<>"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {"headers": [{"name": "Return-Path", "value": "<>"}]}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_return_path_empty(self):
|
||||
"""Return-Path with empty value is detected."""
|
||||
headers = {"Return-Path": ""}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {"headers": [{"name": "Return-Path", "value": ""}]}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_list_post_header(self):
|
||||
"""List-Post header is detected."""
|
||||
headers = {"List-Post": "<mailto:list@example.com>"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [{"name": "List-Post", "value": "<mailto:list@example.com>"}]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_list_help_header(self):
|
||||
"""List-Help header is detected."""
|
||||
headers = {"List-Help": "<mailto:help@example.com>"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [{"name": "List-Help", "value": "<mailto:help@example.com>"}]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_list_subscribe_header(self):
|
||||
"""List-Subscribe header is detected."""
|
||||
headers = {"List-Subscribe": "<mailto:sub@example.com>"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [{"name": "List-Subscribe", "value": "<mailto:sub@example.com>"}]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_list_owner_header(self):
|
||||
"""List-Owner header is detected."""
|
||||
headers = {"List-Owner": "<mailto:owner@example.com>"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [{"name": "List-Owner", "value": "<mailto:owner@example.com>"}]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_list_archive_header(self):
|
||||
"""List-Archive header is detected."""
|
||||
headers = {"List-Archive": "<https://archive.example.com>"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [
|
||||
{"name": "List-Archive", "value": "<https://archive.example.com>"}
|
||||
]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_x_loop_header(self):
|
||||
"""X-Loop header is detected."""
|
||||
headers = {"X-Loop": "yes"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {"headers": [{"name": "X-Loop", "value": "yes"}]}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
def test_feedback_id_header(self):
|
||||
"""Feedback-ID header is detected (Gmail newsletters)."""
|
||||
headers = {"Feedback-ID": "123:campaign:gmail"}
|
||||
assert _is_auto_reply_message(headers) is True
|
||||
parsed_email = {
|
||||
"headers": [{"name": "Feedback-ID", "value": "123:campaign:gmail"}]
|
||||
}
|
||||
assert _is_auto_reply_message(parsed_email) is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -404,10 +442,10 @@ class TestShouldSendAutoreply:
|
||||
def test_eligible_message(self, mailbox, autoreply_template):
|
||||
"""Eligible message triggers autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "sender@example.com"},
|
||||
"from": [{"email": "sender@example.com"}],
|
||||
"to": [{"email": str(mailbox)}],
|
||||
"subject": "Hello",
|
||||
"headers": {},
|
||||
"headers": [],
|
||||
}
|
||||
result = should_send_autoreply(mailbox, parsed)
|
||||
assert result is not None
|
||||
@@ -416,33 +454,33 @@ class TestShouldSendAutoreply:
|
||||
def test_skip_spam(self, mailbox, autoreply_template):
|
||||
"""Spam messages do not trigger autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "sender@example.com"},
|
||||
"headers": {},
|
||||
"from": [{"email": "sender@example.com"}],
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed, is_spam=True) is None
|
||||
|
||||
def test_skip_auto_reply_message(self, mailbox, autoreply_template):
|
||||
"""Auto-reply messages do not trigger autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "sender@example.com"},
|
||||
"headers": {"Auto-Submitted": "auto-replied"},
|
||||
"from": [{"email": "sender@example.com"}],
|
||||
"headers": [{"name": "Auto-Submitted", "value": "auto-replied"}],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_skip_self_reply(self, mailbox, autoreply_template):
|
||||
"""Messages from the mailbox itself do not trigger autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": str(mailbox)},
|
||||
"headers": {},
|
||||
"from": [{"email": str(mailbox)}],
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_no_autoreply_template(self, mailbox):
|
||||
"""No autoreply template means no autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "sender@example.com"},
|
||||
"from": [{"email": "sender@example.com"}],
|
||||
"to": [{"email": str(mailbox)}],
|
||||
"headers": {},
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
@@ -456,9 +494,9 @@ class TestShouldSendAutoreply:
|
||||
}
|
||||
autoreply_template.save()
|
||||
parsed = {
|
||||
"from": {"email": "sender@example.com"},
|
||||
"from": [{"email": "sender@example.com"}],
|
||||
"to": [{"email": str(mailbox)}],
|
||||
"headers": {},
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
@@ -467,9 +505,9 @@ class TestShouldSendAutoreply:
|
||||
"""Rate-limited sender does not trigger autoreply."""
|
||||
# First call consumes the throttle allowance
|
||||
parsed = {
|
||||
"from": {"email": "sender@example.com"},
|
||||
"from": [{"email": "sender@example.com"}],
|
||||
"to": [{"email": str(mailbox)}],
|
||||
"headers": {},
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is not None
|
||||
# Second call is throttled
|
||||
@@ -478,82 +516,82 @@ class TestShouldSendAutoreply:
|
||||
def test_skip_noreply_sender(self, mailbox, autoreply_template):
|
||||
"""noreply@ sender does not trigger autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "noreply@example.com"},
|
||||
"headers": {},
|
||||
"from": [{"email": "noreply@example.com"}],
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_skip_mailer_daemon_sender(self, mailbox, autoreply_template):
|
||||
"""mailer-daemon@ sender does not trigger autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "mailer-daemon@example.com"},
|
||||
"headers": {},
|
||||
"from": [{"email": "mailer-daemon@example.com"}],
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_skip_postmaster_sender(self, mailbox, autoreply_template):
|
||||
"""postmaster@ sender does not trigger autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "postmaster@example.com"},
|
||||
"headers": {},
|
||||
"from": [{"email": "postmaster@example.com"}],
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_skip_bounce_sender(self, mailbox, autoreply_template):
|
||||
"""bounces-123@ sender does not trigger autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "bounces-123@example.com"},
|
||||
"headers": {},
|
||||
"from": [{"email": "bounces-123@example.com"}],
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_skip_owner_prefix_sender(self, mailbox, autoreply_template):
|
||||
"""owner-list@ sender does not trigger autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "owner-list@example.com"},
|
||||
"headers": {},
|
||||
"from": [{"email": "owner-list@example.com"}],
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_skip_missing_from(self, mailbox, autoreply_template):
|
||||
"""Missing 'from' key in parsed headers returns None."""
|
||||
parsed = {
|
||||
"headers": {},
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_skip_empty_sender_email(self, mailbox, autoreply_template):
|
||||
"""Empty sender email returns None."""
|
||||
parsed = {
|
||||
"from": {"email": ""},
|
||||
"headers": {},
|
||||
"from": [{"email": ""}],
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_case_insensitive_self_reply(self, mailbox, autoreply_template):
|
||||
"""Case-insensitive self-reply detection."""
|
||||
parsed = {
|
||||
"from": {"email": str(mailbox).upper()},
|
||||
"headers": {},
|
||||
"from": [{"email": str(mailbox).upper()}],
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_skip_bcc_recipient(self, mailbox, autoreply_template):
|
||||
"""Mailbox not in To/Cc/Bcc suppresses autoreply (RFC 5230 §4.5)."""
|
||||
parsed = {
|
||||
"from": {"email": "sender@example.com"},
|
||||
"from": [{"email": "sender@example.com"}],
|
||||
"to": [{"email": "someone-else@example.com"}],
|
||||
"headers": {},
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
def test_mailbox_in_cc_triggers(self, mailbox, autoreply_template):
|
||||
"""Mailbox in Cc still triggers autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "sender@example.com"},
|
||||
"from": [{"email": "sender@example.com"}],
|
||||
"to": [{"email": "someone-else@example.com"}],
|
||||
"cc": [{"email": str(mailbox)}],
|
||||
"headers": {},
|
||||
"headers": [],
|
||||
}
|
||||
result = should_send_autoreply(mailbox, parsed)
|
||||
assert result is not None
|
||||
@@ -561,8 +599,8 @@ class TestShouldSendAutoreply:
|
||||
def test_skip_no_recipients(self, mailbox, autoreply_template):
|
||||
"""No To/Cc/Bcc fields suppresses autoreply."""
|
||||
parsed = {
|
||||
"from": {"email": "sender@example.com"},
|
||||
"headers": {},
|
||||
"from": [{"email": "sender@example.com"}],
|
||||
"headers": [],
|
||||
}
|
||||
assert should_send_autoreply(mailbox, parsed) is None
|
||||
|
||||
@@ -686,9 +724,9 @@ class TestSendAutoreplyForMessage:
|
||||
):
|
||||
"""should_send_autoreply increments throttle, blocking subsequent calls."""
|
||||
parsed = {
|
||||
"from": {"email": inbound_message.sender.email},
|
||||
"from": [{"email": inbound_message.sender.email}],
|
||||
"to": [{"email": str(mailbox)}],
|
||||
"headers": {},
|
||||
"headers": [],
|
||||
}
|
||||
# First call succeeds and increments the throttle
|
||||
assert should_send_autoreply(mailbox, parsed) is not None
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user