diff --git a/.dockerignore b/.dockerignore index 8ff4689c..9fe9be60 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,7 +12,7 @@ venv # Docker compose.* -env.d +deploy/env # Docs docs diff --git a/.github/workflows/crowdin_download.yml b/.github/workflows/crowdin_download.yml deleted file mode 100644 index 7b934c08..00000000 --- a/.github/workflows/crowdin_download.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Download translations from Crowdin - -on: - workflow_dispatch: - push: - branches: - - 'release/**' - -jobs: - - synchronize-with-crowdin: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - name: Create env files - run: make create-env-files - # crowdin workflow - - name: crowdin action - uses: crowdin/github-action@v2 - with: - config: crowdin/config.yml - upload_sources: false - upload_translations: false - download_translations: true - create_pull_request: false - push_translations: false - push_sources: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # A numeric ID, found at https://crowdin.com/project//tools/api - CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} - - # Visit https://crowdin.com/settings#api-key to create this token - CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} - - CROWDIN_BASE_PATH: "../src/" - # Create a new PR - - name: Create a new Pull Request with new translated strings - uses: peter-evans/create-pull-request@v7 - with: - commit-message: | - 🌐(i18n) update translated strings - - Update translated files with new translations - title: 🌐(i18n) update translated strings - body: | - ## Purpose - - update translated strings - - ## Proposal - - - [x] update translated strings - branch: i18n/update-translations - labels: i18n diff --git a/.github/workflows/messages-ghcr.yml b/.github/workflows/messages-ghcr.yml index fce80eb1..dd187700 100644 --- a/.github/workflows/messages-ghcr.yml +++ b/.github/workflows/messages-ghcr.yml @@ -10,7 +10,7 @@ name: Build and publish OCI images jobs: # Shared base (debian-trixie + uv + managed CPython 3.14.6), built + pushed once - # from docker/python-uv/Dockerfile, then referenced by digest via + # from deploy/python-uv/Dockerfile, then referenced by digest via # PYTHON_UV_IMAGE in the backend and MTA builds below. docker-publish-python-uv: uses: ./.github/workflows/docker-publish.yml @@ -22,7 +22,7 @@ jobs: secrets: inherit with: image_name: "python-uv" - context: "docker/python-uv" + context: "deploy/python-uv" docker-publish-mta-in: needs: docker-publish-python-uv diff --git a/.gitignore b/.gitignore index cb7dbaa3..e642bbac 100644 --- a/.gitignore +++ b/.gitignore @@ -38,9 +38,9 @@ venv/ ENV/ env.bak/ venv.bak/ -env.d/development/* -!env.d/development/*.defaults -!env.d/development/*.e2e +deploy/env/* +!deploy/env/*.defaults +!deploy/env/*.e2e env.d/terraform # npm diff --git a/Makefile b/Makefile index 400951f7..b609ea51 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,11 @@ DOCKER_UID = $(shell id -u) DOCKER_GID = $(shell id -g) DOCKER_USER = $(DOCKER_UID):$(DOCKER_GID) COMPOSE = DOCKER_USER=$(DOCKER_USER) docker compose -# Shared base image (docker/python-uv) that the backend + MTA Dockerfiles inherit -# from via the PYTHON_UV_IMAGE build arg. Overridable for CI. +# Local tag for the shared base image (deploy/python-uv). `build-python-base` +# builds it, and the Dockerfiles default `FROM ${PYTHON_UV_IMAGE}` to this same +# tag, so compose builds resolve it. (CI publishes/overrides the base via the +# docker-publish workflow build-args, not this variable — overriding it here +# would only retag the local build, leaving the compose `FROM` unchanged.) PYTHON_UV_IMAGE ?= messages-python-uv:local COMPOSE_E2E = DOCKER_USER=$(DOCKER_USER) docker compose -f src/e2e/compose.yaml COMPOSE_EXEC = $(COMPOSE) exec @@ -57,15 +60,15 @@ data/static: create-env-files: ## Create empty .local env files for local development create-env-files: \ - env.d/development/crowdin.local \ - env.d/development/postgresql.local \ - env.d/development/keycloak.local \ - env.d/development/backend.local \ - env.d/development/frontend.local \ - env.d/development/mta-in.local \ - env.d/development/mta-in-py.local \ - env.d/development/mta-out.local \ - env.d/development/socks-proxy.local + deploy/env/crowdin.local \ + deploy/env/postgresql.local \ + deploy/env/keycloak.local \ + deploy/env/backend.local \ + deploy/env/frontend.local \ + deploy/env/mta-in.local \ + deploy/env/mta-in-py.local \ + deploy/env/mta-out.local \ + deploy/env/socks-proxy.local .PHONY: create-env-files bootstrap: ## Prepare the project for local development @@ -89,6 +92,7 @@ bootstrap: ## Prepare the project for local development @echo "$(RESET)" @echo "$(GREEN)Starting bootstrap process...$(RESET)" @echo "" + @$(MAKE) create-env-files @$(MAKE) start-deps @$(MAKE) update @$(MAKE) superuser @@ -106,6 +110,7 @@ bootstrap: ## Prepare the project for local development bootstrap-full: ## Prepare the project for local development with the full stack @echo "$(GREEN)Starting full bootstrap process...$(RESET)" @echo "" + @$(MAKE) create-env-files @$(MAKE) start-deps @$(MAKE) update-full @$(MAKE) superuser @@ -168,15 +173,25 @@ logs: ## display all services logs (follow mode) @$(COMPOSE) logs -f .PHONY: logs -build-python-base: ## build the shared python+uv base image (docker/python-uv) that the backend and MTA images inherit from - @docker build -t $(PYTHON_UV_IMAGE) docker/python-uv +build-python-base: ## build the shared python+uv base image (deploy/python-uv) that the backend and MTA images inherit from + @docker build -t $(PYTHON_UV_IMAGE) deploy/python-uv .PHONY: build-python-base start-deps: ## start the slow infra deps (postgres, redis, keycloak) in the background so they warm up while the rest of bootstrap runs @$(COMPOSE) up -d --no-recreate postgresql redis keycloak .PHONY: start-deps -start: build-python-base ## start the light dev stack (backend, worker, frontend, keycloak, postgresql, redis) +# Fail fast (before booting a broken stack) when the project has not been +# bootstrapped: `make bootstrap` creates the gitignored env files and the +# frontend node_modules volume. start/start-full depend on this. +check-bootstrapped: + @test -f deploy/env/backend.local || { \ + printf "\n$(BOLD)✗ Not bootstrapped$(RESET): env files are missing.\n Run $(BOLD)make bootstrap$(RESET) first.\n\n" >&2; exit 1; } + @docker volume inspect st-messages_frontend-node-modules >/dev/null 2>&1 || { \ + printf "\n$(BOLD)✗ Not bootstrapped$(RESET): frontend dependencies are not installed.\n Run $(BOLD)make bootstrap$(RESET) first.\n\n" >&2; exit 1; } +.PHONY: check-bootstrapped + +start: check-bootstrapped build-python-base ## start the light dev stack (backend, worker, frontend, keycloak, postgresql, redis) @$(COMPOSE) stop backend-dev worker-dev worker-ui opensearch objectstorage mailcatcher mta-in-py mpa >/dev/null 2>&1 || true @$(COMPOSE) up --build -d --wait \ postgresql \ @@ -187,7 +202,7 @@ start: build-python-base ## start the light dev stack (backend, worker, frontend worker-dev-light .PHONY: start -start-full: build-python-base ## start the full dev stack (adds OpenSearch, object storage, mailcatcher and the MTAs) +start-full: check-bootstrapped build-python-base ## start the full dev stack (adds OpenSearch, object storage, mailcatcher and the MTAs) @$(COMPOSE) stop backend-dev-light worker-dev-light >/dev/null 2>&1 || true @$(COMPOSE) up --build -d --wait \ postgresql \ @@ -365,25 +380,25 @@ test-mta-out: build-python-base ## run the mta-out tests @$(COMPOSE) run --build --rm mta-out-test .PHONY: test-mta-out -test-mpa: ## run the mpa tests +test-mpa: build-python-base ## run the mpa tests @$(COMPOSE) run --build --rm mpa-test .PHONY: test-mpa -test-jmap-email: ## run the jmap-email package tests (zero infrastructure deps) +test-jmap-email: build-python-base ## run the jmap-email package tests (zero infrastructure deps) @$(COMPOSE) run --build --rm jmap-email-test .PHONY: test-jmap-email -fuzz-jmap-email: ## run the jmap-email Hypothesis fuzz suite +fuzz-jmap-email: build-python-base ## run the jmap-email Hypothesis fuzz suite @$(COMPOSE) run --build --rm jmap-email-test pytest -m fuzz tests/ .PHONY: fuzz-jmap-email -lint-jmap-email: ## lint the jmap-email library (ruff check + format check + pylint) +lint-jmap-email: build-python-base ## lint the jmap-email library (ruff check + format check + pylint) @$(COMPOSE) run --build --rm --entrypoint ruff jmap-email-test check jmap_email tests @$(COMPOSE) run --build --rm --entrypoint ruff jmap-email-test format --check jmap_email tests @$(COMPOSE) run --build --rm --entrypoint pylint jmap-email-test jmap_email tests .PHONY: lint-jmap-email -typecheck-jmap-email: ## type-check the jmap-email library with ty (Astral, Rust) +typecheck-jmap-email: build-python-base ## type-check the jmap-email library with ty (Astral, Rust) @$(COMPOSE) run --build --rm --entrypoint ty jmap-email-test check .PHONY: typecheck-jmap-email @@ -391,7 +406,7 @@ release-jmap-email: ## publish jmap-email to PyPI (interactive: TestPyPI → smo @bin/release-jmap-email.sh .PHONY: release-jmap-email -test-socks-proxy: ## run the socks-proxy tests +test-socks-proxy: build-python-base ## run the socks-proxy tests @$(COMPOSE) run --build --rm socks-proxy-test .PHONY: test-socks-proxy @@ -574,7 +589,7 @@ reset-db-full: build ## flush database, including schema $(MANAGE_DB) migrate .PHONY: reset-db-full -env.d/development/%.local: +deploy/env/%.local: @echo "# Local development overrides for $(notdir $*)" > $@ @echo "# Add your local-specific environment variables below:" >> $@ @echo "# Example: DJANGO_DEBUG=True" >> $@ diff --git a/Procfile b/Procfile index 0f30abfa..f867d8e7 100644 --- a/Procfile +++ b/Procfile @@ -1,4 +1,4 @@ -web: bin/scalingo_run_web +web: deploy/paas/scalingo_run_web workerall: python worker.py workerimports: python worker.py --concurrency=1 --queues=imports --disable-scheduler workerreindex: python worker.py --concurrency=2 --queues=reindex --disable-scheduler diff --git a/bin/release-jmap-email.sh b/bin/release-jmap-email.sh old mode 100644 new mode 100755 diff --git a/bin/scalingo_postcompile b/bin/scalingo_postcompile old mode 100644 new mode 100755 index f61bfa29..4fe593c4 --- a/bin/scalingo_postcompile +++ b/bin/scalingo_postcompile @@ -1,11 +1,6 @@ #!/bin/bash - -set -o errexit # always exit on error -set -o pipefail # don't ignore exit codes when piping output - -echo "-----> Running post-compile script" - -# Remove all the files we don't need -rm -rf src docker env.d .cursor .github compose.yaml README.md .cache - -chmod +x bin/scalingo_run_web \ No newline at end of file +# DEPRECATED shim — the Scalingo scripts moved to deploy/paas/. This passthrough +# keeps external Scalingo hooks that still reference bin/scalingo_postcompile working; update the +# hook to deploy/paas/scalingo_postcompile and delete this shim. +echo "⚠️ bin/scalingo_postcompile is DEPRECATED → use deploy/paas/scalingo_postcompile (update your Scalingo hook)" >&2 +exec bash "$(dirname "$0")/../deploy/paas/scalingo_postcompile" "$@" diff --git a/bin/scalingo_postfrontend b/bin/scalingo_postfrontend old mode 100644 new mode 100755 index a72d9d98..56ab081e --- a/bin/scalingo_postfrontend +++ b/bin/scalingo_postfrontend @@ -1,23 +1,6 @@ #!/bin/bash - -set -o errexit # always exit on error -set -o pipefail # don't ignore exit codes when piping output - -echo "-----> Running post-frontend script" - -# Move the frontend build to the app root and clean up -mkdir -p build/ -mv src/frontend/dist build/frontend-out - -mv src/backend/* ./ -mkdir -p messages_backend && touch messages_backend/__init__.py - -# Download Caddy binary -CADDY_VERSION="2.9.1" -curl -fsSL "https://github.com/caddyserver/caddy/releases/download/v${CADDY_VERSION}/caddy_${CADDY_VERSION}_linux_amd64.tar.gz" | tar -xz -C bin/ caddy -chmod +x bin/caddy - -# Copy Caddyfile (uses {$ENV} vars natively, no ERB needed) -cp src/frontend/caddy/Caddyfile ./Caddyfile - -echo "3.14" > .python-version +# DEPRECATED shim — the Scalingo scripts moved to deploy/paas/. This passthrough +# keeps external Scalingo hooks that still reference bin/scalingo_postfrontend working; update the +# hook to deploy/paas/scalingo_postfrontend and delete this shim. +echo "⚠️ bin/scalingo_postfrontend is DEPRECATED → use deploy/paas/scalingo_postfrontend (update your Scalingo hook)" >&2 +exec bash "$(dirname "$0")/../deploy/paas/scalingo_postfrontend" "$@" diff --git a/compose.yaml b/compose.yaml index 23c43c46..4894e3b8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -24,8 +24,8 @@ x-worker-base: &worker-base ulimits: *nofile-ulimits command: ["python", "worker.py", "--loglevel=DEBUG"] env_file: - - env.d/development/backend.defaults - - env.d/development/backend.local + - deploy/env/backend.defaults + - deploy/env/backend.local volumes: - ./src/backend:/app - ./data/static:/data/static @@ -42,8 +42,8 @@ services: timeout: 2s retries: 300 env_file: - - env.d/development/postgresql.defaults - - env.d/development/postgresql.local + - deploy/env/postgresql.defaults + - deploy/env/postgresql.local redis: image: redis:5 @@ -133,8 +133,8 @@ services: # healthcheck below can tell when Keycloak is actually serving. - --health-enabled=true env_file: - - env.d/development/keycloak.defaults - - env.d/development/keycloak.local + - deploy/env/keycloak.defaults + - deploy/env/keycloak.local ports: - "8902:8802" # Without a healthcheck, `--wait` / `depends_on: service_started` consider @@ -192,8 +192,8 @@ services: - PYLINTHOME=/app/.pylint.d - DJANGO_CONFIGURATION=Development env_file: - - env.d/development/backend.defaults - - env.d/development/backend.local + - deploy/env/backend.defaults + - deploy/env/backend.local ports: - "8901:8000" depends_on: @@ -230,8 +230,8 @@ services: # start-full feature). - OPENSEARCH_INDEX_THREADS=False env_file: - - env.d/development/backend.defaults - - env.d/development/backend.local + - deploy/env/backend.defaults + - deploy/env/backend.local ports: - "8901:8000" depends_on: @@ -249,8 +249,8 @@ services: environment: - DJANGO_CONFIGURATION=DevelopmentMinimal env_file: - - env.d/development/backend.defaults - - env.d/development/backend.local + - deploy/env/backend.defaults + - deploy/env/backend.local ports: - "8901:8000" depends_on: @@ -306,8 +306,8 @@ services: - FLOWER_UNAUTHENTICATED_API=true - DJANGO_CONFIGURATION=Development env_file: - - env.d/development/backend.defaults - - env.d/development/backend.local + - deploy/env/backend.defaults + - deploy/env/backend.local volumes: - ./src/backend:/app ports: @@ -328,8 +328,8 @@ services: frontend-dev: extends: frontend-base env_file: - - env.d/development/frontend.defaults - - env.d/development/frontend.local + - deploy/env/frontend.defaults + - deploy/env/frontend.local command: ["npm", "run", "dev"] volumes: - ./src/frontend/:/home/frontend/ @@ -363,8 +363,8 @@ services: volumes: - ".:/app" env_file: - - env.d/development/crowdin.defaults - - env.d/development/crowdin.local + - deploy/env/crowdin.defaults + - deploy/env/crowdin.local user: "${DOCKER_USER:-1000}" working_dir: /app @@ -373,8 +373,8 @@ services: context: src/mta-in target: runtime-prod env_file: - - env.d/development/mta-in.defaults - - env.d/development/mta-in.local + - deploy/env/mta-in.defaults + - deploy/env/mta-in.local ports: - "8910:25" depends_on: @@ -387,8 +387,8 @@ services: context: src/mta-in target: runtime-dev env_file: - - env.d/development/mta-in.defaults - - env.d/development/mta-in.local + - deploy/env/mta-in.defaults + - deploy/env/mta-in.local environment: - EXEC_CMD=true - MDA_API_BASE_URL=http://localhost:8000/api/mail/ @@ -424,10 +424,10 @@ services: DOCKER_USER: ${DOCKER_USER:-65532} user: ${DOCKER_USER:-65532} env_file: - - env.d/development/mta-in.defaults - - env.d/development/mta-in.local - - env.d/development/mta-in-py.defaults - - env.d/development/mta-in-py.local + - deploy/env/mta-in.defaults + - deploy/env/mta-in.local + - deploy/env/mta-in-py.defaults + - deploy/env/mta-in-py.local ports: - "8920:25" - "9120:9100" # Prometheus metrics @@ -459,10 +459,10 @@ services: security_opt: - no-new-privileges:true env_file: - - env.d/development/mta-in.defaults - - env.d/development/mta-in.local - - env.d/development/mta-in-py.defaults - - env.d/development/mta-in-py.local + - deploy/env/mta-in.defaults + - deploy/env/mta-in.local + - deploy/env/mta-in-py.defaults + - deploy/env/mta-in-py.local environment: - EXEC_CMD=true - MDA_API_BASE_URL=http://localhost:8000/api/mail/ @@ -479,8 +479,8 @@ services: context: src/mta-out target: runtime-prod env_file: - - env.d/development/mta-out.defaults - - env.d/development/mta-out.local + - deploy/env/mta-out.defaults + - deploy/env/mta-out.local ports: - "8911:587" depends_on: @@ -494,8 +494,8 @@ services: context: src/mta-out target: runtime-dev env_file: - - env.d/development/mta-out.defaults - - env.d/development/mta-out.local + - deploy/env/mta-out.defaults + - deploy/env/mta-out.local environment: - EXEC_CMD=true - MTA_OUT_SMTP_HOST=localhost:587 @@ -521,8 +521,8 @@ services: context: src/socks-proxy target: runtime env_file: - - env.d/development/socks-proxy.defaults - - env.d/development/socks-proxy.local + - deploy/env/socks-proxy.defaults + - deploy/env/socks-proxy.local ports: - "8916:1080" diff --git a/env.d/development/backend.defaults b/deploy/env/backend.defaults similarity index 100% rename from env.d/development/backend.defaults rename to deploy/env/backend.defaults diff --git a/env.d/development/backend.e2e b/deploy/env/backend.e2e similarity index 100% rename from env.d/development/backend.e2e rename to deploy/env/backend.e2e diff --git a/env.d/development/crowdin.defaults b/deploy/env/crowdin.defaults similarity index 100% rename from env.d/development/crowdin.defaults rename to deploy/env/crowdin.defaults diff --git a/env.d/development/frontend.defaults b/deploy/env/frontend.defaults similarity index 100% rename from env.d/development/frontend.defaults rename to deploy/env/frontend.defaults diff --git a/env.d/development/frontend.e2e b/deploy/env/frontend.e2e similarity index 100% rename from env.d/development/frontend.e2e rename to deploy/env/frontend.e2e diff --git a/env.d/development/keycloak.defaults b/deploy/env/keycloak.defaults similarity index 100% rename from env.d/development/keycloak.defaults rename to deploy/env/keycloak.defaults diff --git a/env.d/development/keycloak.e2e b/deploy/env/keycloak.e2e similarity index 100% rename from env.d/development/keycloak.e2e rename to deploy/env/keycloak.e2e diff --git a/env.d/development/mta-in-py.defaults b/deploy/env/mta-in-py.defaults similarity index 100% rename from env.d/development/mta-in-py.defaults rename to deploy/env/mta-in-py.defaults diff --git a/env.d/development/mta-in.defaults b/deploy/env/mta-in.defaults similarity index 100% rename from env.d/development/mta-in.defaults rename to deploy/env/mta-in.defaults diff --git a/env.d/development/mta-in.e2e b/deploy/env/mta-in.e2e similarity index 100% rename from env.d/development/mta-in.e2e rename to deploy/env/mta-in.e2e diff --git a/env.d/development/mta-out.defaults b/deploy/env/mta-out.defaults similarity index 100% rename from env.d/development/mta-out.defaults rename to deploy/env/mta-out.defaults diff --git a/env.d/development/postgresql.defaults b/deploy/env/postgresql.defaults similarity index 100% rename from env.d/development/postgresql.defaults rename to deploy/env/postgresql.defaults diff --git a/env.d/development/socks-proxy.defaults b/deploy/env/socks-proxy.defaults similarity index 100% rename from env.d/development/socks-proxy.defaults rename to deploy/env/socks-proxy.defaults diff --git a/deploy/paas/scalingo_postcompile b/deploy/paas/scalingo_postcompile new file mode 100644 index 00000000..f246c358 --- /dev/null +++ b/deploy/paas/scalingo_postcompile @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o errexit # always exit on error +set -o pipefail # don't ignore exit codes when piping output + +echo "-----> Running post-compile script" + +# Remove all the files we don't need +rm -rf src deploy/python-uv deploy/env .cursor .github compose.yaml README.md .cache + +chmod +x deploy/paas/scalingo_run_web \ No newline at end of file diff --git a/deploy/paas/scalingo_postfrontend b/deploy/paas/scalingo_postfrontend new file mode 100644 index 00000000..a72d9d98 --- /dev/null +++ b/deploy/paas/scalingo_postfrontend @@ -0,0 +1,23 @@ +#!/bin/bash + +set -o errexit # always exit on error +set -o pipefail # don't ignore exit codes when piping output + +echo "-----> Running post-frontend script" + +# Move the frontend build to the app root and clean up +mkdir -p build/ +mv src/frontend/dist build/frontend-out + +mv src/backend/* ./ +mkdir -p messages_backend && touch messages_backend/__init__.py + +# Download Caddy binary +CADDY_VERSION="2.9.1" +curl -fsSL "https://github.com/caddyserver/caddy/releases/download/v${CADDY_VERSION}/caddy_${CADDY_VERSION}_linux_amd64.tar.gz" | tar -xz -C bin/ caddy +chmod +x bin/caddy + +# Copy Caddyfile (uses {$ENV} vars natively, no ERB needed) +cp src/frontend/caddy/Caddyfile ./Caddyfile + +echo "3.14" > .python-version diff --git a/bin/scalingo_run_web b/deploy/paas/scalingo_run_web similarity index 100% rename from bin/scalingo_run_web rename to deploy/paas/scalingo_run_web diff --git a/docker/python-uv/Dockerfile b/deploy/python-uv/Dockerfile similarity index 100% rename from docker/python-uv/Dockerfile rename to deploy/python-uv/Dockerfile diff --git a/docker/python-uv/strip-python.sh b/deploy/python-uv/strip-python.sh similarity index 100% rename from docker/python-uv/strip-python.sh rename to deploy/python-uv/strip-python.sh diff --git a/docs/env.md b/docs/env.md index f9f4743e..87d17035 100644 --- a/docs/env.md +++ b/docs/env.md @@ -539,7 +539,7 @@ _Set only to receive a startup deprecation warning; otherwise ignored._ ## Environment Files -The application uses environment files located in `env.d/development/` for different services: +The application uses environment files located in `deploy/env/` for different services: - `backend.defaults` - Main Django application settings - `common.defaults` - Shared settings across services diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 7c6e2ec8..a9709609 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -27,7 +27,7 @@ Messages supports multiple deployment strategies depending on your infrastructur **Process**: 1. Start from the `compose.yaml` in the repository -2. Create production environment files (`env.d/production/*.defaults`) +2. Create production environment files (`deploy/env/production/*.defaults`) 3. Deploy to any environment where Docker Compose runs 4. Configure DNS and SSL certificates @@ -114,24 +114,24 @@ The DNS records for each customer domains are available either via API at http:/ Messages uses environment variables as the primary configuration method: **Environment File Structure:** -- `env.d/production/backend.defaults` - Main Django application settings -- `env.d/production/frontend.defaults` - Frontend configuration -- `env.d/production/mta-in.defaults` - Inbound mail server settings -- `env.d/production/mta-out.defaults` - Outbound mail server settings -- `env.d/production/postgresql.defaults` - Database configuration -- `env.d/production/keycloak.defaults` - Identity provider settings +- `deploy/env/production/backend.defaults` - Main Django application settings +- `deploy/env/production/frontend.defaults` - Frontend configuration +- `deploy/env/production/mta-in.defaults` - Inbound mail server settings +- `deploy/env/production/mta-out.defaults` - Outbound mail server settings +- `deploy/env/production/postgresql.defaults` - Database configuration +- `deploy/env/production/keycloak.defaults` - Identity provider settings **For detailed environment variable documentation, see [Environment Variables](./env.md).** ### 3. MTA Configuration #### MTA-in (Inbound Email) -- Configured via `env.d/production/mta-in.defaults` +- Configured via `deploy/env/production/mta-in.defaults` - Uses custom milter for synchronous delivery during SMTP sessions - Validates recipients via REST API before accepting messages #### MTA-out (Outbound Email) -- Configured via `env.d/production/mta-out.defaults` +- Configured via `deploy/env/production/mta-out.defaults` - Supports relay configuration for external SMTP providers - Requires TLS certificates for production @@ -180,12 +180,12 @@ Messages uses OpenID Connect (OIDC) for user authentication. This is the only au - Keycloak is included in the default Docker Compose setup - Pre-configured with Messages realm and users - Suitable for organizations wanting a self-hosted identity provider - - Configure via `env.d/production/keycloak.defaults` + - Configure via `deploy/env/production/keycloak.defaults` 2. **External OIDC Provider** - Use any OIDC-compliant identity provider - Examples: Auth0, Okta, Azure AD, Google Workspace - - Configure via `env.d/production/backend.defaults` + - Configure via `deploy/env/production/backend.defaults` - Requires proper OIDC endpoint configuration **User Management:** @@ -197,7 +197,7 @@ Messages uses OpenID Connect (OIDC) for user authentication. This is the only au For production deployment, create your own Docker Compose configuration based on `compose.yaml`: **Key Considerations:** -- Use production environment files (`env.d/production/*.defaults`) +- Use production environment files (`deploy/env/production/*.defaults`) - Configure SSL/TLS certificates - Set up persistent volumes for databases - Implement proper restart policies diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index f8330fb0..a606d8a7 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -1,4 +1,4 @@ -# Shared base image (docker/python-uv/Dockerfile): debian-trixie + apt upgrade + +# Shared base image (deploy/python-uv/Dockerfile): debian-trixie + apt upgrade + # ca-certificates + uv 0.11.28 (digest-pinned) + uv-managed CPython 3.14.6 at # /opt/python. Global ARG so it can be used in `FROM` below. Built by # `make build-python-base`; overridden in CI. @@ -148,7 +148,7 @@ RUN mkdir -p /shared-libs/usr/lib /shared-libs/usr/share/misc && \ cp /usr/share/misc/magic.mgc /shared-libs/usr/share/misc/ -# ---- Strip Python for the distroless image (see docker/python-uv/strip-python.sh) ---- +# ---- Strip Python for the distroless image (see deploy/python-uv/strip-python.sh) ---- # Only runtime-distroless-prod (the real production target) uses this. The slim # runtime-prod inherits the full managed Python from the shared base instead. FROM ${PYTHON_UV_IMAGE} AS python-stripped-runtime diff --git a/src/backend/entrypoint b/src/backend/entrypoint old mode 100755 new mode 100644 index d3e8cf19..d5e18d18 --- a/src/backend/entrypoint +++ b/src/backend/entrypoint @@ -17,11 +17,11 @@ # # or define new variables in an environment file to use with docker or docker compose: # -# # env.d/production +# # deploy/env # USER_NAME=foo # HOME=/home/foo # -# docker run --rm --env-file env.d/production st-messages-backend:latest python manage.py migrate +# docker run --rm --env-file deploy/env st-messages-backend:latest python manage.py migrate # echo "🐳(entrypoint) creating user running in the container..." diff --git a/src/backend/messages/settings.py b/src/backend/messages/settings.py index 7f697e7e..a3fb679d 100644 --- a/src/backend/messages/settings.py +++ b/src/backend/messages/settings.py @@ -371,7 +371,7 @@ class Base(Configuration): # No default on purpose: the MTA-to-MDA channel is authenticated solely by # an HS256 JWT signed with this shared secret, so a hardcoded fallback would # be internet-spoofable in production. The development value is supplied via - # env.d/development/backend.defaults (and the matching mta-in.defaults), the + # deploy/env/backend.defaults (and the matching mta-in.defaults), the # same way DJANGO_SECRET_KEY is handled. Unset → None → all MTA auth fails # closed. MDA_API_SECRET = values.Value( diff --git a/src/e2e/.npmrc b/src/e2e/.npmrc new file mode 100644 index 00000000..e7721991 --- /dev/null +++ b/src/e2e/.npmrc @@ -0,0 +1,5 @@ +# Supply-chain cooldown: only resolve package versions that have been public for +# at least this many DAYS (npm >= 11.16). This image runs `npm install`, so the +# cooldown actively gates the Playwright dependency tree at build time. Override +# a deliberately-fresh pin for a single run with `npm install --min-release-age=0`. +min-release-age=1 diff --git a/src/e2e/Dockerfile b/src/e2e/Dockerfile index cbfa6e62..4ff2b205 100644 --- a/src/e2e/Dockerfile +++ b/src/e2e/Dockerfile @@ -22,7 +22,7 @@ WORKDIR /app # Install deps and run all npm commands as the user running the container -COPY package*.json ./ +COPY package*.json .npmrc ./ # Install dependencies and postinstall playwright with deps RUN npm install diff --git a/src/e2e/README.md b/src/e2e/README.md index 4ded0536..e8ca8a75 100644 --- a/src/e2e/README.md +++ b/src/e2e/README.md @@ -45,7 +45,7 @@ Caddy is used as a reverse proxy to serve the frontend and the backend on the sa ### Environment variables -E2E configuration files are located in `env.d/development/*.e2e`: +E2E configuration files are located in `deploy/env/*.e2e`: - `backend.e2e`: Backend configuration for tests - `frontend.e2e`: Frontend configuration for tests - `keycloak.e2e`: Keycloak configuration for tests diff --git a/src/e2e/compose.yaml b/src/e2e/compose.yaml index f748e5a1..71466968 100644 --- a/src/e2e/compose.yaml +++ b/src/e2e/compose.yaml @@ -20,8 +20,8 @@ services: environment: - DJANGO_CONFIGURATION=E2E env_file: - - ../../env.d/development/backend.defaults - - ../../env.d/development/backend.e2e + - ../../deploy/env/backend.defaults + - ../../deploy/env/backend.e2e depends_on: !override - backend @@ -55,8 +55,8 @@ services: - --hostname-admin=$${ADMIN_HOST} - --http-port=8802 env_file: - - ../../env.d/development/keycloak.defaults - - ../../env.d/development/keycloak.e2e + - ../../deploy/env/keycloak.defaults + - ../../deploy/env/keycloak.e2e ports: !reset [] depends_on: !override - postgresql @@ -68,8 +68,8 @@ services: environment: - DJANGO_CONFIGURATION=E2E env_file: - - ../../env.d/development/backend.defaults - - ../../env.d/development/backend.e2e + - ../../deploy/env/backend.defaults + - ../../deploy/env/backend.e2e depends_on: !override postgresql: condition: service_healthy @@ -87,8 +87,8 @@ services: file: ../../compose.yaml service: mta-in env_file: - - ../../env.d/development/mta-in.defaults - - ../../env.d/development/mta-in.e2e + - ../../deploy/env/mta-in.defaults + - ../../deploy/env/mta-in.e2e ports: !reset [] depends_on: !override - backend @@ -128,8 +128,8 @@ services: service: frontend-base command: ["npm", "run", "dev"] env_file: - - ../../env.d/development/frontend.defaults - - ../../env.d/development/frontend.e2e + - ../../deploy/env/frontend.defaults + - ../../deploy/env/frontend.e2e volumes: - ../frontend/:/home/frontend/ ports: !reset [] @@ -159,7 +159,7 @@ services: volumes: - ./src:/app/src - ./playwright.config.ts:/app/playwright.config.ts - - ../../env.d/development:/app/env.d/development:ro + - ../../deploy/env:/app/deploy/env:ro depends_on: docker-sock-proxy: condition: service_started diff --git a/src/frontend/.npmrc b/src/frontend/.npmrc index 1b4ab6be..9a36fc84 100644 --- a/src/frontend/.npmrc +++ b/src/frontend/.npmrc @@ -3,3 +3,10 @@ # copy per version (a `.store/` + symlinks), shrinking node_modules ~3x and # cutting install time from minutes to ~20s on a fast filesystem. install-strategy=linked + +# Supply-chain cooldown: only resolve package versions that have been public for +# at least this many DAYS (npm >= 11.16). Blunts "publish malware, get pulled in +# instantly" attacks by refusing brand-new releases when the tree is (re)resolved +# at `npm install` time. Frozen `npm ci` installs the already-vetted lockfile as +# is. Complements the check:deps guardrail (scripts/check-node-modules.mjs). +min-release-age=1 diff --git a/src/frontend/Dockerfile b/src/frontend/Dockerfile index fa7c9de3..abbc3d82 100644 --- a/src/frontend/Dockerfile +++ b/src/frontend/Dockerfile @@ -1,6 +1,6 @@ ARG FRONTEND_IMAGE=frontend-build-output -FROM node:24.18.0-alpine@sha256:a0b9bf06e4e6193cf7a0f58816cc935ff8c2a908f81e6f1a95432d679c54fbfd AS frontend-dev-base +FROM node:24.18.0-slim@sha256:cb4e8f7c443347358b7875e717c29e27bf9befc8f5a26cf18af3c3dec80e58c5 AS frontend-dev-base ENV npm_config_cache=/tmp/npm-cache diff --git a/src/jmap-email/Dockerfile b/src/jmap-email/Dockerfile index b0bc5d9b..6a8e6421 100644 --- a/src/jmap-email/Dockerfile +++ b/src/jmap-email/Dockerfile @@ -1,14 +1,21 @@ -# Minimal test image for the jmap-email package. -# Zero non-stdlib runtime deps; only pytest + hypothesis for the suite, -# plus ``ty`` for static type-checking. -FROM python:3.14.6-slim@sha256:b877e50bd90de10af8d82c57a022fc2e0dc731c5320d762a27986facfc3355c1 +# Test image for the jmap-email package. Inherits the shared python-uv base +# (managed CPython 3.14.6 + uv). NOTE: the PyPI *release* still builds on the +# official python:3.14.6-slim image via bin/release-jmap-email.sh — that is the +# release-parity environment; this image is only for CI lint/type/test. +# Zero non-stdlib runtime deps; only pytest + hypothesis for the suite, plus +# ``ty``/``ruff``/``pylint`` for type-checking and linting. +ARG PYTHON_UV_IMAGE=messages-python-uv:local +FROM ${PYTHON_UV_IMAGE} WORKDIR /app -# ``ty`` ships as a single Rust binary on PyPI — no Node, no native -# libs, no bootstrap step. Pin pytest / hypothesis / ty so the image -# is deterministic; ``--no-cache-dir`` keeps the layer small. -RUN pip install --no-cache-dir \ +# The shared base ships uv (no bundled pip on PATH); create a venv on the managed +# Python and install the pinned dev tools into it. ``ty`` ships as a single Rust +# binary on PyPI — no Node, no native libs. +ENV VIRTUAL_ENV=/venv +ENV PATH="/venv/bin:$PATH" +RUN uv venv /venv \ + && uv pip install --no-cache \ "pytest==9.0.2" \ "pytest-cov==7.0.0" \ "pytest-xdist==3.8.0" \ @@ -22,10 +29,9 @@ COPY pyproject.toml README.md LICENSE CHANGELOG.md ./ COPY jmap_email ./jmap_email COPY tests ./tests -# Install the package itself (no runtime deps; zero new packages -# resolved). Editable so that the mount-overlay in development -# instantly picks up source edits. -RUN pip install --no-cache-dir -e . +# Install the package itself (no runtime deps; zero new packages resolved). +# Editable so that the mount-overlay in development instantly picks up edits. +RUN uv pip install --no-cache -e . # Smoke import on build to catch package-layout breakage early. RUN python -c "import jmap_email; print('jmap-email', jmap_email.__version__)" diff --git a/src/mpa/rspamd/Dockerfile b/src/mpa/rspamd/Dockerfile index 0d61dc68..b89720de 100644 --- a/src/mpa/rspamd/Dockerfile +++ b/src/mpa/rspamd/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:12.10-slim@sha256:4b50eb66f977b4062683ff434ef18ac191da862dbe966961bc11990cf5791a8d +FROM debian:trixie-slim@sha256:28de0877c2189802884ccd20f15ee41c203573bd87bb6b883f5f46362d24c5c2 RUN apt-get update && \ apt-get install -y lsb-release wget gpg ruby nginx netcat-openbsd procps && \ diff --git a/src/mpa/tests/Dockerfile b/src/mpa/tests/Dockerfile index d08dd953..f476c417 100644 --- a/src/mpa/tests/Dockerfile +++ b/src/mpa/tests/Dockerfile @@ -1,8 +1,14 @@ -FROM python:3.13-slim@sha256:eb43ff125d8d58d7449dcba7d336c23bcac412f526d861db493b9994d8010280 +# Test harness for the rspamd MPA. Inherits the shared python-uv base +# (managed CPython 3.14.6 + uv); built by `make build-python-base`. +ARG PYTHON_UV_IMAGE=messages-python-uv:local +FROM ${PYTHON_UV_IMAGE} WORKDIR /app +# The shared base ships uv (no bundled pip on PATH); install into a venv. +ENV VIRTUAL_ENV=/venv +ENV PATH="/venv/bin:$PATH" COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN uv venv /venv && uv pip install --no-cache -r requirements.txt COPY . . diff --git a/src/mta-in/Dockerfile b/src/mta-in/Dockerfile index 89d5ad38..e549f10b 100644 --- a/src/mta-in/Dockerfile +++ b/src/mta-in/Dockerfile @@ -1,7 +1,7 @@ # Postfix + milter inbound MTA image — counterpart to ./Dockerfile.pymta # (pure-Python aiosmtpd). This is the production default for now. # -# Python comes from the shared uv-managed base (docker/python-uv/Dockerfile: +# Python comes from the shared uv-managed base (deploy/python-uv/Dockerfile: # debian-trixie + uv 0.11.28 digest-pinned + CPython 3.14.6), NOT the official # python image — so the interpreter matches the backend and pymta images. # diff --git a/src/mta-in/Dockerfile.pymta b/src/mta-in/Dockerfile.pymta index 238f7db8..14a75adb 100644 --- a/src/mta-in/Dockerfile.pymta +++ b/src/mta-in/Dockerfile.pymta @@ -12,7 +12,7 @@ # * `runtime-prod` (slim) and `runtime-distroless-prod` (cc-debian13:nonroot) # both available; the distroless variant is the security target. # -# Shared base image (docker/python-uv/Dockerfile): debian-trixie + apt upgrade + +# Shared base image (deploy/python-uv/Dockerfile): debian-trixie + apt upgrade + # ca-certificates + uv (digest-pinned) + uv-managed CPython at /opt/python. # Global ARG so it can be used in `FROM` below. Built by `make build-python-base`; # overridden in CI. @@ -48,7 +48,7 @@ FROM base-with-deps AS base-with-deps-dev RUN --mount=type=cache,target=/root/.cache/uv uv sync --locked --no-install-project --no-editable --extra dev -# ---- Strip Python for the distroless image (see docker/python-uv/strip-python.sh) ---- +# ---- Strip Python for the distroless image (see deploy/python-uv/strip-python.sh) ---- # Only runtime-distroless-prod (the real production target) uses this. The slim # runtime-prod inherits the full managed Python from the shared base instead. FROM ${PYTHON_UV_IMAGE} AS python-runtime diff --git a/src/mta-out/Dockerfile b/src/mta-out/Dockerfile index 49972938..1600abbf 100644 --- a/src/mta-out/Dockerfile +++ b/src/mta-out/Dockerfile @@ -1,6 +1,6 @@ # Outbound MTA image (Postfix + SASL relay). # -# Python comes from the shared uv-managed base (docker/python-uv/Dockerfile: +# Python comes from the shared uv-managed base (deploy/python-uv/Dockerfile: # debian-trixie + uv 0.11.28 digest-pinned + CPython 3.14.6), NOT the official # python image — so the interpreter matches the backend and mta-in images. # diff --git a/src/socks-proxy/tests/Dockerfile b/src/socks-proxy/tests/Dockerfile index f050a66e..e31596b2 100644 --- a/src/socks-proxy/tests/Dockerfile +++ b/src/socks-proxy/tests/Dockerfile @@ -1,19 +1,21 @@ -FROM python:3.13-slim-bookworm@sha256:fcbd8dfc2605ba7c2eca646846c5e892b2931e41f6227985154a596f26ab8ed7 +# Test harness for the socks-proxy (dante). Inherits the shared python-uv base +# (managed CPython 3.14.6 + uv); built by `make build-python-base`. +ARG PYTHON_UV_IMAGE=messages-python-uv:local +FROM ${PYTHON_UV_IMAGE} -# Install system dependencies +# netcat for the proxy connectivity checks. RUN apt-get update && apt-get install -y --no-install-recommends \ netcat-openbsd \ && rm -rf /var/lib/apt/lists/* -# Set working directory WORKDIR /app -# Copy requirements and install Python dependencies +# The shared base ships uv (no bundled pip on PATH); install into a venv. +ENV VIRTUAL_ENV=/venv +ENV PATH="/venv/bin:$PATH" COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN uv venv /venv && uv pip install --no-cache -r requirements.txt -# Copy test files COPY . . -# Default command (can be overridden) CMD ["pytest", "-v"]