diff --git a/CHANGELOG.md b/CHANGELOG.md index 806970d6d..2c2086821 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,7 +85,12 @@ and this project adheres to top of content that exists. Backend reads carrying the admin JWT are seeded too, so a server-side read of an unmigrated document never answers with an empty one. Enabled in the dev - stack via compose.yml + stack via compose.yml. The bucket it reads is configured under `LEGACY_S3_*` + (`_ENDPOINT_URL`, `_ACCESS_KEY_ID`, `_SECRET_ACCESS_KEY`, `_REGION_NAME`, + `_BUCKET_NAME`), a set of its own and not the backend's `AWS_S3_*`: this is + the bucket the collaboration server migrates *out of*, while the one it will + persist *into* when the yhub S3 persistence plugin is enabled is a separate + bucket that may well sit on another provider with credentials of its own - ✨(collaboration) add a migrate endpoint on yhub: `POST /collaboration/migrate/v1/docs/{id}` replays a document's **full** legacy version history from the versioned S3 media bucket into a diff --git a/env.d/development/common b/env.d/development/common index 4b09afab7..954128b04 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -37,6 +37,15 @@ AWS_S3_ACCESS_KEY_ID=impress AWS_S3_SECRET_ACCESS_KEY=password MEDIA_BASE_URL=http://localhost:8083 +# The same bucket, read by yhub's soft migration under names of its own: the +# bucket the collaboration server migrates *out of* is not the one it will +# persist *into* once the S3 persistence plugin is turned on, so it does not +# read the backend's AWS_S3_* settings. Locally they hold the same minio, and +# an override of the three above wants the same override here. +LEGACY_S3_ENDPOINT_URL=http://minio:9000 +LEGACY_S3_ACCESS_KEY_ID=impress +LEGACY_S3_SECRET_ACCESS_KEY=password + # OIDC OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs OIDC_OP_AUTHORIZATION_ENDPOINT=http://localhost:8083/realms/impress/protocol/openid-connect/auth diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index d59691da7..d20b05479 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -339,88 +339,93 @@ ### yhub -| Name | Description | Value | -| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | ------------------------- | -| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` | -| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` | -| `yhub.image.tag` | yhub container tag | `latest` | -| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` | -| `yhub.command` | Override the yhub container command | `[]` | -| `yhub.args` | Override the yhub container args | `[]` | -| `yhub.replicas` | Amount of yhub replicas | `3` | -| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` | -| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` | -| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` | -| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` | -| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` | -| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` | -| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` | -| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` | -| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` | -| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` | -| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | -| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | -| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | -| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` | -| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` | -| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` | -| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` | -| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` | -| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` | -| `yhub.envVars` | Configure yhub container environment variables | `undefined` | -| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | | -| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | | -| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | | -| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | | -| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | -| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | -| `yhub.envVars.YHUB_TASK_CONCURRENCY` | Tasks one worker process claims at once, times the replicas running a worker (default: 5) | | -| `yhub.envVars.YHUB_TASK_DEBOUNCE_MS` | How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) | | -| `yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS` | How long persisted updates stay replayable from redis, in ms (default: 60000) | | -| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | -| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | -| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | -| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | -| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | -| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | -| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` | -| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` | -| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` | -| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` | -| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` | -| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` | -| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` | -| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` | -| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` | -| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` | -| `yhub.service.type` | yhub Service type | `ClusterIP` | -| `yhub.service.port` | yhub Service listening port | `443` | -| `yhub.service.targetPort` | yhub container listening port | `3002` | -| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` | -| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/ping/v1` | -| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` | -| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | `2` | -| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/ready/v1` | -| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` | -| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | `3` | -| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | | -| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | | -| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | | -| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | | -| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | | -| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | | -| `yhub.resources` | Resource requirements for the yhub container | `{}` | -| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` | -| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` | -| `yhub.affinity` | Affinity for the yhub Pod | `{}` | -| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` | -| `yhub.persistence.volume-name.size` | Size of the additional volume | | -| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | -| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | -| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | -| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | -| `yhub.pdb.enabled` | Enable pdb on yhub | `true` | -| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` | +| Name | Description | Value | +| -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------------------------- | +| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` | +| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` | +| `yhub.image.tag` | yhub container tag | `latest` | +| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` | +| `yhub.command` | Override the yhub container command | `[]` | +| `yhub.args` | Override the yhub container args | `[]` | +| `yhub.replicas` | Amount of yhub replicas | `3` | +| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` | +| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` | +| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` | +| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` | +| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` | +| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` | +| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` | +| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` | +| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` | +| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` | +| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | +| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | +| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | +| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` | +| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` | +| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` | +| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` | +| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` | +| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` | +| `yhub.envVars` | Configure yhub container environment variables | `undefined` | +| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | | +| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | | +| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | | +| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | | +| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | +| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | +| `yhub.envVars.YHUB_TASK_CONCURRENCY` | Tasks one worker process claims at once, times the replicas running a worker (default: 5) | | +| `yhub.envVars.YHUB_TASK_DEBOUNCE_MS` | How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) | | +| `yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS` | How long persisted updates stay replayable from redis, in ms (default: 60000) | | +| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | +| `yhub.envVars.LEGACY_S3_ENDPOINT_URL` | Required by SOFT_MIGRATION, endpoint of the legacy Django media bucket, without a path (e.g. https://s3.example.com) | | +| `yhub.envVars.LEGACY_S3_ACCESS_KEY_ID` | Required by SOFT_MIGRATION, read access to the legacy bucket (or LEGACY_S3_ACCESS_KEY_ID_FILE) | | +| `yhub.envVars.LEGACY_S3_SECRET_ACCESS_KEY` | Required by SOFT_MIGRATION, secret of the key above (or LEGACY_S3_SECRET_ACCESS_KEY_FILE) | | +| `yhub.envVars.LEGACY_S3_REGION_NAME` | Region of the legacy bucket, when its provider needs one | | +| `yhub.envVars.LEGACY_S3_BUCKET_NAME` | Name of the legacy Django media bucket (default: impress-media-storage) | | +| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | +| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | +| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | +| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | +| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | +| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` | +| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` | +| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` | +| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` | +| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` | +| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` | +| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` | +| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` | +| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` | +| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` | +| `yhub.service.type` | yhub Service type | `ClusterIP` | +| `yhub.service.port` | yhub Service listening port | `443` | +| `yhub.service.targetPort` | yhub container listening port | `3002` | +| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` | +| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/ping/v1` | +| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` | +| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | `2` | +| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/ready/v1` | +| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` | +| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | `3` | +| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | | +| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | | +| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | | +| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | | +| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | | +| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | | +| `yhub.resources` | Resource requirements for the yhub container | `{}` | +| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` | +| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` | +| `yhub.affinity` | Affinity for the yhub Pod | `{}` | +| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` | +| `yhub.persistence.volume-name.size` | Size of the additional volume | | +| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | +| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | +| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | +| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | +| `yhub.pdb.enabled` | Enable pdb on yhub | `true` | +| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` | ### docSpec diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index 88d646ca5..906f849c0 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -909,6 +909,13 @@ jwtKeys: ## PostgreSQL database, so it needs both — set `yhub.envVars.REDIS` and ## `yhub.envVars.POSTGRES`, there is nothing sensible to default them to. ## Disabling it sends the /collaboration/ ingresses back to the y-provider. +## +## Turning on `SOFT_MIGRATION` adds a bucket to that list, the legacy Django +## media one it reads old documents out of. It is configured under +## `LEGACY_S3_*` rather than the backend's `AWS_S3_*`: the two are read by +## different processes, may be different buckets on different providers, and +## the collaboration server is meant to gain a bucket of its own — the S3 +## persistence plugin, once it is enabled — without either being ambiguous. yhub: ## @param yhub.enabled Enable the yhub collaboration server, its service and its init-db job enabled: true @@ -1017,6 +1024,11 @@ yhub: ## @extra yhub.envVars.YHUB_TASK_DEBOUNCE_MS How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) ## @extra yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS How long persisted updates stay replayable from redis, in ms (default: 60000) ## @extra yhub.envVars.SOFT_MIGRATION Set to "true" to seed rooms from the legacy Django/S3 document store on first access + ## @extra yhub.envVars.LEGACY_S3_ENDPOINT_URL Required by SOFT_MIGRATION, endpoint of the legacy Django media bucket, without a path (e.g. https://s3.example.com) + ## @extra yhub.envVars.LEGACY_S3_ACCESS_KEY_ID Required by SOFT_MIGRATION, read access to the legacy bucket (or LEGACY_S3_ACCESS_KEY_ID_FILE) + ## @extra yhub.envVars.LEGACY_S3_SECRET_ACCESS_KEY Required by SOFT_MIGRATION, secret of the key above (or LEGACY_S3_SECRET_ACCESS_KEY_FILE) + ## @extra yhub.envVars.LEGACY_S3_REGION_NAME Region of the legacy bucket, when its provider needs one + ## @extra yhub.envVars.LEGACY_S3_BUCKET_NAME Name of the legacy Django media bucket (default: impress-media-storage) ## @extra yhub.envVars.BY_VALUE Example environment variable by setting value directly ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 0a7d97e3f..e427cf918 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -367,14 +367,22 @@ Guarantees and failure behavior: document then opens as an *empty* room. Keep the flag on until a backfill has migrated the full corpus. -Configuration: `AWS_S3_ENDPOINT_URL`, `AWS_S3_ACCESS_KEY_ID`, -`AWS_S3_SECRET_ACCESS_KEY` (both with `*_FILE` indirection), optional -`AWS_S3_REGION_NAME`, and `AWS_STORAGE_BUCKET_NAME` (defaults to Django's dev +Configuration: `LEGACY_S3_ENDPOINT_URL`, `LEGACY_S3_ACCESS_KEY_ID`, +`LEGACY_S3_SECRET_ACCESS_KEY` (both with `*_FILE` indirection), optional +`LEGACY_S3_REGION_NAME`, and `LEGACY_S3_BUCKET_NAME` (defaults to Django's dev default `impress-media-storage`; production uses a different bucket name and must set it explicitly). The server refuses to boot when the flag is set without endpoint and credentials. In development the values arrive via `env.d/development/common`. +The prefix is deliberate: these name **the bucket this server migrates out +of**, which is the backend's media bucket and not the one yhub will persist +into once the S3 persistence plugin is enabled. That one gets a set of its own, +and the two are free to be different buckets, on different providers, with +different credentials. Nothing here reads the backend's `AWS_S3_*` settings — +a pod that carries them, for the backend's own reasons, must not quietly +migrate documents out of whatever they point at. + Operational notes: - Use **read-only, bucket-scoped S3 credentials** in production — never the @@ -383,7 +391,7 @@ Operational notes: to `s3:GetObject`: without it, S3 reports a missing object as `403 AccessDenied` instead of `404 NoSuchKey`, and every brand-new document would fail closed instead of starting empty. -- `AWS_S3_ENDPOINT_URL` must not contain a path (the minio client cannot +- `LEGACY_S3_ENDPOINT_URL` must not contain a path (the minio client cannot address a base path); the server refuses to boot otherwise. - After manually wiping a room's yhub state (postgres row + stream key), **restart yhub** so the in-process verdict cache cannot serve a stale diff --git a/src/yhub-server/migration.js b/src/yhub-server/migration.js index 70401a5c0..0235b9f75 100644 --- a/src/yhub-server/migration.js +++ b/src/yhub-server/migration.js @@ -29,13 +29,19 @@ import { Client as S3Client } from 'minio'; import { secret } from './env.js'; export const SOFT_MIGRATION = process.env.SOFT_MIGRATION === 'true'; -const AWS_S3_ENDPOINT_URL = process.env.AWS_S3_ENDPOINT_URL; -const AWS_S3_ACCESS_KEY_ID = secret('AWS_S3_ACCESS_KEY_ID'); -const AWS_S3_SECRET_ACCESS_KEY = secret('AWS_S3_SECRET_ACCESS_KEY'); -const AWS_S3_REGION_NAME = process.env.AWS_S3_REGION_NAME; +// The legacy Django media bucket, the one documents are migrated *out of*. It +// carries a prefix of its own because it is not the only bucket in play: the +// S3 persistence plugin, once it is enabled, persists *into* a bucket that may +// sit on another provider with credentials of its own, and the backend's +// `AWS_S3_*` settings — which a pod may perfectly well carry — name a third. +// Each set is read by exactly the process it belongs to. +const LEGACY_S3_ENDPOINT_URL = process.env.LEGACY_S3_ENDPOINT_URL; +const LEGACY_S3_ACCESS_KEY_ID = secret('LEGACY_S3_ACCESS_KEY_ID'); +const LEGACY_S3_SECRET_ACCESS_KEY = secret('LEGACY_S3_SECRET_ACCESS_KEY'); +const LEGACY_S3_REGION_NAME = process.env.LEGACY_S3_REGION_NAME; // Django's default bucket name (impress settings.py) — prod overrides it -const AWS_STORAGE_BUCKET_NAME = - process.env.AWS_STORAGE_BUCKET_NAME || 'impress-media-storage'; +const LEGACY_S3_BUCKET_NAME = + process.env.LEGACY_S3_BUCKET_NAME || 'impress-media-storage'; // the same limit create-ydoc applies to a posted update in server.js: one // legacy snapshot handed to a compute worker, or written to the stream as a // single message @@ -60,21 +66,23 @@ const EMPTY_YDOC = Y.encodeStateAsUpdate(new Y.Doc()); if ( SOFT_MIGRATION && - (!AWS_S3_ENDPOINT_URL || !AWS_S3_ACCESS_KEY_ID || !AWS_S3_SECRET_ACCESS_KEY) + (!LEGACY_S3_ENDPOINT_URL || + !LEGACY_S3_ACCESS_KEY_ID || + !LEGACY_S3_SECRET_ACCESS_KEY) ) { // fail at boot instead of as an opaque 401 storm on first connect throw new Error( - 'SOFT_MIGRATION=true requires AWS_S3_ENDPOINT_URL, AWS_S3_ACCESS_KEY_ID and AWS_S3_SECRET_ACCESS_KEY', + 'SOFT_MIGRATION=true requires LEGACY_S3_ENDPOINT_URL, LEGACY_S3_ACCESS_KEY_ID and LEGACY_S3_SECRET_ACCESS_KEY', ); } const s3 = SOFT_MIGRATION ? (() => { - const url = new URL(AWS_S3_ENDPOINT_URL); + const url = new URL(LEGACY_S3_ENDPOINT_URL); if (url.pathname !== '/' && url.pathname !== '') { // boto3 accepts path-prefixed endpoints but the minio client cannot // address a base path — dropping it silently would probe the wrong // keys and "migrate" every doc as empty - throw new Error('AWS_S3_ENDPOINT_URL must not contain a path'); + throw new Error('LEGACY_S3_ENDPOINT_URL must not contain a path'); } return new S3Client({ endPoint: url.hostname, @@ -85,9 +93,9 @@ const s3 = SOFT_MIGRATION ? 443 : 80, useSSL: url.protocol === 'https:', - accessKey: AWS_S3_ACCESS_KEY_ID, - secretKey: AWS_S3_SECRET_ACCESS_KEY, - ...(AWS_S3_REGION_NAME ? { region: AWS_S3_REGION_NAME } : {}), + accessKey: LEGACY_S3_ACCESS_KEY_ID, + secretKey: LEGACY_S3_SECRET_ACCESS_KEY, + ...(LEGACY_S3_REGION_NAME ? { region: LEGACY_S3_REGION_NAME } : {}), }); })() : null; @@ -133,7 +141,7 @@ const fetchLegacyDoc = async (docid, versionId = null) => { let objPromise; try { objPromise = s3.getObject( - AWS_STORAGE_BUCKET_NAME, + LEGACY_S3_BUCKET_NAME, `${docid}/file`, // minio stringifies the whole opts object into the query — pass // undefined, not {}, so the unversioned read stays byte-identical @@ -202,7 +210,7 @@ const listLegacyVersions = async (docid) => { const key = `${docid}/file`; const found = await new Promise((resolve, reject) => { const versions = []; - const stream = s3.listObjects(AWS_STORAGE_BUCKET_NAME, key, true, { + const stream = s3.listObjects(LEGACY_S3_BUCKET_NAME, key, true, { IncludeVersion: true, }); const timer = setTimeout(() => { @@ -439,7 +447,7 @@ export const maybeMigrate = async (yhub, room) => { err, docid: room.docid, permanent, - bucket: AWS_STORAGE_BUCKET_NAME, + bucket: LEGACY_S3_BUCKET_NAME, key: `${room.docid}/file`, }, permanent