mirror of
https://github.com/suitenumerique/docs.git
synced 2026-08-17 21:25:43 +02:00
♻️(yhub) prefix S3 envirionment variables used by the migration
We want to allow the usage of different buckets when migrating legascy documents.
This commit is contained in:
Manuel Raynaud
+6
-1
@@ -85,7 +85,12 @@ and this project adheres to
|
||||
top of content that exists.
|
||||
Backend reads carrying the admin JWT are seeded too, so a server-side read of
|
||||
an unmigrated document never answers with an empty one. Enabled in the dev
|
||||
stack via compose.yml
|
||||
stack via compose.yml. The bucket it reads is configured under `LEGACY_S3_*`
|
||||
(`_ENDPOINT_URL`, `_ACCESS_KEY_ID`, `_SECRET_ACCESS_KEY`, `_REGION_NAME`,
|
||||
`_BUCKET_NAME`), a set of its own and not the backend's `AWS_S3_*`: this is
|
||||
the bucket the collaboration server migrates *out of*, while the one it will
|
||||
persist *into* when the yhub S3 persistence plugin is enabled is a separate
|
||||
bucket that may well sit on another provider with credentials of its own
|
||||
- ✨(collaboration) add a migrate endpoint on yhub:
|
||||
`POST /collaboration/migrate/v1/docs/{id}` replays a document's **full**
|
||||
legacy version history from the versioned S3 media bucket into a
|
||||
|
||||
@@ -37,6 +37,15 @@ AWS_S3_ACCESS_KEY_ID=impress
|
||||
AWS_S3_SECRET_ACCESS_KEY=password
|
||||
MEDIA_BASE_URL=http://localhost:8083
|
||||
|
||||
# The same bucket, read by yhub's soft migration under names of its own: the
|
||||
# bucket the collaboration server migrates *out of* is not the one it will
|
||||
# persist *into* once the S3 persistence plugin is turned on, so it does not
|
||||
# read the backend's AWS_S3_* settings. Locally they hold the same minio, and
|
||||
# an override of the three above wants the same override here.
|
||||
LEGACY_S3_ENDPOINT_URL=http://minio:9000
|
||||
LEGACY_S3_ACCESS_KEY_ID=impress
|
||||
LEGACY_S3_SECRET_ACCESS_KEY=password
|
||||
|
||||
# OIDC
|
||||
OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs
|
||||
OIDC_OP_AUTHORIZATION_ENDPOINT=http://localhost:8083/realms/impress/protocol/openid-connect/auth
|
||||
|
||||
+87
-82
@@ -339,88 +339,93 @@
|
||||
|
||||
### yhub
|
||||
|
||||
| Name | Description | Value |
|
||||
| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | ------------------------- |
|
||||
| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` |
|
||||
| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` |
|
||||
| `yhub.image.tag` | yhub container tag | `latest` |
|
||||
| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` |
|
||||
| `yhub.command` | Override the yhub container command | `[]` |
|
||||
| `yhub.args` | Override the yhub container args | `[]` |
|
||||
| `yhub.replicas` | Amount of yhub replicas | `3` |
|
||||
| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` |
|
||||
| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` |
|
||||
| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` |
|
||||
| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` |
|
||||
| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` |
|
||||
| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` |
|
||||
| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` |
|
||||
| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` |
|
||||
| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` |
|
||||
| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` |
|
||||
| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` |
|
||||
| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` |
|
||||
| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` |
|
||||
| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` |
|
||||
| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` |
|
||||
| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` |
|
||||
| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` |
|
||||
| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` |
|
||||
| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` |
|
||||
| `yhub.envVars` | Configure yhub container environment variables | `undefined` |
|
||||
| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | |
|
||||
| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | |
|
||||
| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | |
|
||||
| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | |
|
||||
| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | |
|
||||
| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | |
|
||||
| `yhub.envVars.YHUB_TASK_CONCURRENCY` | Tasks one worker process claims at once, times the replicas running a worker (default: 5) | |
|
||||
| `yhub.envVars.YHUB_TASK_DEBOUNCE_MS` | How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) | |
|
||||
| `yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS` | How long persisted updates stay replayable from redis, in ms (default: 60000) | |
|
||||
| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | |
|
||||
| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | |
|
||||
| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | |
|
||||
| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | |
|
||||
| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | |
|
||||
| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | |
|
||||
| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` |
|
||||
| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` |
|
||||
| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` |
|
||||
| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` |
|
||||
| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` |
|
||||
| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` |
|
||||
| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` |
|
||||
| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` |
|
||||
| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` |
|
||||
| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` |
|
||||
| `yhub.service.type` | yhub Service type | `ClusterIP` |
|
||||
| `yhub.service.port` | yhub Service listening port | `443` |
|
||||
| `yhub.service.targetPort` | yhub container listening port | `3002` |
|
||||
| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` |
|
||||
| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/ping/v1` |
|
||||
| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` |
|
||||
| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | `2` |
|
||||
| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/ready/v1` |
|
||||
| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` |
|
||||
| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | `3` |
|
||||
| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | |
|
||||
| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | |
|
||||
| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | |
|
||||
| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | |
|
||||
| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | |
|
||||
| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | |
|
||||
| `yhub.resources` | Resource requirements for the yhub container | `{}` |
|
||||
| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` |
|
||||
| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` |
|
||||
| `yhub.affinity` | Affinity for the yhub Pod | `{}` |
|
||||
| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` |
|
||||
| `yhub.persistence.volume-name.size` | Size of the additional volume | |
|
||||
| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | |
|
||||
| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | |
|
||||
| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` |
|
||||
| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` |
|
||||
| `yhub.pdb.enabled` | Enable pdb on yhub | `true` |
|
||||
| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` |
|
||||
| Name | Description | Value |
|
||||
| -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------------------------- |
|
||||
| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` |
|
||||
| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` |
|
||||
| `yhub.image.tag` | yhub container tag | `latest` |
|
||||
| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` |
|
||||
| `yhub.command` | Override the yhub container command | `[]` |
|
||||
| `yhub.args` | Override the yhub container args | `[]` |
|
||||
| `yhub.replicas` | Amount of yhub replicas | `3` |
|
||||
| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` |
|
||||
| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` |
|
||||
| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` |
|
||||
| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` |
|
||||
| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` |
|
||||
| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` |
|
||||
| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` |
|
||||
| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` |
|
||||
| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` |
|
||||
| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` |
|
||||
| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` |
|
||||
| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` |
|
||||
| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` |
|
||||
| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` |
|
||||
| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` |
|
||||
| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` |
|
||||
| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` |
|
||||
| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` |
|
||||
| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` |
|
||||
| `yhub.envVars` | Configure yhub container environment variables | `undefined` |
|
||||
| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | |
|
||||
| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | |
|
||||
| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | |
|
||||
| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | |
|
||||
| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | |
|
||||
| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | |
|
||||
| `yhub.envVars.YHUB_TASK_CONCURRENCY` | Tasks one worker process claims at once, times the replicas running a worker (default: 5) | |
|
||||
| `yhub.envVars.YHUB_TASK_DEBOUNCE_MS` | How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) | |
|
||||
| `yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS` | How long persisted updates stay replayable from redis, in ms (default: 60000) | |
|
||||
| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | |
|
||||
| `yhub.envVars.LEGACY_S3_ENDPOINT_URL` | Required by SOFT_MIGRATION, endpoint of the legacy Django media bucket, without a path (e.g. https://s3.example.com) | |
|
||||
| `yhub.envVars.LEGACY_S3_ACCESS_KEY_ID` | Required by SOFT_MIGRATION, read access to the legacy bucket (or LEGACY_S3_ACCESS_KEY_ID_FILE) | |
|
||||
| `yhub.envVars.LEGACY_S3_SECRET_ACCESS_KEY` | Required by SOFT_MIGRATION, secret of the key above (or LEGACY_S3_SECRET_ACCESS_KEY_FILE) | |
|
||||
| `yhub.envVars.LEGACY_S3_REGION_NAME` | Region of the legacy bucket, when its provider needs one | |
|
||||
| `yhub.envVars.LEGACY_S3_BUCKET_NAME` | Name of the legacy Django media bucket (default: impress-media-storage) | |
|
||||
| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | |
|
||||
| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | |
|
||||
| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | |
|
||||
| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | |
|
||||
| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | |
|
||||
| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` |
|
||||
| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` |
|
||||
| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` |
|
||||
| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` |
|
||||
| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` |
|
||||
| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` |
|
||||
| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` |
|
||||
| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` |
|
||||
| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` |
|
||||
| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` |
|
||||
| `yhub.service.type` | yhub Service type | `ClusterIP` |
|
||||
| `yhub.service.port` | yhub Service listening port | `443` |
|
||||
| `yhub.service.targetPort` | yhub container listening port | `3002` |
|
||||
| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` |
|
||||
| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/ping/v1` |
|
||||
| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` |
|
||||
| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | `2` |
|
||||
| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/ready/v1` |
|
||||
| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` |
|
||||
| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | `3` |
|
||||
| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | |
|
||||
| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | |
|
||||
| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | |
|
||||
| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | |
|
||||
| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | |
|
||||
| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | |
|
||||
| `yhub.resources` | Resource requirements for the yhub container | `{}` |
|
||||
| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` |
|
||||
| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` |
|
||||
| `yhub.affinity` | Affinity for the yhub Pod | `{}` |
|
||||
| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` |
|
||||
| `yhub.persistence.volume-name.size` | Size of the additional volume | |
|
||||
| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | |
|
||||
| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | |
|
||||
| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` |
|
||||
| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` |
|
||||
| `yhub.pdb.enabled` | Enable pdb on yhub | `true` |
|
||||
| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` |
|
||||
|
||||
### docSpec
|
||||
|
||||
|
||||
@@ -909,6 +909,13 @@ jwtKeys:
|
||||
## PostgreSQL database, so it needs both — set `yhub.envVars.REDIS` and
|
||||
## `yhub.envVars.POSTGRES`, there is nothing sensible to default them to.
|
||||
## Disabling it sends the /collaboration/ ingresses back to the y-provider.
|
||||
##
|
||||
## Turning on `SOFT_MIGRATION` adds a bucket to that list, the legacy Django
|
||||
## media one it reads old documents out of. It is configured under
|
||||
## `LEGACY_S3_*` rather than the backend's `AWS_S3_*`: the two are read by
|
||||
## different processes, may be different buckets on different providers, and
|
||||
## the collaboration server is meant to gain a bucket of its own — the S3
|
||||
## persistence plugin, once it is enabled — without either being ambiguous.
|
||||
yhub:
|
||||
## @param yhub.enabled Enable the yhub collaboration server, its service and its init-db job
|
||||
enabled: true
|
||||
@@ -1017,6 +1024,11 @@ yhub:
|
||||
## @extra yhub.envVars.YHUB_TASK_DEBOUNCE_MS How long an update waits on the redis stream before a worker persists it, in ms (default: 10000)
|
||||
## @extra yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS How long persisted updates stay replayable from redis, in ms (default: 60000)
|
||||
## @extra yhub.envVars.SOFT_MIGRATION Set to "true" to seed rooms from the legacy Django/S3 document store on first access
|
||||
## @extra yhub.envVars.LEGACY_S3_ENDPOINT_URL Required by SOFT_MIGRATION, endpoint of the legacy Django media bucket, without a path (e.g. https://s3.example.com)
|
||||
## @extra yhub.envVars.LEGACY_S3_ACCESS_KEY_ID Required by SOFT_MIGRATION, read access to the legacy bucket (or LEGACY_S3_ACCESS_KEY_ID_FILE)
|
||||
## @extra yhub.envVars.LEGACY_S3_SECRET_ACCESS_KEY Required by SOFT_MIGRATION, secret of the key above (or LEGACY_S3_SECRET_ACCESS_KEY_FILE)
|
||||
## @extra yhub.envVars.LEGACY_S3_REGION_NAME Region of the legacy bucket, when its provider needs one
|
||||
## @extra yhub.envVars.LEGACY_S3_BUCKET_NAME Name of the legacy Django media bucket (default: impress-media-storage)
|
||||
## @extra yhub.envVars.BY_VALUE Example environment variable by setting value directly
|
||||
## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap
|
||||
## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap
|
||||
|
||||
@@ -367,14 +367,22 @@ Guarantees and failure behavior:
|
||||
document then opens as an *empty* room. Keep the flag on until a backfill
|
||||
has migrated the full corpus.
|
||||
|
||||
Configuration: `AWS_S3_ENDPOINT_URL`, `AWS_S3_ACCESS_KEY_ID`,
|
||||
`AWS_S3_SECRET_ACCESS_KEY` (both with `*_FILE` indirection), optional
|
||||
`AWS_S3_REGION_NAME`, and `AWS_STORAGE_BUCKET_NAME` (defaults to Django's dev
|
||||
Configuration: `LEGACY_S3_ENDPOINT_URL`, `LEGACY_S3_ACCESS_KEY_ID`,
|
||||
`LEGACY_S3_SECRET_ACCESS_KEY` (both with `*_FILE` indirection), optional
|
||||
`LEGACY_S3_REGION_NAME`, and `LEGACY_S3_BUCKET_NAME` (defaults to Django's dev
|
||||
default `impress-media-storage`; production uses a different bucket name and
|
||||
must set it explicitly). The server refuses to boot when the flag is set
|
||||
without endpoint and credentials. In development the values arrive via
|
||||
`env.d/development/common`.
|
||||
|
||||
The prefix is deliberate: these name **the bucket this server migrates out
|
||||
of**, which is the backend's media bucket and not the one yhub will persist
|
||||
into once the S3 persistence plugin is enabled. That one gets a set of its own,
|
||||
and the two are free to be different buckets, on different providers, with
|
||||
different credentials. Nothing here reads the backend's `AWS_S3_*` settings —
|
||||
a pod that carries them, for the backend's own reasons, must not quietly
|
||||
migrate documents out of whatever they point at.
|
||||
|
||||
Operational notes:
|
||||
|
||||
- Use **read-only, bucket-scoped S3 credentials** in production — never the
|
||||
@@ -383,7 +391,7 @@ Operational notes:
|
||||
to `s3:GetObject`: without it, S3 reports a missing object as
|
||||
`403 AccessDenied` instead of `404 NoSuchKey`, and every brand-new document
|
||||
would fail closed instead of starting empty.
|
||||
- `AWS_S3_ENDPOINT_URL` must not contain a path (the minio client cannot
|
||||
- `LEGACY_S3_ENDPOINT_URL` must not contain a path (the minio client cannot
|
||||
address a base path); the server refuses to boot otherwise.
|
||||
- After manually wiping a room's yhub state (postgres row + stream key),
|
||||
**restart yhub** so the in-process verdict cache cannot serve a stale
|
||||
|
||||
@@ -29,13 +29,19 @@ import { Client as S3Client } from 'minio';
|
||||
import { secret } from './env.js';
|
||||
|
||||
export const SOFT_MIGRATION = process.env.SOFT_MIGRATION === 'true';
|
||||
const AWS_S3_ENDPOINT_URL = process.env.AWS_S3_ENDPOINT_URL;
|
||||
const AWS_S3_ACCESS_KEY_ID = secret('AWS_S3_ACCESS_KEY_ID');
|
||||
const AWS_S3_SECRET_ACCESS_KEY = secret('AWS_S3_SECRET_ACCESS_KEY');
|
||||
const AWS_S3_REGION_NAME = process.env.AWS_S3_REGION_NAME;
|
||||
// The legacy Django media bucket, the one documents are migrated *out of*. It
|
||||
// carries a prefix of its own because it is not the only bucket in play: the
|
||||
// S3 persistence plugin, once it is enabled, persists *into* a bucket that may
|
||||
// sit on another provider with credentials of its own, and the backend's
|
||||
// `AWS_S3_*` settings — which a pod may perfectly well carry — name a third.
|
||||
// Each set is read by exactly the process it belongs to.
|
||||
const LEGACY_S3_ENDPOINT_URL = process.env.LEGACY_S3_ENDPOINT_URL;
|
||||
const LEGACY_S3_ACCESS_KEY_ID = secret('LEGACY_S3_ACCESS_KEY_ID');
|
||||
const LEGACY_S3_SECRET_ACCESS_KEY = secret('LEGACY_S3_SECRET_ACCESS_KEY');
|
||||
const LEGACY_S3_REGION_NAME = process.env.LEGACY_S3_REGION_NAME;
|
||||
// Django's default bucket name (impress settings.py) — prod overrides it
|
||||
const AWS_STORAGE_BUCKET_NAME =
|
||||
process.env.AWS_STORAGE_BUCKET_NAME || 'impress-media-storage';
|
||||
const LEGACY_S3_BUCKET_NAME =
|
||||
process.env.LEGACY_S3_BUCKET_NAME || 'impress-media-storage';
|
||||
// the same limit create-ydoc applies to a posted update in server.js: one
|
||||
// legacy snapshot handed to a compute worker, or written to the stream as a
|
||||
// single message
|
||||
@@ -60,21 +66,23 @@ const EMPTY_YDOC = Y.encodeStateAsUpdate(new Y.Doc());
|
||||
|
||||
if (
|
||||
SOFT_MIGRATION &&
|
||||
(!AWS_S3_ENDPOINT_URL || !AWS_S3_ACCESS_KEY_ID || !AWS_S3_SECRET_ACCESS_KEY)
|
||||
(!LEGACY_S3_ENDPOINT_URL ||
|
||||
!LEGACY_S3_ACCESS_KEY_ID ||
|
||||
!LEGACY_S3_SECRET_ACCESS_KEY)
|
||||
) {
|
||||
// fail at boot instead of as an opaque 401 storm on first connect
|
||||
throw new Error(
|
||||
'SOFT_MIGRATION=true requires AWS_S3_ENDPOINT_URL, AWS_S3_ACCESS_KEY_ID and AWS_S3_SECRET_ACCESS_KEY',
|
||||
'SOFT_MIGRATION=true requires LEGACY_S3_ENDPOINT_URL, LEGACY_S3_ACCESS_KEY_ID and LEGACY_S3_SECRET_ACCESS_KEY',
|
||||
);
|
||||
}
|
||||
const s3 = SOFT_MIGRATION
|
||||
? (() => {
|
||||
const url = new URL(AWS_S3_ENDPOINT_URL);
|
||||
const url = new URL(LEGACY_S3_ENDPOINT_URL);
|
||||
if (url.pathname !== '/' && url.pathname !== '') {
|
||||
// boto3 accepts path-prefixed endpoints but the minio client cannot
|
||||
// address a base path — dropping it silently would probe the wrong
|
||||
// keys and "migrate" every doc as empty
|
||||
throw new Error('AWS_S3_ENDPOINT_URL must not contain a path');
|
||||
throw new Error('LEGACY_S3_ENDPOINT_URL must not contain a path');
|
||||
}
|
||||
return new S3Client({
|
||||
endPoint: url.hostname,
|
||||
@@ -85,9 +93,9 @@ const s3 = SOFT_MIGRATION
|
||||
? 443
|
||||
: 80,
|
||||
useSSL: url.protocol === 'https:',
|
||||
accessKey: AWS_S3_ACCESS_KEY_ID,
|
||||
secretKey: AWS_S3_SECRET_ACCESS_KEY,
|
||||
...(AWS_S3_REGION_NAME ? { region: AWS_S3_REGION_NAME } : {}),
|
||||
accessKey: LEGACY_S3_ACCESS_KEY_ID,
|
||||
secretKey: LEGACY_S3_SECRET_ACCESS_KEY,
|
||||
...(LEGACY_S3_REGION_NAME ? { region: LEGACY_S3_REGION_NAME } : {}),
|
||||
});
|
||||
})()
|
||||
: null;
|
||||
@@ -133,7 +141,7 @@ const fetchLegacyDoc = async (docid, versionId = null) => {
|
||||
let objPromise;
|
||||
try {
|
||||
objPromise = s3.getObject(
|
||||
AWS_STORAGE_BUCKET_NAME,
|
||||
LEGACY_S3_BUCKET_NAME,
|
||||
`${docid}/file`,
|
||||
// minio stringifies the whole opts object into the query — pass
|
||||
// undefined, not {}, so the unversioned read stays byte-identical
|
||||
@@ -202,7 +210,7 @@ const listLegacyVersions = async (docid) => {
|
||||
const key = `${docid}/file`;
|
||||
const found = await new Promise((resolve, reject) => {
|
||||
const versions = [];
|
||||
const stream = s3.listObjects(AWS_STORAGE_BUCKET_NAME, key, true, {
|
||||
const stream = s3.listObjects(LEGACY_S3_BUCKET_NAME, key, true, {
|
||||
IncludeVersion: true,
|
||||
});
|
||||
const timer = setTimeout(() => {
|
||||
@@ -439,7 +447,7 @@ export const maybeMigrate = async (yhub, room) => {
|
||||
err,
|
||||
docid: room.docid,
|
||||
permanent,
|
||||
bucket: AWS_STORAGE_BUCKET_NAME,
|
||||
bucket: LEGACY_S3_BUCKET_NAME,
|
||||
key: `${room.docid}/file`,
|
||||
},
|
||||
permanent
|
||||
|
||||
Reference in New Issue
Block a user