From 2dfdae59b17393a3cc02c7a8d0cc433354ab16a3 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Mon, 17 Aug 2026 15:34:53 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(yhub)=20allow=20optional=20usage=20of?= =?UTF-8?q?=20s3=20persistent=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Yhub has a s3 persistent plugin, since now we didn't use it but we wanto to give the possibility to use it optionnaly. To enable it set the YHUB_S3_PERSISTENCE environment value to true and configure it. --- CHANGELOG.md | 28 ++- compose.yml | 3 +- env.d/development/yhub | 12 ++ src/helm/env.d/dev/values.impress.yaml.gotmpl | 5 + .../env.d/feature/values.impress.yaml.gotmpl | 6 + src/helm/impress/README.md | 182 +++++++++--------- src/helm/impress/values.yaml | 23 ++- src/yhub-server/README.md | 77 +++++++- src/yhub-server/migration.js | 4 +- src/yhub-server/server.js | 81 +++++++- 10 files changed, 318 insertions(+), 103 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a3cb9ca1..c57e4ca97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,28 @@ and this project adheres to ### Added +- ✨(collaboration) let the collaboration server keep the document blobs in a + bucket instead of its own PostgreSQL database, through yhub's S3 persistence + plugin: `YHUB_S3_PERSISTENCE=true`, plus `YHUB_S3_ENDPOINT_URL`, + `YHUB_S3_ACCESS_KEY_ID`, `YHUB_S3_SECRET_ACCESS_KEY`, `YHUB_S3_BUCKET_NAME` + and, when the provider needs one told rather than discovered, + `YHUB_S3_REGION_NAME`. Off by default, which keeps everything in postgres — + the configuration Docs has been running. Turned on, every compaction writes + its four blobs (the garbage-collected document, the one that keeps its + history, the content map and the content ids) to the bucket and leaves a + reference in the row: postgres holds the index of the corpus, the bucket + holds its bytes. It is a third bucket, configured under a prefix of its own + next to the backend's `AWS_S3_*` and the legacy document store's + `LEGACY_S3_*`, since the three may sit on three providers and each is read by + the process it belongs to. Note that it is a one-way setting: a row pointing + at an object is unreadable without the plugin that wrote it, and yhub reports + such a version as having no content rather than as an error, so removing the + setting after a compaction serves those documents empty. That, the + permissions the credentials need and what the objects are is in + `src/yhub-server/README.md`, worth reading before enabling it. An incomplete + configuration is refused at startup, naming what is missing, rather than + surfacing on the first compaction — a background task, where it would look + like documents quietly not being persisted - ✨(collaboration) erase the content of a document on the collaboration server when `clean_document` resets it. The command cleared the database and the object storage, but the content lives on the collaboration server now: it kept @@ -90,9 +112,9 @@ and this project adheres to (`_ENDPOINT_URL`, `_ACCESS_KEY_ID`, `_SECRET_ACCESS_KEY`, `_REGION_NAME`, `_BUCKET_NAME`, `_SIGNATURE_VERSION`), a set of its own and not the backend's `AWS_S3_*`: this is the bucket the collaboration server migrates *out of*, - while the one it will persist *into* when the yhub S3 persistence plugin is - enabled is a separate bucket that may well sit on another provider with - credentials of its own. It is read with the AWS SDK for JavaScript v3, whose + while the one it persists *into* when `YHUB_S3_PERSISTENCE` is on is a + separate bucket that may well sit on another provider with credentials of its + own. It is read with the AWS SDK for JavaScript v3, whose signature version is configurable (`s3v4` by default, as in Django) because a provider expecting another one answers 403, which reads exactly like wrong credentials diff --git a/compose.yml b/compose.yml index 509587c08..0fd96fb24 100644 --- a/compose.yml +++ b/compose.yml @@ -262,7 +262,8 @@ services: yhub-postgres: condition: service_healthy # soft migration reads the legacy document store at startup traffic — - # starting before minio would cache 401s for the first accessed docs + # starting before minio would cache 401s for the first accessed docs — + # and YHUB_S3_PERSISTENCE, when it is on, checks its own bucket at boot minio: condition: service_healthy diff --git a/env.d/development/yhub b/env.d/development/yhub index 8cf36c467..556396174 100644 --- a/env.d/development/yhub +++ b/env.d/development/yhub @@ -32,3 +32,15 @@ SOFT_MIGRATION=true LEGACY_S3_ENDPOINT_URL=http://minio:9000 LEGACY_S3_ACCESS_KEY_ID=impress LEGACY_S3_SECRET_ACCESS_KEY=password + +# Document storage: where the blobs of a compaction are written. Off, they stay +# in yhub's postgres, which is what this stack runs. Turning it on stores them +# in object storage instead — here the same minio, in a bucket of its own that +# the server creates on startup when it is missing. Read the "Document storage" +# section of src/yhub-server/README.md first: a document persisted this way +# cannot be read back with the plugin turned off again. +YHUB_S3_PERSISTENCE=false +YHUB_S3_ENDPOINT_URL=http://minio:9000 +YHUB_S3_ACCESS_KEY_ID=impress +YHUB_S3_SECRET_ACCESS_KEY=password +YHUB_S3_BUCKET_NAME=yhub-storage diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index ff1930c25..ce9530123 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -220,6 +220,11 @@ yhub: NODE_EXTRA_CA_CERTS: /cert/cacert.pem # YHUB_JWT_PRIVATE_KEY_FILE comes from the jwtKeys job below LOG_LEVEL: debug + YHUB_S3_PERSISTENCE: true + YHUB_S3_ENDPOINT_URL: http://dev-backend-minio.impress.svc.cluster.local:9000 + YHUB_S3_ACCESS_KEY_ID: dinum + YHUB_S3_SECRET_ACCESS_KEY: password + YHUB_S3_BUCKET_NAME: docs-media-storage # Extra volume mounts to manage our local custom CA and avoid to set ssl_verify: false extraVolumeMounts: diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index 9fc55e9e3..183d72d53 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -182,6 +182,12 @@ yhub: NODE_OPTIONS: "--max-old-space-size=1024" UWS_HTTP_MAX_HEADERS_SIZE: 32768 LOG_LEVEL: debug + YHUB_S3_PERSISTENCE: true + YHUB_S3_ENDPOINT_URL: http://dev-backend-minio.{{ .Namespace }}.svc.cluster.local:9000 + YHUB_S3_ACCESS_KEY_ID: dinum + YHUB_S3_SECRET_ACCESS_KEY: password + YHUB_S3_BUCKET_NAME: docs-media-storage + docSpec: enabled: true diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index 219eadd90..95bd7ff23 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -339,94 +339,100 @@ ### yhub -| Name | Description | Value | -| -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------------------------- | -| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` | -| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` | -| `yhub.image.tag` | yhub container tag | `latest` | -| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` | -| `yhub.command` | Override the yhub container command | `[]` | -| `yhub.args` | Override the yhub container args | `[]` | -| `yhub.replicas` | Amount of yhub replicas | `3` | -| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` | -| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` | -| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` | -| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` | -| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` | -| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` | -| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` | -| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` | -| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` | -| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` | -| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | -| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | -| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | -| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` | -| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` | -| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` | -| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` | -| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` | -| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` | -| `yhub.envVars` | Configure yhub container environment variables | `undefined` | -| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | | -| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | | -| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | | -| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | | -| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | -| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | -| `yhub.envVars.YHUB_TASK_CONCURRENCY` | Tasks one worker process claims at once, times the replicas running a worker (default: 5) | | -| `yhub.envVars.YHUB_TASK_DEBOUNCE_MS` | How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) | | -| `yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS` | How long persisted updates stay replayable from redis, in ms (default: 60000) | | -| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | -| `yhub.envVars.LEGACY_S3_ENDPOINT_URL` | Required by SOFT_MIGRATION, endpoint of the legacy Django media bucket, without a path (e.g. https://s3.example.com) | | -| `yhub.envVars.LEGACY_S3_ACCESS_KEY_ID` | Required by SOFT_MIGRATION, read access to the legacy bucket (or LEGACY_S3_ACCESS_KEY_ID_FILE) | | -| `yhub.envVars.LEGACY_S3_SECRET_ACCESS_KEY` | Required by SOFT_MIGRATION, secret of the key above (or LEGACY_S3_SECRET_ACCESS_KEY_FILE) | | -| `yhub.envVars.LEGACY_S3_REGION_NAME` | Region of the legacy bucket, when its provider needs one | | -| `yhub.envVars.LEGACY_S3_BUCKET_NAME` | Name of the legacy Django media bucket (default: impress-media-storage) | | -| `yhub.envVars.LEGACY_S3_SIGNATURE_VERSION` | How the calls to the legacy bucket are signed, s3v4 or v4 (default: s3v4) | | -| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | -| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | -| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | -| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | -| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | -| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` | -| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` | -| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` | -| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` | -| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` | -| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` | -| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` | -| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` | -| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` | -| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` | -| `yhub.service.type` | yhub Service type | `ClusterIP` | -| `yhub.service.port` | yhub Service listening port | `443` | -| `yhub.service.targetPort` | yhub container listening port | `3002` | -| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` | -| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/ping/v1` | -| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` | -| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | `2` | -| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/ready/v1` | -| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` | -| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | `3` | -| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | | -| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | | -| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | | -| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | | -| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | | -| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | | -| `yhub.resources` | Resource requirements for the yhub container | `{}` | -| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` | -| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` | -| `yhub.affinity` | Affinity for the yhub Pod | `{}` | -| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` | -| `yhub.persistence.volume-name.size` | Size of the additional volume | | -| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | -| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | -| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | -| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | -| `yhub.pdb.enabled` | Enable pdb on yhub | `true` | -| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` | +| Name | Description | Value | +| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | +| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` | +| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` | +| `yhub.image.tag` | yhub container tag | `latest` | +| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` | +| `yhub.command` | Override the yhub container command | `[]` | +| `yhub.args` | Override the yhub container args | `[]` | +| `yhub.replicas` | Amount of yhub replicas | `3` | +| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` | +| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` | +| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` | +| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` | +| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` | +| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` | +| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` | +| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` | +| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` | +| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` | +| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | +| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | +| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | +| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` | +| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` | +| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` | +| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` | +| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` | +| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` | +| `yhub.envVars` | Configure yhub container environment variables | `undefined` | +| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | | +| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | | +| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | | +| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | | +| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | +| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | +| `yhub.envVars.YHUB_TASK_CONCURRENCY` | Tasks one worker process claims at once, times the replicas running a worker (default: 5) | | +| `yhub.envVars.YHUB_TASK_DEBOUNCE_MS` | How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) | | +| `yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS` | How long persisted updates stay replayable from redis, in ms (default: 60000) | | +| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | +| `yhub.envVars.LEGACY_S3_ENDPOINT_URL` | Required by SOFT_MIGRATION, endpoint of the legacy Django media bucket, without a path (e.g. https://s3.example.com) | | +| `yhub.envVars.LEGACY_S3_ACCESS_KEY_ID` | Required by SOFT_MIGRATION, read access to the legacy bucket (or LEGACY_S3_ACCESS_KEY_ID_FILE) | | +| `yhub.envVars.LEGACY_S3_SECRET_ACCESS_KEY` | Required by SOFT_MIGRATION, secret of the key above (or LEGACY_S3_SECRET_ACCESS_KEY_FILE) | | +| `yhub.envVars.LEGACY_S3_REGION_NAME` | Region of the legacy bucket, when its provider needs one | | +| `yhub.envVars.LEGACY_S3_BUCKET_NAME` | Name of the legacy Django media bucket (default: impress-media-storage) | | +| `yhub.envVars.LEGACY_S3_SIGNATURE_VERSION` | How the calls to the legacy bucket are signed, s3v4 or v4 (default: s3v4) | | +| `yhub.envVars.YHUB_S3_PERSISTENCE` | Set to "true" to store the document blobs in a bucket instead of the yhub database — read src/yhub-server/README.md first, it cannot be turned back off | | +| `yhub.envVars.YHUB_S3_ENDPOINT_URL` | Required by YHUB_S3_PERSISTENCE, endpoint of the bucket the blobs are stored in, without a path (e.g. https://s3.example.com) | | +| `yhub.envVars.YHUB_S3_ACCESS_KEY_ID` | Required by YHUB_S3_PERSISTENCE, read/write/delete access to that bucket (or YHUB_S3_ACCESS_KEY_ID_FILE) | | +| `yhub.envVars.YHUB_S3_SECRET_ACCESS_KEY` | Required by YHUB_S3_PERSISTENCE, secret of the key above (or YHUB_S3_SECRET_ACCESS_KEY_FILE) | | +| `yhub.envVars.YHUB_S3_BUCKET_NAME` | Required by YHUB_S3_PERSISTENCE, name of that bucket, created on startup when missing | | +| `yhub.envVars.YHUB_S3_REGION_NAME` | Region of that bucket, when its provider needs one | | +| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | +| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | +| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | +| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | +| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | +| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` | +| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` | +| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` | +| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` | +| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` | +| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` | +| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` | +| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` | +| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` | +| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` | +| `yhub.service.type` | yhub Service type | `ClusterIP` | +| `yhub.service.port` | yhub Service listening port | `443` | +| `yhub.service.targetPort` | yhub container listening port | `3002` | +| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` | +| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/ping/v1` | +| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` | +| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | `2` | +| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/ready/v1` | +| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` | +| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | `3` | +| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | | +| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | | +| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | | +| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | | +| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | | +| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | | +| `yhub.resources` | Resource requirements for the yhub container | `{}` | +| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` | +| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` | +| `yhub.affinity` | Affinity for the yhub Pod | `{}` | +| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` | +| `yhub.persistence.volume-name.size` | Size of the additional volume | | +| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | +| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | +| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | +| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | +| `yhub.pdb.enabled` | Enable pdb on yhub | `true` | +| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` | ### docSpec diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index a4a8b0829..9729995eb 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -910,12 +910,17 @@ jwtKeys: ## `yhub.envVars.POSTGRES`, there is nothing sensible to default them to. ## Disabling it sends the /collaboration/ ingresses back to the y-provider. ## -## Turning on `SOFT_MIGRATION` adds a bucket to that list, the legacy Django -## media one it reads old documents out of. It is configured under -## `LEGACY_S3_*` rather than the backend's `AWS_S3_*`: the two are read by -## different processes, may be different buckets on different providers, and -## the collaboration server is meant to gain a bucket of its own — the S3 -## persistence plugin, once it is enabled — without either being ambiguous. +## Two buckets can be added to that list, each with a prefix of its own so that +## none of them is ambiguous — they may sit on different providers, with +## different credentials, and are read by different processes: +## +## - `LEGACY_S3_*`, turned on by `SOFT_MIGRATION`, is the legacy Django media +## bucket it reads old documents *out of*. Not the backend's `AWS_S3_*`, +## which names the same bucket for the backend's own use, +## - `YHUB_S3_*`, turned on by `YHUB_S3_PERSISTENCE`, is a bucket of its own it +## stores the document blobs *into*, instead of its PostgreSQL database. +## Read `src/yhub-server/README.md` before enabling it: a document persisted +## this way cannot be read back once the setting is removed. yhub: ## @param yhub.enabled Enable the yhub collaboration server, its service and its init-db job enabled: true @@ -1030,6 +1035,12 @@ yhub: ## @extra yhub.envVars.LEGACY_S3_REGION_NAME Region of the legacy bucket, when its provider needs one ## @extra yhub.envVars.LEGACY_S3_BUCKET_NAME Name of the legacy Django media bucket (default: impress-media-storage) ## @extra yhub.envVars.LEGACY_S3_SIGNATURE_VERSION How the calls to the legacy bucket are signed, s3v4 or v4 (default: s3v4) + ## @extra yhub.envVars.YHUB_S3_PERSISTENCE Set to "true" to store the document blobs in a bucket instead of the yhub database — read src/yhub-server/README.md first, it cannot be turned back off + ## @extra yhub.envVars.YHUB_S3_ENDPOINT_URL Required by YHUB_S3_PERSISTENCE, endpoint of the bucket the blobs are stored in, without a path (e.g. https://s3.example.com) + ## @extra yhub.envVars.YHUB_S3_ACCESS_KEY_ID Required by YHUB_S3_PERSISTENCE, read/write/delete access to that bucket (or YHUB_S3_ACCESS_KEY_ID_FILE) + ## @extra yhub.envVars.YHUB_S3_SECRET_ACCESS_KEY Required by YHUB_S3_PERSISTENCE, secret of the key above (or YHUB_S3_SECRET_ACCESS_KEY_FILE) + ## @extra yhub.envVars.YHUB_S3_BUCKET_NAME Required by YHUB_S3_PERSISTENCE, name of that bucket, created on startup when missing + ## @extra yhub.envVars.YHUB_S3_REGION_NAME Region of that bucket, when its provider needs one ## @extra yhub.envVars.BY_VALUE Example environment variable by setting value directly ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index d3500a79c..757937132 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -14,7 +14,9 @@ It is not a fork of yhub — it is a thin wrapper: `server.js`: - starts a yhub instance (websocket sync on port 3002, backed by Redis/Valkey - and PostgreSQL), + and PostgreSQL — and, when `YHUB_S3_PERSISTENCE` asks for it, a bucket the + document blobs are stored in instead of the database, see "Document storage" + below), - plugs in an auth plugin that resolves users and per-document access rights by calling the Docs Django backend (`/api/v1.0/users/me/` and `/api/v1.0/documents/{id}/`), @@ -158,9 +160,80 @@ kubernetes variable left blank behaves as if it were absent. The effective values are logged at startup, next to the role: ```json -{"role":"all","server":true,"worker":true,"taskConcurrency":5,"taskDebounceMs":10000,"minMessageLifetimeMs":60000,"msg":"yhub configuration"} +{"role":"all","server":true,"worker":true,"taskConcurrency":5,"s3Bucket":null,"taskDebounceMs":10000,"minMessageLifetimeMs":60000,"msg":"yhub configuration"} ``` +## Document storage (`YHUB_S3_PERSISTENCE`) + +Every compaction writes one row in `yhub_ydoc_v1`, and that row carries four +blobs: the garbage-collected document, the one that keeps its history, the +content map and the content ids. By default they are `bytea` columns — the +whole corpus lives on the database disk, which is the configuration Docs has +been running and what this server does when nothing below is set. + +`YHUB_S3_PERSISTENCE=true` plugs yhub's own S3 persistence plugin +(`S3PersistenceV1`, shipped with `@y/hub`) into the chain it consults before +writing a blob and before reading one back. The blobs then go to a bucket and +the row keeps a reference to them, `_is_reference` saying which of the +four it is: postgres holds the index of the documents, the bucket holds their +bytes. + +| Variable | Required | What it is | +| -------- | -------- | ---------- | +| `YHUB_S3_PERSISTENCE` | — | `true` to store the blobs in a bucket (default: postgres) | +| `YHUB_S3_ENDPOINT_URL` | yes | Endpoint of that bucket, without a path (e.g. `https://s3.example.com`) | +| `YHUB_S3_ACCESS_KEY_ID` | yes | Key with read, write and delete on the bucket (or `…_FILE`) | +| `YHUB_S3_SECRET_ACCESS_KEY` | yes | Secret of that key (or `…_FILE`) | +| `YHUB_S3_BUCKET_NAME` | yes | Name of the bucket. No default: a typo would create one | +| `YHUB_S3_REGION_NAME` | no | Region, when the provider needs one told rather than discovered | + +"Required" means required *when the plugin is on*: it is a startup error naming +what is missing, rather than a client that ends up anonymous and only says so +on the first compaction — which is a background task, so the failure would show +up as documents quietly not being persisted. The bucket in use is logged next +to the role (`"s3Bucket":"yhub-storage"`, `null` for postgres). + +This is a **third** bucket, and it is deliberately configured apart from the +other two: the backend's media bucket (`AWS_S3_*`, Django's own settings) and +the legacy document store the migrations read (`LEGACY_S3_*`, see below). They +may sit on three providers with three sets of credentials, and each is read by +the process it belongs to. + +A few things worth knowing before turning it on: + +- **It cannot be turned back off.** A row pointing at an object is unreadable + without the plugin that wrote it, and yhub reports such a version as having + no content rather than as an error — so a document compacted while the plugin + was on comes back *empty* once it is off, silently. Turning it on is safe in + the other direction: rows written before keep their bytes inline and are + served exactly as they were, +- **the bucket is created at startup** when it does not exist, so the + credentials need `HeadBucket` and, the first time, `CreateBucket`. It is + checked on every boot, which is also what makes a wrong endpoint or a wrong + key fail loudly and immediately, +- **both halves need it.** The worker writes the blobs and the server reads + them back, so a split deployment (`YHUB_ROLE`) configures the bucket on both + — in the helm chart the worker inherits `yhub.envVars`, so there is nothing + to repeat, +- **only the `main` branch is offloaded.** The plugin declines everything else + and those blobs stay in postgres, which is yhub's behaviour, not a setting, +- **objects are deleted late.** When a version's row is dropped (pruning, a + reset, a hard deletion), the object is removed about ten seconds later, so + that readers holding the reference are not left with a 404. A delete that + fails is logged and forgotten: the bucket may accumulate objects no row names + anymore, and nothing collects them, +- the objects are Yjs blobs keyed by + `id:ydoc:v1/{org}/{docid}/{branch}/{gc}/{clock}` (and `id:contentmap:v1/…`, + `id:contentids:v1/…`) — one object per version and per column, not one file + per document, and **not** a format anything but yhub reads. It is a storage + backend, not an export and not a backup. + +In the dev stack the variables are in `env.d/development/yhub`, pointing at the +same minio the rest of the stack uses with a bucket of its own +(`yhub-storage`), and the toggle is off. Flipping it to `true` and restarting +the service is enough to exercise the path — on a dev database, where losing +the documents already compacted costs nothing. + ## Container image The `Dockerfile` has two final stages, like the other services of this diff --git a/src/yhub-server/migration.js b/src/yhub-server/migration.js index ea248e638..cf59ce686 100644 --- a/src/yhub-server/migration.js +++ b/src/yhub-server/migration.js @@ -35,8 +35,8 @@ import { secret } from './env.js'; export const SOFT_MIGRATION = process.env.SOFT_MIGRATION === 'true'; // The legacy Django media bucket, the one documents are migrated *out of*. It // carries a prefix of its own because it is not the only bucket in play: the -// S3 persistence plugin, once it is enabled, persists *into* a bucket that may -// sit on another provider with credentials of its own, and the backend's +// S3 persistence plugin (`YHUB_S3_*`, server.js) persists *into* a bucket that +// may sit on another provider with credentials of its own, and the backend's // `AWS_S3_*` settings — which a pod may perfectly well carry — name a third. // Each set is read by exactly the process it belongs to. const LEGACY_S3_ENDPOINT_URL = process.env.LEGACY_S3_ENDPOINT_URL; diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 2d4c10e18..7b67dc1b7 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -8,6 +8,7 @@ import { createYHub, logger, } from '@y/hub'; +import { S3PersistenceV1 } from '@y/hub/plugins/s3'; import { calculateJwkThumbprint, createRemoteJWKSet, @@ -90,6 +91,27 @@ const RUNS_WORKER = ROLE !== 'server'; // point where a pod runs out of memory, each task holding the document it // merges. const TASK_CONCURRENCY = intEnv('YHUB_TASK_CONCURRENCY', 5, 1); +// Where the blobs of a compaction go — the garbage-collected document, the one +// that keeps its history, the content map and the content ids. yhub writes the +// four of them into its own postgres; a persistence plugin takes them out of +// it, the row then holding a reference and the bytes living in the plugin's +// store. Off by default, which is postgres alone, the way Docs has been +// running. +// +// Not a switch that can be flipped back: a row pointing at an object is +// unreadable without the plugin that wrote it — yhub reports that version as +// having no content rather than as an error — so turning it off after a +// compaction strands what was stored while it was on. See README.md. +const S3_PERSISTENCE = process.env.YHUB_S3_PERSISTENCE === 'true'; +// Its own bucket, named apart from the backend's `AWS_S3_*` and from the legacy +// document store's `LEGACY_S3_*` (migration.js): three buckets that may sit on +// three providers with credentials of their own, each read by the process it +// belongs to. +const YHUB_S3_ENDPOINT_URL = process.env.YHUB_S3_ENDPOINT_URL; +const YHUB_S3_ACCESS_KEY_ID = secret('YHUB_S3_ACCESS_KEY_ID'); +const YHUB_S3_SECRET_ACCESS_KEY = secret('YHUB_S3_SECRET_ACCESS_KEY'); +const YHUB_S3_BUCKET_NAME = process.env.YHUB_S3_BUCKET_NAME; +const YHUB_S3_REGION_NAME = process.env.YHUB_S3_REGION_NAME; // Segment every route is mounted under (`server.apiPrefix` below), matching the // URL scheme Docs already routes to the collaboration server. Hardcoded like // the audiences: the backend builds its urls with the same prefix. @@ -858,6 +880,60 @@ const workerEvents = { }, }; +// The persistence plugins yhub consults, in order, before writing a blob to +// postgres and before reading one back. An empty list keeps everything in the +// database, which is the default. +// +// Read here rather than in the call below so that an incomplete configuration +// is a startup error naming what is missing: the client would otherwise be +// built anonymous or against the wrong host and only say so on the first +// compaction, which is a background task — the failure would show up as +// documents quietly not being persisted. +const persistencePlugins = () => { + if (!S3_PERSISTENCE) return []; + + const missing = [ + ['YHUB_S3_ENDPOINT_URL', YHUB_S3_ENDPOINT_URL], + ['YHUB_S3_ACCESS_KEY_ID', YHUB_S3_ACCESS_KEY_ID], + ['YHUB_S3_SECRET_ACCESS_KEY', YHUB_S3_SECRET_ACCESS_KEY], + ['YHUB_S3_BUCKET_NAME', YHUB_S3_BUCKET_NAME], + ] + .filter(([, value]) => !value) + .map(([name]) => name); + if (missing.length > 0) { + throw new Error(`YHUB_S3_PERSISTENCE=true requires ${missing.join(', ')}`); + } + + const url = new URL(YHUB_S3_ENDPOINT_URL); + if (url.pathname !== '/' && url.pathname !== '') { + // the client is given a host and a port, so a base path would be dropped + // without a word and the objects written next to where they belong + throw new Error('YHUB_S3_ENDPOINT_URL must not contain a path'); + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + // the client is told "SSL or not", so any other scheme would read as "not" + // and send the credentials in clear + throw new Error('YHUB_S3_ENDPOINT_URL must be http:// or https://'); + } + const useSSL = url.protocol === 'https:'; + + return [ + new S3PersistenceV1({ + bucket: YHUB_S3_BUCKET_NAME, + endPoint: url.hostname, + // an implicit port parses as "", which the client reads as 0 — its way + // of saying "whatever the scheme defaults to" + port: Number(url.port), + useSSL, + accessKey: YHUB_S3_ACCESS_KEY_ID, + secretKey: YHUB_S3_SECRET_ACCESS_KEY, + // left out rather than passed empty: unset, the client discovers the + // region of the bucket instead of validating an empty string + ...(YHUB_S3_REGION_NAME ? { region: YHUB_S3_REGION_NAME } : {}), + }), + ]; +}; + // the instance is referenced by the soft-migration helpers above — safe: auth // callbacks only fire once the server is up, i.e. after this assignment const yhub = await createYHub({ @@ -868,7 +944,8 @@ const yhub = await createYHub({ minMessageLifetime: MIN_MESSAGE_LIFETIME_MS, }, postgres: POSTGRES, - persistence: [], // blobs live in yhub's postgres + // where the blobs live: nothing here keeps them in yhub's postgres + persistence: persistencePlugins(), // Both halves are declared, and YHUB_ROLE decides which are built: a null // server binds no port at all (a `worker` pod has no http surface, hence no // probes and no service in front of it), a null worker claims no task. @@ -894,6 +971,8 @@ logger.info( server: RUNS_SERVER, worker: RUNS_WORKER, taskConcurrency: RUNS_WORKER ? TASK_CONCURRENCY : null, + // where the compaction blobs go — null is yhub's own postgres + s3Bucket: S3_PERSISTENCE ? YHUB_S3_BUCKET_NAME : null, taskDebounceMs: yhub.stream.taskDebounce, minMessageLifetimeMs: yhub.stream.minMessageLifetime, },