diff --git a/CHANGELOG.md b/CHANGELOG.md index acf096892..d1abdb92c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,16 @@ and this project adheres to and rolling the keys is deleting the secret and letting the next run create it again. `jwtKeys.existingSecret` points at keys of your own instead, and skips both the job and its rights +- ✨(collaboration) split the yhub server and worker with `YHUB_ROLE`: unset (or + `all`) runs both halves in one process as before, `server` holds the + websockets and the routes without claiming a task, `worker` drains the redis + stream into postgres without binding a port. They share the two stores and + nothing else, so each scales on what drives it — connected editors on one + side, write throughput on the other. Any other value is refused at startup. + In the helm chart, `yhub.worker.enabled` turns the single deployment into + two, sets the variable on each, and gives the worker no service and no probes + since it binds nothing; everything not named under `yhub.worker` is the + server's - ✨(collaboration) serve two probes on yhub: `GET /collaboration/ping/v1` answers `pong` without touching a store — being answered is what a liveness check should conclude, and restarting a server over a store it cannot reach diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index df87cc03a..c4d1cc72d 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -197,7 +197,11 @@ jwtKeys: enabled: true yhub: - replicas: 1 + replicas: 3 + + worker: + enabled: true + replicas: 2 image: repository: localhost:5001/impress-yhub diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index e2ff44aca..dd4a5748f 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -160,7 +160,11 @@ jwtKeys: enabled: true yhub: - replicas: 1 + replicas: 3 + + worker: + enabled: true + replicas: 2 image: repository: lasuite/impress-yhub diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index 1bf5fb7c9..545aaf0e3 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -348,6 +348,16 @@ | `yhub.command` | Override the yhub container command | `[]` | | `yhub.args` | Override the yhub container args | `[]` | | `yhub.replicas` | Amount of yhub replicas | `3` | +| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` | +| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` | +| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` | +| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` | +| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` | +| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` | +| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` | +| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` | +| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` | +| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` | | `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | | `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | | `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | diff --git a/src/helm/impress/templates/_helpers.tpl b/src/helm/impress/templates/_helpers.tpl index 8f8424571..1b5d4a78f 100644 --- a/src/helm/impress/templates/_helpers.tpl +++ b/src/helm/impress/templates/_helpers.tpl @@ -204,6 +204,42 @@ Requires top level scope {{ include "impress.fullname" . }}-yhub {{- end }} +{{/* +Full name for the yhub worker, when it is deployed apart from the server + +Requires top level scope +*/}} +{{- define "impress.yhub.worker.fullname" -}} +{{ include "impress.yhub.fullname" . }}-worker +{{- end }} + +{{/* +yhub worker env vars - combines common yhub.envVars with yhub.worker.envVars +*/}} +{{- define "impress.yhub.worker.env" -}} +{{- $topLevelScope := index . 0 -}} +{{- $workerScope := index . 1 -}} +{{- include "impress.env.transformDict" $workerScope.envVars -}} +{{- include "impress.env.transformDict" (($workerScope.worker | default dict).envVars | default dict) -}} +{{- end }} + +{{/* +The role a yhub pod runs, as an environment variable. Only when the worker is +deployed apart: a single deployment runs both halves, which is what yhub does +when the variable is absent. Skipped when the deployment names the role itself, +in either env map — an explicit value wins, as everywhere else here. + +Usage: {{ include "impress.yhub.roleEnv" (dict "root" $ "role" "server") }} +*/}} +{{- define "impress.yhub.roleEnv" -}} +{{- $root := .root -}} +{{- $named := merge (dict) (($root.Values.yhub.worker | default dict).envVars | default dict) ($root.Values.yhub.envVars | default dict) -}} +{{- if and $root.Values.yhub.worker.enabled (not (hasKey $named "YHUB_ROLE")) }} +- name: "YHUB_ROLE" + value: {{ .role | quote }} +{{- end }} +{{- end }} + {{/* JWT signing keys — the RSA keys the services sign the calls they make to each other with. The jwt-keys job generates them once into a secret every service diff --git a/src/helm/impress/templates/yhub_deployment.yaml b/src/helm/impress/templates/yhub_deployment.yaml index 8ca6772aa..cdaf3da09 100644 --- a/src/helm/impress/templates/yhub_deployment.yaml +++ b/src/helm/impress/templates/yhub_deployment.yaml @@ -54,12 +54,14 @@ spec: args: {{- toYaml . | nindent 12 }} {{- end }} - {{- if or $envVars .Values.jwtKeys.enabled }} + {{- $roleEnv := include "impress.yhub.roleEnv" (dict "root" . "role" "server") }} + {{- if or $envVars .Values.jwtKeys.enabled $roleEnv }} env: {{- $envVars | indent 12 }} {{- if .Values.jwtKeys.enabled }} {{- include "impress.jwtKeys.yhubEnv" . | nindent 12 }} {{- end }} + {{- $roleEnv | indent 12 }} {{- end }} {{- if .Values.yhub.envFrom }} envFrom: diff --git a/src/helm/impress/templates/yhub_worker_deployment.yaml b/src/helm/impress/templates/yhub_worker_deployment.yaml new file mode 100644 index 000000000..0da9a6e7d --- /dev/null +++ b/src/helm/impress/templates/yhub_worker_deployment.yaml @@ -0,0 +1,160 @@ +{{- if and .Values.yhub.enabled .Values.yhub.worker.enabled -}} +{{- $envVars := include "impress.yhub.worker.env" (list . .Values.yhub) -}} +{{- $fullName := include "impress.yhub.worker.fullname" . -}} +{{- $component := "yhub-worker" -}} +{{- $worker := .Values.yhub.worker -}} +# The half of yhub that drains the redis stream into postgres, deployed apart +# from the one serving the websockets (YHUB_ROLE). It scales on the write +# throughput rather than on the connected editors, and redis consumer groups +# hand each task to exactly one of these pods. +# +# No service, no ports and no probes: a worker binds nothing, it claims tasks. +# Its health is its process being alive — the task loop logs and backs off on +# error rather than dying, so a pod that stopped working is one that exited, +# and kubelet restarts it. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + annotations: + {{- with ($worker.dpAnnotations | default .Values.yhub.dpAnnotations) }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} +spec: + replicas: {{ $worker.replicas }} + selector: + matchLabels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }} + template: + metadata: + annotations: + {{- with ($worker.podAnnotations | default .Values.yhub.podAnnotations) }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "impress.secret.dockerconfigjson.name" (dict "fullname" (include "impress.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + {{- if .Values.yhub.serviceAccountName }} + serviceAccountName: {{ .Values.yhub.serviceAccountName }} + {{- end }} + shareProcessNamespace: {{ .Values.yhub.shareProcessNamespace }} + # a task claimed by a pod that goes away is redelivered to another one, + # but letting the current one finish saves that round trip + terminationGracePeriodSeconds: {{ $worker.terminationGracePeriodSeconds | default .Values.yhub.terminationGracePeriodSeconds }} + containers: + {{- with .Values.yhub.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: {{ .Chart.Name }} + image: "{{ (.Values.yhub.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.yhub.image | default dict).tag | default .Values.image.tag }}" + imagePullPolicy: {{ (.Values.yhub.image | default dict).pullPolicy | default .Values.image.pullPolicy }} + {{- with .Values.yhub.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.yhub.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- $roleEnv := include "impress.yhub.roleEnv" (dict "root" . "role" "worker") }} + {{- if or $envVars .Values.jwtKeys.enabled $roleEnv }} + env: + {{- $envVars | indent 12 }} + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.yhubEnv" . | nindent 12 }} + {{- end }} + {{- $roleEnv | indent 12 }} + {{- end }} + {{- if .Values.yhub.envFrom }} + envFrom: + {{- toYaml .Values.yhub.envFrom | nindent 12 }} + {{- end }} + {{- with .Values.yhub.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with ($worker.resources | default .Values.yhub.resources) }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volumeMount" . | nindent 12 }} + {{- end }} + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range .Values.yhub.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with ($worker.nodeSelector | default .Values.yhub.nodeSelector) }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with ($worker.affinity | default .Values.yhub.affinity) }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with ($worker.tolerations | default .Values.yhub.tolerations) }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volume" . | nindent 8 }} + {{- end }} + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "impress.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range .Values.yhub.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .secret }} + secret: + {{ toYaml .secret | nindent 12 }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} +--- +{{ if $worker.pdb.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }} +{{ end }} +{{- end }} diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index 35b33708a..35064b7ef 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -928,11 +928,47 @@ yhub: args: [] ## @param yhub.replicas Amount of yhub replicas - ## Every replica also runs a worker (redis consumer groups hand each task to - ## one of them), and clients editing the same document need not land on the - ## same pod: updates travel through redis. + ## Clients editing the same document need not land on the same pod: updates + ## travel through redis. Each replica also runs a worker unless the worker is + ## deployed apart, see below. replicas: 3 + ## @param yhub.worker.enabled Deploy the worker apart from the server, each scaling on its own + ## @param yhub.worker.replicas Amount of yhub worker replicas + ## @param yhub.worker.resources Resource requirements for the yhub worker container, the server ones when empty + ## @param yhub.worker.podAnnotations Annotations to add to the yhub worker Pod, the server ones when empty + ## @param yhub.worker.dpAnnotations Annotations to add to the yhub worker Deployment, the server ones when empty + ## @param yhub.worker.nodeSelector Node selector for the yhub worker Pod, the server one when empty + ## @param yhub.worker.tolerations Tolerations for the yhub worker Pod, the server ones when empty + ## @param yhub.worker.affinity Affinity for the yhub worker Pod, the server one when empty + ## @param yhub.worker.terminationGracePeriodSeconds Grace period given to a worker pod to finish its task, the server one when empty + ## @param yhub.worker.pdb.enabled Enable pdb on the yhub worker + ## @skip yhub.worker.envVars Environment variables of the worker only, on top of yhub.envVars + ## + ## yhub is two halves sharing nothing but redis and postgres: the server + ## holds the websockets and serves the routes, the worker drains the stream + ## into postgres. One process runs both by default. Enabling this splits them + ## into two deployments — `YHUB_ROLE=server` and `YHUB_ROLE=worker`, the only + ## difference between them — so the server scales with the connected editors + ## and the worker with the write throughput. + ## + ## The worker binds nothing: no service, no ingress, and no probes to give it + ## (its liveness is its process). Everything not named here is the server's: + ## same image, same envVars, same secrets, same volumes. + worker: + enabled: false + replicas: 1 + envVars: {} + resources: {} + podAnnotations: {} + dpAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + terminationGracePeriodSeconds: null + pdb: + enabled: true + ## @param yhub.shareProcessNamespace Enable share process namespace between containers shareProcessNamespace: false diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 324ff0a2b..815b072c3 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -89,6 +89,36 @@ probes are not worth publishing either — kubelet calls them from inside — an the helm chart's ingress lists what it routes rather than what it hides, so they stay in-cluster on their own. +## Roles (`YHUB_ROLE`) + +yhub is two halves that share the two stores and nothing else — no in-process +state, no ordering between them: + +- the **server** accepts the websocket connections, serves the routes above, + and writes every update to the redis stream, +- the **worker** claims tasks from that stream, merges the updates and stores + the result in postgres, then trims what it persisted. + +One process runs both, which is the default and what `YHUB_ROLE` unset means. +Setting it splits them, so each can be scaled on its own — the server with the +connected editors, the worker with the write throughput: + +| `YHUB_ROLE` | websocket + routes | drains the stream | +| ----------- | ------------------ | ----------------- | +| unset, `all` | yes | yes | +| `server` | yes | no | +| `worker` | no | yes | + +A `worker` process binds no port: no probes to give it and no service to put in +front of it. A `server` process claims no task, so a deployment of servers +alone accepts edits and never persists them — the two halves are split +together or not at all. Any other value is refused at startup rather than +guessed. + +Redis consumer groups hand each task to exactly one worker, so the number of +workers is a throughput knob and nothing else: no leader, no partitioning, no +coordination between them. + ## Container image The `Dockerfile` has two final stages, like the other services of this diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index d63a4e1f9..2ac2a0411 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -39,6 +39,24 @@ const allowedOrigins = ( ).split(','); const Y_PROVIDER_API_KEY = secret('Y_PROVIDER_API_KEY', 'yprovider-api-key'); const ORG = process.env.YHUB_ORG || 'docs'; +// Which halves of yhub this process runs. The server accepts the websocket +// connections and serves the REST routes; the worker drains the redis stream +// into postgres. They share the two stores and nothing else — no in-process +// state, no ordering between them — so one process can run both (the default) +// or a deployment can split them and scale each on its own: the server with +// the connected editors, the worker with the write throughput. +// +// A stream is only drained by the workers that are running: a deployment of +// `server` alone keeps accepting edits and never persists them, so the two +// halves are split together or not at all. +const ROLE = process.env.YHUB_ROLE || 'all'; +if (!['all', 'server', 'worker'].includes(ROLE)) { + throw new Error( + `YHUB_ROLE must be one of "all", "server" or "worker" (got "${ROLE}")`, + ); +} +const RUNS_SERVER = ROLE !== 'worker'; +const RUNS_WORKER = ROLE !== 'server'; // Segment every route is mounted under (`server.apiPrefix` below), matching the // URL scheme Docs already routes to the collaboration server. Hardcoded like // the audiences: the backend builds its urls with the same prefix. @@ -818,8 +836,19 @@ const yhub = await createYHub({ }, postgres: POSTGRES, persistence: [], // blobs live in yhub's postgres + // Both halves are declared, and YHUB_ROLE decides which are built: a null + // server binds no port at all (a `worker` pod has no http surface, hence no + // probes and no service in front of it), a null worker claims no task. + // // apiPrefix mounts every route — built-ins, our custom endpoints, and the // websocket (/collaboration/ws/v1/{org}/{docid}) — under /collaboration/. - server: { port: PORT, auth, api, apiPrefix: API_PREFIX }, - worker: { taskConcurrency: 5, events: workerEvents }, + server: RUNS_SERVER + ? { port: PORT, auth, api, apiPrefix: API_PREFIX } + : null, + worker: RUNS_WORKER ? { taskConcurrency: 5, events: workerEvents } : null, }); + +logger.info( + { role: ROLE, server: RUNS_SERVER, worker: RUNS_WORKER }, + 'yhub role', +);