mirror of
https://github.com/suitenumerique/docs.git
synced 2026-09-10 11:47:52 +02:00
✨(yhub) allow to configure yhub worker and server
We want to be able to configure both server and worker, the idea is to be able to deploy separately the server and the worker and to scale them.
This commit is contained in:
Manuel Raynaud
@@ -139,6 +139,16 @@ and this project adheres to
|
||||
and rolling the keys is deleting the secret and letting the next run create
|
||||
it again. `jwtKeys.existingSecret` points at keys of your own instead, and
|
||||
skips both the job and its rights
|
||||
- ✨(collaboration) split the yhub server and worker with `YHUB_ROLE`: unset (or
|
||||
`all`) runs both halves in one process as before, `server` holds the
|
||||
websockets and the routes without claiming a task, `worker` drains the redis
|
||||
stream into postgres without binding a port. They share the two stores and
|
||||
nothing else, so each scales on what drives it — connected editors on one
|
||||
side, write throughput on the other. Any other value is refused at startup.
|
||||
In the helm chart, `yhub.worker.enabled` turns the single deployment into
|
||||
two, sets the variable on each, and gives the worker no service and no probes
|
||||
since it binds nothing; everything not named under `yhub.worker` is the
|
||||
server's
|
||||
- ✨(collaboration) serve two probes on yhub: `GET /collaboration/ping/v1`
|
||||
answers `pong` without touching a store — being answered is what a liveness
|
||||
check should conclude, and restarting a server over a store it cannot reach
|
||||
|
||||
@@ -197,7 +197,11 @@ jwtKeys:
|
||||
enabled: true
|
||||
|
||||
yhub:
|
||||
replicas: 1
|
||||
replicas: 3
|
||||
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 2
|
||||
|
||||
image:
|
||||
repository: localhost:5001/impress-yhub
|
||||
|
||||
@@ -160,7 +160,11 @@ jwtKeys:
|
||||
enabled: true
|
||||
|
||||
yhub:
|
||||
replicas: 1
|
||||
replicas: 3
|
||||
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 2
|
||||
|
||||
image:
|
||||
repository: lasuite/impress-yhub
|
||||
|
||||
@@ -348,6 +348,16 @@
|
||||
| `yhub.command` | Override the yhub container command | `[]` |
|
||||
| `yhub.args` | Override the yhub container args | `[]` |
|
||||
| `yhub.replicas` | Amount of yhub replicas | `3` |
|
||||
| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` |
|
||||
| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` |
|
||||
| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` |
|
||||
| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` |
|
||||
| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` |
|
||||
| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` |
|
||||
| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` |
|
||||
| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` |
|
||||
| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` |
|
||||
| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` |
|
||||
| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` |
|
||||
| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` |
|
||||
| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` |
|
||||
|
||||
@@ -204,6 +204,42 @@ Requires top level scope
|
||||
{{ include "impress.fullname" . }}-yhub
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Full name for the yhub worker, when it is deployed apart from the server
|
||||
|
||||
Requires top level scope
|
||||
*/}}
|
||||
{{- define "impress.yhub.worker.fullname" -}}
|
||||
{{ include "impress.yhub.fullname" . }}-worker
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
yhub worker env vars - combines common yhub.envVars with yhub.worker.envVars
|
||||
*/}}
|
||||
{{- define "impress.yhub.worker.env" -}}
|
||||
{{- $topLevelScope := index . 0 -}}
|
||||
{{- $workerScope := index . 1 -}}
|
||||
{{- include "impress.env.transformDict" $workerScope.envVars -}}
|
||||
{{- include "impress.env.transformDict" (($workerScope.worker | default dict).envVars | default dict) -}}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
The role a yhub pod runs, as an environment variable. Only when the worker is
|
||||
deployed apart: a single deployment runs both halves, which is what yhub does
|
||||
when the variable is absent. Skipped when the deployment names the role itself,
|
||||
in either env map — an explicit value wins, as everywhere else here.
|
||||
|
||||
Usage: {{ include "impress.yhub.roleEnv" (dict "root" $ "role" "server") }}
|
||||
*/}}
|
||||
{{- define "impress.yhub.roleEnv" -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $named := merge (dict) (($root.Values.yhub.worker | default dict).envVars | default dict) ($root.Values.yhub.envVars | default dict) -}}
|
||||
{{- if and $root.Values.yhub.worker.enabled (not (hasKey $named "YHUB_ROLE")) }}
|
||||
- name: "YHUB_ROLE"
|
||||
value: {{ .role | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
JWT signing keys — the RSA keys the services sign the calls they make to each
|
||||
other with. The jwt-keys job generates them once into a secret every service
|
||||
|
||||
@@ -54,12 +54,14 @@ spec:
|
||||
args:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or $envVars .Values.jwtKeys.enabled }}
|
||||
{{- $roleEnv := include "impress.yhub.roleEnv" (dict "root" . "role" "server") }}
|
||||
{{- if or $envVars .Values.jwtKeys.enabled $roleEnv }}
|
||||
env:
|
||||
{{- $envVars | indent 12 }}
|
||||
{{- if .Values.jwtKeys.enabled }}
|
||||
{{- include "impress.jwtKeys.yhubEnv" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- $roleEnv | indent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.yhub.envFrom }}
|
||||
envFrom:
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
{{- if and .Values.yhub.enabled .Values.yhub.worker.enabled -}}
|
||||
{{- $envVars := include "impress.yhub.worker.env" (list . .Values.yhub) -}}
|
||||
{{- $fullName := include "impress.yhub.worker.fullname" . -}}
|
||||
{{- $component := "yhub-worker" -}}
|
||||
{{- $worker := .Values.yhub.worker -}}
|
||||
# The half of yhub that drains the redis stream into postgres, deployed apart
|
||||
# from the one serving the websockets (YHUB_ROLE). It scales on the write
|
||||
# throughput rather than on the connected editors, and redis consumer groups
|
||||
# hand each task to exactly one of these pods.
|
||||
#
|
||||
# No service, no ports and no probes: a worker binds nothing, it claims tasks.
|
||||
# Its health is its process being alive — the task loop logs and backs off on
|
||||
# error rather than dying, so a pod that stopped working is one that exited,
|
||||
# and kubelet restarts it.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ $fullName }}
|
||||
namespace: {{ .Release.Namespace | quote }}
|
||||
annotations:
|
||||
{{- with ($worker.dpAnnotations | default .Values.yhub.dpAnnotations) }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "impress.common.labels" (list . $component) | nindent 4 }}
|
||||
spec:
|
||||
replicas: {{ $worker.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
{{- with ($worker.podAnnotations | default .Values.yhub.podAnnotations) }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "impress.common.selectorLabels" (list . $component) | nindent 8 }}
|
||||
spec:
|
||||
{{- if $.Values.image.credentials }}
|
||||
imagePullSecrets:
|
||||
- name: {{ include "impress.secret.dockerconfigjson.name" (dict "fullname" (include "impress.fullname" .) "imageCredentials" $.Values.image.credentials) }}
|
||||
{{- end}}
|
||||
{{- if .Values.yhub.serviceAccountName }}
|
||||
serviceAccountName: {{ .Values.yhub.serviceAccountName }}
|
||||
{{- end }}
|
||||
shareProcessNamespace: {{ .Values.yhub.shareProcessNamespace }}
|
||||
# a task claimed by a pod that goes away is redelivered to another one,
|
||||
# but letting the current one finish saves that round trip
|
||||
terminationGracePeriodSeconds: {{ $worker.terminationGracePeriodSeconds | default .Values.yhub.terminationGracePeriodSeconds }}
|
||||
containers:
|
||||
{{- with .Values.yhub.sidecars }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
- name: {{ .Chart.Name }}
|
||||
image: "{{ (.Values.yhub.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.yhub.image | default dict).tag | default .Values.image.tag }}"
|
||||
imagePullPolicy: {{ (.Values.yhub.image | default dict).pullPolicy | default .Values.image.pullPolicy }}
|
||||
{{- with .Values.yhub.command }}
|
||||
command:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.yhub.args }}
|
||||
args:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- $roleEnv := include "impress.yhub.roleEnv" (dict "root" . "role" "worker") }}
|
||||
{{- if or $envVars .Values.jwtKeys.enabled $roleEnv }}
|
||||
env:
|
||||
{{- $envVars | indent 12 }}
|
||||
{{- if .Values.jwtKeys.enabled }}
|
||||
{{- include "impress.jwtKeys.yhubEnv" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- $roleEnv | indent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.yhub.envFrom }}
|
||||
envFrom:
|
||||
{{- toYaml .Values.yhub.envFrom | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.yhub.securityContext }}
|
||||
securityContext:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with ($worker.resources | default .Values.yhub.resources) }}
|
||||
resources:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- if .Values.jwtKeys.enabled }}
|
||||
{{- include "impress.jwtKeys.volumeMount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- range $index, $value := .Values.mountFiles }}
|
||||
- name: "files-{{ $index }}"
|
||||
mountPath: {{ $value.path }}
|
||||
subPath: content
|
||||
{{- end }}
|
||||
{{- range .Values.yhub.extraVolumeMounts }}
|
||||
- name: {{ .name }}
|
||||
mountPath: {{ .mountPath }}
|
||||
subPath: {{ .subPath | default "" }}
|
||||
readOnly: {{ .readOnly }}
|
||||
{{- end }}
|
||||
{{- with ($worker.nodeSelector | default .Values.yhub.nodeSelector) }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with ($worker.affinity | default .Values.yhub.affinity) }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with ($worker.tolerations | default .Values.yhub.tolerations) }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.jwtKeys.enabled }}
|
||||
{{- include "impress.jwtKeys.volume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- range $index, $value := .Values.mountFiles }}
|
||||
- name: "files-{{ $index }}"
|
||||
configMap:
|
||||
name: "{{ include "impress.fullname" $ }}-files-{{ $index }}"
|
||||
{{- end }}
|
||||
{{- range .Values.yhub.extraVolumes }}
|
||||
- name: {{ .name }}
|
||||
{{- if .existingClaim }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .existingClaim }}
|
||||
{{- else if .secret }}
|
||||
secret:
|
||||
{{ toYaml .secret | nindent 12 }}
|
||||
{{- else if .hostPath }}
|
||||
hostPath:
|
||||
{{ toYaml .hostPath | nindent 12 }}
|
||||
{{- else if .csi }}
|
||||
csi:
|
||||
{{- toYaml .csi | nindent 12 }}
|
||||
{{- else if .configMap }}
|
||||
configMap:
|
||||
{{- toYaml .configMap | nindent 12 }}
|
||||
{{- else if .emptyDir }}
|
||||
emptyDir:
|
||||
{{- toYaml .emptyDir | nindent 12 }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
---
|
||||
{{ if $worker.pdb.enabled }}
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: {{ $fullName }}
|
||||
namespace: {{ .Release.Namespace | quote }}
|
||||
spec:
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }}
|
||||
{{ end }}
|
||||
{{- end }}
|
||||
@@ -928,11 +928,47 @@ yhub:
|
||||
args: []
|
||||
|
||||
## @param yhub.replicas Amount of yhub replicas
|
||||
## Every replica also runs a worker (redis consumer groups hand each task to
|
||||
## one of them), and clients editing the same document need not land on the
|
||||
## same pod: updates travel through redis.
|
||||
## Clients editing the same document need not land on the same pod: updates
|
||||
## travel through redis. Each replica also runs a worker unless the worker is
|
||||
## deployed apart, see below.
|
||||
replicas: 3
|
||||
|
||||
## @param yhub.worker.enabled Deploy the worker apart from the server, each scaling on its own
|
||||
## @param yhub.worker.replicas Amount of yhub worker replicas
|
||||
## @param yhub.worker.resources Resource requirements for the yhub worker container, the server ones when empty
|
||||
## @param yhub.worker.podAnnotations Annotations to add to the yhub worker Pod, the server ones when empty
|
||||
## @param yhub.worker.dpAnnotations Annotations to add to the yhub worker Deployment, the server ones when empty
|
||||
## @param yhub.worker.nodeSelector Node selector for the yhub worker Pod, the server one when empty
|
||||
## @param yhub.worker.tolerations Tolerations for the yhub worker Pod, the server ones when empty
|
||||
## @param yhub.worker.affinity Affinity for the yhub worker Pod, the server one when empty
|
||||
## @param yhub.worker.terminationGracePeriodSeconds Grace period given to a worker pod to finish its task, the server one when empty
|
||||
## @param yhub.worker.pdb.enabled Enable pdb on the yhub worker
|
||||
## @skip yhub.worker.envVars Environment variables of the worker only, on top of yhub.envVars
|
||||
##
|
||||
## yhub is two halves sharing nothing but redis and postgres: the server
|
||||
## holds the websockets and serves the routes, the worker drains the stream
|
||||
## into postgres. One process runs both by default. Enabling this splits them
|
||||
## into two deployments — `YHUB_ROLE=server` and `YHUB_ROLE=worker`, the only
|
||||
## difference between them — so the server scales with the connected editors
|
||||
## and the worker with the write throughput.
|
||||
##
|
||||
## The worker binds nothing: no service, no ingress, and no probes to give it
|
||||
## (its liveness is its process). Everything not named here is the server's:
|
||||
## same image, same envVars, same secrets, same volumes.
|
||||
worker:
|
||||
enabled: false
|
||||
replicas: 1
|
||||
envVars: {}
|
||||
resources: {}
|
||||
podAnnotations: {}
|
||||
dpAnnotations: {}
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
terminationGracePeriodSeconds: null
|
||||
pdb:
|
||||
enabled: true
|
||||
|
||||
## @param yhub.shareProcessNamespace Enable share process namespace between containers
|
||||
shareProcessNamespace: false
|
||||
|
||||
|
||||
@@ -89,6 +89,36 @@ probes are not worth publishing either — kubelet calls them from inside — an
|
||||
the helm chart's ingress lists what it routes rather than what it hides, so
|
||||
they stay in-cluster on their own.
|
||||
|
||||
## Roles (`YHUB_ROLE`)
|
||||
|
||||
yhub is two halves that share the two stores and nothing else — no in-process
|
||||
state, no ordering between them:
|
||||
|
||||
- the **server** accepts the websocket connections, serves the routes above,
|
||||
and writes every update to the redis stream,
|
||||
- the **worker** claims tasks from that stream, merges the updates and stores
|
||||
the result in postgres, then trims what it persisted.
|
||||
|
||||
One process runs both, which is the default and what `YHUB_ROLE` unset means.
|
||||
Setting it splits them, so each can be scaled on its own — the server with the
|
||||
connected editors, the worker with the write throughput:
|
||||
|
||||
| `YHUB_ROLE` | websocket + routes | drains the stream |
|
||||
| ----------- | ------------------ | ----------------- |
|
||||
| unset, `all` | yes | yes |
|
||||
| `server` | yes | no |
|
||||
| `worker` | no | yes |
|
||||
|
||||
A `worker` process binds no port: no probes to give it and no service to put in
|
||||
front of it. A `server` process claims no task, so a deployment of servers
|
||||
alone accepts edits and never persists them — the two halves are split
|
||||
together or not at all. Any other value is refused at startup rather than
|
||||
guessed.
|
||||
|
||||
Redis consumer groups hand each task to exactly one worker, so the number of
|
||||
workers is a throughput knob and nothing else: no leader, no partitioning, no
|
||||
coordination between them.
|
||||
|
||||
## Container image
|
||||
|
||||
The `Dockerfile` has two final stages, like the other services of this
|
||||
|
||||
@@ -39,6 +39,24 @@ const allowedOrigins = (
|
||||
).split(',');
|
||||
const Y_PROVIDER_API_KEY = secret('Y_PROVIDER_API_KEY', 'yprovider-api-key');
|
||||
const ORG = process.env.YHUB_ORG || 'docs';
|
||||
// Which halves of yhub this process runs. The server accepts the websocket
|
||||
// connections and serves the REST routes; the worker drains the redis stream
|
||||
// into postgres. They share the two stores and nothing else — no in-process
|
||||
// state, no ordering between them — so one process can run both (the default)
|
||||
// or a deployment can split them and scale each on its own: the server with
|
||||
// the connected editors, the worker with the write throughput.
|
||||
//
|
||||
// A stream is only drained by the workers that are running: a deployment of
|
||||
// `server` alone keeps accepting edits and never persists them, so the two
|
||||
// halves are split together or not at all.
|
||||
const ROLE = process.env.YHUB_ROLE || 'all';
|
||||
if (!['all', 'server', 'worker'].includes(ROLE)) {
|
||||
throw new Error(
|
||||
`YHUB_ROLE must be one of "all", "server" or "worker" (got "${ROLE}")`,
|
||||
);
|
||||
}
|
||||
const RUNS_SERVER = ROLE !== 'worker';
|
||||
const RUNS_WORKER = ROLE !== 'server';
|
||||
// Segment every route is mounted under (`server.apiPrefix` below), matching the
|
||||
// URL scheme Docs already routes to the collaboration server. Hardcoded like
|
||||
// the audiences: the backend builds its urls with the same prefix.
|
||||
@@ -818,8 +836,19 @@ const yhub = await createYHub({
|
||||
},
|
||||
postgres: POSTGRES,
|
||||
persistence: [], // blobs live in yhub's postgres
|
||||
// Both halves are declared, and YHUB_ROLE decides which are built: a null
|
||||
// server binds no port at all (a `worker` pod has no http surface, hence no
|
||||
// probes and no service in front of it), a null worker claims no task.
|
||||
//
|
||||
// apiPrefix mounts every route — built-ins, our custom endpoints, and the
|
||||
// websocket (/collaboration/ws/v1/{org}/{docid}) — under /collaboration/.
|
||||
server: { port: PORT, auth, api, apiPrefix: API_PREFIX },
|
||||
worker: { taskConcurrency: 5, events: workerEvents },
|
||||
server: RUNS_SERVER
|
||||
? { port: PORT, auth, api, apiPrefix: API_PREFIX }
|
||||
: null,
|
||||
worker: RUNS_WORKER ? { taskConcurrency: 5, events: workerEvents } : null,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
{ role: ROLE, server: RUNS_SERVER, worker: RUNS_WORKER },
|
||||
'yhub role',
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user