From 5acd4c29026f39e3d9907a6f7d3eaa5a213fa159 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 16 Sep 2026 15:08:58 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7(tilt)=20run=20two=20valkey=20senti?= =?UTF-8?q?nel=20instances=20through=20valkey-operator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need to use valkey instead of an old redis version. We decided to use the operator https://github.com/chideat/valkey-operator/ and configure it in both feature and dev environments. --- CHANGELOG.md | 1 + bin/Tiltfile | 40 +++++++++++- .../env.d/dev/values.dev-backend.yaml.gotmpl | 7 +-- src/helm/env.d/dev/values.impress.yaml.gotmpl | 10 +-- src/helm/env.d/dev/values.valkey.yaml.gotmpl | 3 + .../feature/values.dev-backend.yaml.gotmpl | 6 +- .../env.d/feature/values.impress.yaml.gotmpl | 9 ++- .../env.d/feature/values.valkey.yaml.gotmpl | 3 + src/helm/extra/valkey/Chart.yaml | 5 ++ src/helm/extra/valkey/templates/_helpers.tpl | 16 +++++ src/helm/extra/valkey/templates/user.yaml | 27 ++++++++ src/helm/extra/valkey/templates/valkey.yaml | 49 +++++++++++++++ src/helm/extra/valkey/values.yaml | 61 +++++++++++++++++++ src/helm/helmfile.yaml.gotmpl | 31 ++++++++++ src/helm/overlays/patch.yaml | 21 +++++++ 15 files changed, 269 insertions(+), 20 deletions(-) create mode 100644 src/helm/env.d/dev/values.valkey.yaml.gotmpl create mode 100644 src/helm/env.d/feature/values.valkey.yaml.gotmpl create mode 100644 src/helm/extra/valkey/Chart.yaml create mode 100644 src/helm/extra/valkey/templates/_helpers.tpl create mode 100644 src/helm/extra/valkey/templates/user.yaml create mode 100644 src/helm/extra/valkey/templates/valkey.yaml create mode 100644 src/helm/extra/valkey/values.yaml create mode 100644 src/helm/overlays/patch.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb26f4d1..c764d5aed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ and this project adheres to - 💥(backend) move the resource server JWKS from `/api/{version}/jwks` to `/external_api/{version}/jwks` - 🔧(collaboration) adapt docker stack for development purpose +- 🔧(tilt) run two valkey sentinel instances through valkey-operator ### Fixed diff --git a/bin/Tiltfile b/bin/Tiltfile index 032e23428..4c7c28b67 100644 --- a/bin/Tiltfile +++ b/bin/Tiltfile @@ -56,10 +56,44 @@ docker_build( k8s_resource('impress-docs-backend-migrate', resource_deps=['dev-backend-postgres']) k8s_resource('impress-docs-backend-createsuperuser', resource_deps=['impress-docs-backend-migrate']) -k8s_resource('impress-docs-yhub-init-db', resource_deps=['dev-backend-postgres']) +k8s_resource('impress-docs-yhub-init-db', resource_deps=['dev-backend-postgres', 'valkey-yhub']) k8s_resource('dev-backend-keycloak', resource_deps=['dev-backend-keycloak-pg']) -k8s_resource('impress-docs-backend', resource_deps=['impress-docs-backend-migrate', 'dev-backend-redis', 'dev-backend-keycloak', 'dev-backend-postgres', 'dev-backend-minio:statefulset']) -k8s_yaml(local('cd ../src/helm && helmfile -n impress -e dev template .')) +k8s_resource('impress-docs-backend', resource_deps=['impress-docs-backend-migrate', 'valkey-docs', 'dev-backend-keycloak', 'dev-backend-postgres', 'dev-backend-minio:statefulset']) +k8s_resource('impress-docs-celery-worker', resource_deps=['valkey-docs']) +k8s_resource('impress-docs-yhub', resource_deps=['valkey-yhub']) +k8s_resource('impress-docs-yhub-worker', resource_deps=['valkey-yhub']) + +# the operator needs its CRDs, which land in uncategorized +k8s_resource('valkey-operator', resource_deps=['uncategorized']) +# The pods of a Valkey instance are created by the operator: group them with the +# objects they come from, and wait for them +for instance in ['valkey-docs', 'valkey-yhub']: + k8s_resource( + new_name=instance, + objects=['%s:valkey' % instance, '%s-user:user' % instance, '%s-user:secret' % instance], + extra_pod_selectors=[{'buf.red/name': instance}], + pod_readiness='wait', + resource_deps=['valkey-operator'], + ) + +# without --include-crds, the CRDs of the valkey-operator chart are left out +manifests = decode_yaml_stream(local('cd ../src/helm && helmfile -n impress -e dev template --include-crds .')) +for manifest in manifests: + if not manifest: + continue + # tilt down leaves the operator and its CRDs in place: deleted along with the + # Valkey resources, nothing would remain to clear their finalizers, and the + # stuck resources would block the CRDs of the next tilt up + if ((manifest['kind'] == 'CustomResourceDefinition' and manifest['spec']['group'].endswith('valkey.buf.red')) or + manifest['metadata'].get('labels', {}).get('app.kubernetes.io/instance') == 'valkey-operator'): + manifest['metadata'].setdefault('annotations', {})['tilt.dev/down-policy'] = 'keep' + # The chart's webhook.enabled=false only drops the webhook configuration: the + # operator still serves its webhooks and crashes on the missing certificate + # unless ENABLE_WEBHOOKS=false, which the chart cannot set (argocd gets it + # from src/helm/extra/valkey-operator/disable-webhooks.ytt.yaml) + if manifest['kind'] == 'Deployment' and manifest['metadata']['name'] == 'valkey-operator': + manifest['spec']['template']['spec']['containers'][0]['env'].append({'name': 'ENABLE_WEBHOOKS', 'value': 'false'}) +k8s_yaml(encode_yaml_stream(manifests)) migration = ''' set -eu diff --git a/src/helm/env.d/dev/values.dev-backend.yaml.gotmpl b/src/helm/env.d/dev/values.dev-backend.yaml.gotmpl index deaabb964..de80f325f 100644 --- a/src/helm/env.d/dev/values.dev-backend.yaml.gotmpl +++ b/src/helm/env.d/dev/values.dev-backend.yaml.gotmpl @@ -8,11 +8,8 @@ postgres: database: docs size: 1Gi redis: - enabled: true - name: redis - image: redis:8.2-alpine - username: user - password: pass + # replaced by the valkey-docs and valkey-yhub releases + enabled: false minio: enabled: true image: quay.io/minio/minio diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index ed3cf374b..24ed8d28a 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -62,8 +62,9 @@ backend: name: dev-backend-postgres key: password DB_PORT: 5432 - REDIS_URL: redis://user:pass@dev-backend-redis:6379/1 - DJANGO_CELERY_BROKER_URL: redis://user:pass@dev-backend-redis:6379/1 + # the operator keeps the readwrite service pointed at the current primary + REDIS_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0 + DJANGO_CELERY_BROKER_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0 AWS_S3_ENDPOINT_URL: http://dev-backend-minio.impress.svc.cluster.local:9000 AWS_S3_ACCESS_KEY_ID: dinum AWS_S3_SECRET_ACCESS_KEY: password @@ -212,8 +213,9 @@ yhub: # its own logical database on the dev-backend postgres: the init-db job # creates it, the backend never touches it POSTGRES: postgres://dinum:pass@dev-backend-postgres:5432/yhub - # a redis database of its own too — the backend cache and celery live in /1 - REDIS: redis://user:pass@dev-backend-redis:6379/2 + # a valkey instance of its own too — the backend cache and celery live on + # valkey-docs + REDIS: redis://user:password-1@rfr-valkey-yhub-readwrite:6379/0 REDIS_PREFIX: yhub COLLABORATION_BACKEND_BASE_URL: https://docs.127.0.0.1.nip.io COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io diff --git a/src/helm/env.d/dev/values.valkey.yaml.gotmpl b/src/helm/env.d/dev/values.valkey.yaml.gotmpl new file mode 100644 index 000000000..5b8179e8c --- /dev/null +++ b/src/helm/env.d/dev/values.valkey.yaml.gotmpl @@ -0,0 +1,3 @@ +# Both valkey-docs and valkey-yhub +storage: + capacity: 1Gi diff --git a/src/helm/env.d/feature/values.dev-backend.yaml.gotmpl b/src/helm/env.d/feature/values.dev-backend.yaml.gotmpl index 2b48547a4..e9bc9b6ff 100644 --- a/src/helm/env.d/feature/values.dev-backend.yaml.gotmpl +++ b/src/helm/env.d/feature/values.dev-backend.yaml.gotmpl @@ -8,11 +8,7 @@ postgres: database: docs size: 5Gi redis: - enabled: true - name: redis - image: redis:8.2-alpine - username: user - password: pass + enabled: false minio: enabled: true image: quay.io/minio/minio diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index be28957aa..3d93e37e2 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -62,8 +62,9 @@ backend: name: dev-backend-postgres key: password DB_PORT: 5432 - REDIS_URL: redis://user:pass@dev-backend-redis:6379/1 - DJANGO_CELERY_BROKER_URL: redis://user:pass@dev-backend-redis:6379/1 + # the operator keeps the readwrite service pointed at the current primary + REDIS_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0 + DJANGO_CELERY_BROKER_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0 AWS_S3_ENDPOINT_URL: http://dev-backend-minio.{{ .Namespace }}.svc.cluster.local:9000 AWS_S3_ACCESS_KEY_ID: dinum AWS_S3_SECRET_ACCESS_KEY: password @@ -181,7 +182,9 @@ yhub: # its own logical database on the dev-backend postgres, created by the # init-db job; redis /2, the backend cache and celery live in /1 POSTGRES: postgres://dinum:pass@dev-backend-postgres:5432/yhub - REDIS: redis://user:pass@dev-backend-redis:6379/2 + # a valkey instance of its own too — the backend cache and celery live on + # valkey-docs + REDIS: redis://user:password-1@rfr-valkey-yhub-readwrite:6379/0 REDIS_PREFIX: yhub COLLABORATION_BACKEND_BASE_URL: https://{{ .Values.feature }}-docs.{{ .Values.domain }} COLLABORATION_SERVER_ORIGIN: https://{{ .Values.feature }}-docs.{{ .Values.domain }} diff --git a/src/helm/env.d/feature/values.valkey.yaml.gotmpl b/src/helm/env.d/feature/values.valkey.yaml.gotmpl new file mode 100644 index 000000000..c2ca72123 --- /dev/null +++ b/src/helm/env.d/feature/values.valkey.yaml.gotmpl @@ -0,0 +1,3 @@ +# Both valkey-docs and valkey-yhub +storage: + capacity: 5Gi diff --git a/src/helm/extra/valkey/Chart.yaml b/src/helm/extra/valkey/Chart.yaml new file mode 100644 index 000000000..963520001 --- /dev/null +++ b/src/helm/extra/valkey/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: valkey +description: A Valkey primary with its replicas and Sentinels, managed by the valkey-operator, for local development +type: application +version: 0.1.0 diff --git a/src/helm/extra/valkey/templates/_helpers.tpl b/src/helm/extra/valkey/templates/_helpers.tpl new file mode 100644 index 000000000..e8b895849 --- /dev/null +++ b/src/helm/extra/valkey/templates/_helpers.tpl @@ -0,0 +1,16 @@ +{{/* +Spread the pods of one role across nodes when possible, instead of the +operator's hard anti-affinity. Takes a dict with the instance `name` and the +pod `type` (failover or sentinel). +*/}} +{{- define "valkey.softAntiAffinity" -}} +podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + buf.red/name: {{ .name }} + buf.red/type: {{ .type }} +{{- end }} diff --git a/src/helm/extra/valkey/templates/user.yaml b/src/helm/extra/valkey/templates/user.yaml new file mode 100644 index 000000000..09fd34b57 --- /dev/null +++ b/src/helm/extra/valkey/templates/user.yaml @@ -0,0 +1,27 @@ +{{- with .Values.user }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $.Release.Name }}-{{ .username }} + namespace: {{ $.Release.Namespace }} +type: Opaque +stringData: + password: {{ .password | quote }} +--- +apiVersion: valkey.buf.red/v1alpha1 +kind: User +metadata: + name: {{ $.Release.Name }}-{{ .username }} + namespace: {{ $.Release.Namespace }} + annotations: + # see valkey.yaml + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true +spec: + accountType: custom + arch: failover + instanceName: {{ $.Release.Name }} + username: {{ .username | quote }} + passwordSecrets: + - {{ $.Release.Name }}-{{ .username }} + aclRules: {{ .aclRules | quote }} +{{- end }} diff --git a/src/helm/extra/valkey/templates/valkey.yaml b/src/helm/extra/valkey/templates/valkey.yaml new file mode 100644 index 000000000..644d709db --- /dev/null +++ b/src/helm/extra/valkey/templates/valkey.yaml @@ -0,0 +1,49 @@ +apiVersion: rds.valkey.buf.red/v1alpha1 +kind: Valkey +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + annotations: + # argocd dry-runs the whole sync first, before the operator CRDs exist on + # a first deployment + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true +spec: + version: {{ .Values.version | quote }} + arch: failover + replicas: + shards: 1 + replicasOfShard: {{ .Values.replicas }} + resources: + {{- toYaml .Values.resources | nindent 4 }} + {{- with .Values.storage }} + storage: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.customConfigs }} + customConfigs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.softAntiAffinity }} + customAffinity: + {{- include "valkey.softAntiAffinity" (dict "name" .Release.Name "type" "failover") | nindent 4 }} + {{- end }} + sentinel: + replicas: {{ .Values.sentinel.replicas }} + {{- with .Values.sentinel.monitorConfig }} + monitorConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + resources: + {{- toYaml .Values.sentinel.resources | nindent 6 }} + {{- if .Values.softAntiAffinity }} + affinity: + {{- include "valkey.softAntiAffinity" (dict "name" .Release.Name "type" "sentinel") | nindent 6 }} + {{- end }} + access: + serviceType: ClusterIP + ipFamilyPrefer: {{ .Values.ipFamily }} + access: + serviceType: ClusterIP + ipFamilyPrefer: {{ .Values.ipFamily }} + exporter: + disable: {{ .Values.exporter.disable }} diff --git a/src/helm/extra/valkey/values.yaml b/src/helm/extra/valkey/values.yaml new file mode 100644 index 000000000..23473f96a --- /dev/null +++ b/src/helm/extra/valkey/values.yaml @@ -0,0 +1,61 @@ +# The Valkey resource is named after the release: clients reach the current +# primary through the `rfr--readwrite` service the operator keeps +# pointed at it. + +# One of the versions the operator supports +version: "9.0" + +# Valkey pods: one primary, the others replicate it +replicas: 2 + +resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 256Mi + +# The volume of each Valkey pod, passed to the operator as is: capacity, +# storageClassName (the cluster default when unset), retainAfterDeleted. +# Without storage the data lives in an emptyDir and is gone with the pod +storage: + capacity: 1Gi + +customConfigs: + # only keys with a TTL are evicted: celery queues and yhub streams have none + maxmemory-policy: volatile-lru + appendonly: "yes" + appendfsync: everysec + +# The operator requires every Valkey pod, and every Sentinel pod, on a node of +# its own, which a one-node kind cluster cannot schedule: only prefer it +softAntiAffinity: true + +# The IP family of the services. The operator compares the services it +# generates with the live ones: left empty, it never matches the family the API +# server filled in, and every reconcile updates a service and deletes its pod +ipFamily: IPv4 + +sentinel: + replicas: 3 + monitorConfig: + down-after-milliseconds: "5000" + failover-timeout: "60000" + parallel-syncs: "1" + resources: + requests: + cpu: 20m + memory: 32Mi + limits: + memory: 64Mi + +# The account the applications connect with. The operator refuses a password +# shorter than 8 characters, or without a letter, a digit and one of +# ~!@#$%^&*()-_=+? +user: + username: user + password: password-1 + aclRules: "+@all -acl -flushall -flushdb -keys ~* &*" + +exporter: + disable: true diff --git a/src/helm/helmfile.yaml.gotmpl b/src/helm/helmfile.yaml.gotmpl index ce7f2ab16..a60bd3e00 100644 --- a/src/helm/helmfile.yaml.gotmpl +++ b/src/helm/helmfile.yaml.gotmpl @@ -22,6 +22,37 @@ releases: values: - env.d/{{ .Environment.Name }}/values.dev-backend.yaml.gotmpl + # One Valkey instance for the backend (cache, sessions, celery) and + # one for yhub, each a primary, a replica and three Sentinels + - name: valkey-operator + namespace: {{ .Namespace }} + chart: git::https://github.com/chideat/valkey-operator.git@charts/valkey-operator?ref=v2.0.1 + # webhooks off: no cert-manager needed, and a User is not refused for being + # created before the operator made the Failover of its instance. The chart + # alone cannot turn them off: the ytt overlay + # extra/valkey-operator/disable-webhooks.ytt.yaml does the rest in argocd, + # bin/Tiltfile in the local stack + values: + - webhook: + enabled: false + certManager: + enabled: false + + - name: valkey-docs + namespace: {{ .Namespace }} + chart: ./extra/valkey + needs: + - valkey-operator + values: + - env.d/{{ .Environment.Name }}/values.valkey.yaml.gotmpl + + - name: valkey-yhub + namespace: {{ .Namespace }} + chart: ./extra/valkey + needs: + - valkey-operator + values: + - env.d/{{ .Environment.Name }}/values.valkey.yaml.gotmpl - name: impress version: {{ .Values.version }} diff --git a/src/helm/overlays/patch.yaml b/src/helm/overlays/patch.yaml new file mode 100644 index 000000000..313adad89 --- /dev/null +++ b/src/helm/overlays/patch.yaml @@ -0,0 +1,21 @@ +#! ytt overlay for the valkey-operator chart, applied by argocd on the rendered +#! helmfile. The chart's webhook.enabled=false only drops the webhook +#! configuration: the operator still serves its webhooks, and crashes on the +#! missing certificate unless ENABLE_WEBHOOKS=false, an env var the chart cannot +#! set. Fails when the operator Deployment or its manager container is missing, +#! rather than deploying an operator that crashes. bin/Tiltfile does the same +#! for the local stack. +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.subset({"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": "valkey-operator"}}) +--- +spec: + template: + spec: + containers: + #@overlay/match by="name" + - name: manager + env: + #@overlay/append + - name: ENABLE_WEBHOOKS + value: "false"