🔧(tilt) run two valkey sentinel instances through valkey-operator

We need to use valkey instead of an old redis version. We decided to use
the operator https://github.com/chideat/valkey-operator/ and configure
it in both feature and dev environments.

CHANGELOG.md

@@ -46,6 +46,7 @@ and this project adheres to
 - 💥(backend) move the resource server JWKS from `/api/{version}/jwks` to
   `/external_api/{version}/jwks`
 - 🔧(collaboration) adapt docker stack for development purpose
+- 🔧(tilt) run two valkey sentinel instances through valkey-operator
 
 ### Fixed
 

bin/Tiltfile

@@ -56,10 +56,44 @@ docker_build(
 
 k8s_resource('impress-docs-backend-migrate', resource_deps=['dev-backend-postgres'])
 k8s_resource('impress-docs-backend-createsuperuser', resource_deps=['impress-docs-backend-migrate'])
-k8s_resource('impress-docs-yhub-init-db', resource_deps=['dev-backend-postgres'])
+k8s_resource('impress-docs-yhub-init-db', resource_deps=['dev-backend-postgres', 'valkey-yhub'])
 k8s_resource('dev-backend-keycloak', resource_deps=['dev-backend-keycloak-pg'])
-k8s_resource('impress-docs-backend', resource_deps=['impress-docs-backend-migrate', 'dev-backend-redis', 'dev-backend-keycloak', 'dev-backend-postgres', 'dev-backend-minio:statefulset'])
-k8s_yaml(local('cd ../src/helm && helmfile -n impress -e dev template .'))
+k8s_resource('impress-docs-backend', resource_deps=['impress-docs-backend-migrate', 'valkey-docs', 'dev-backend-keycloak', 'dev-backend-postgres', 'dev-backend-minio:statefulset'])
+k8s_resource('impress-docs-celery-worker', resource_deps=['valkey-docs'])
+k8s_resource('impress-docs-yhub', resource_deps=['valkey-yhub'])
+k8s_resource('impress-docs-yhub-worker', resource_deps=['valkey-yhub'])
+
+# the operator needs its CRDs, which land in uncategorized
+k8s_resource('valkey-operator', resource_deps=['uncategorized'])
+# The pods of a Valkey instance are created by the operator: group them with the
+# objects they come from, and wait for them
+for instance in ['valkey-docs', 'valkey-yhub']:
+    k8s_resource(
+        new_name=instance,
+        objects=['%s:valkey' % instance, '%s-user:user' % instance, '%s-user:secret' % instance],
+        extra_pod_selectors=[{'buf.red/name': instance}],
+        pod_readiness='wait',
+        resource_deps=['valkey-operator'],
+    )
+
+# without --include-crds, the CRDs of the valkey-operator chart are left out
+manifests = decode_yaml_stream(local('cd ../src/helm && helmfile -n impress -e dev template --include-crds .'))
+for manifest in manifests:
+    if not manifest:
+        continue
+    # tilt down leaves the operator and its CRDs in place: deleted along with the
+    # Valkey resources, nothing would remain to clear their finalizers, and the
+    # stuck resources would block the CRDs of the next tilt up
+    if ((manifest['kind'] == 'CustomResourceDefinition' and manifest['spec']['group'].endswith('valkey.buf.red')) or
+            manifest['metadata'].get('labels', {}).get('app.kubernetes.io/instance') == 'valkey-operator'):
+        manifest['metadata'].setdefault('annotations', {})['tilt.dev/down-policy'] = 'keep'
+    # The chart's webhook.enabled=false only drops the webhook configuration: the
+    # operator still serves its webhooks and crashes on the missing certificate
+    # unless ENABLE_WEBHOOKS=false, which the chart cannot set (argocd gets it
+    # from src/helm/extra/valkey-operator/disable-webhooks.ytt.yaml)
+    if manifest['kind'] == 'Deployment' and manifest['metadata']['name'] == 'valkey-operator':
+        manifest['spec']['template']['spec']['containers'][0]['env'].append({'name': 'ENABLE_WEBHOOKS', 'value': 'false'})
+k8s_yaml(encode_yaml_stream(manifests))
 
 migration = '''
 set -eu

src/helm/env.d/dev/values.dev-backend.yaml.gotmpl

@@ -8,11 +8,8 @@ postgres:
   database: docs
   size: 1Gi
 redis:
-  enabled: true
-  name: redis
-  image: redis:8.2-alpine
-  username: user
-  password: pass
+  # replaced by the valkey-docs and valkey-yhub releases
+  enabled: false
 minio:
   enabled: true
   image: quay.io/minio/minio

src/helm/env.d/dev/values.impress.yaml.gotmpl

@@ -62,8 +62,9 @@ backend:
         name: dev-backend-postgres
         key: password
     DB_PORT: 5432
-    REDIS_URL: redis://user:pass@dev-backend-redis:6379/1
-    DJANGO_CELERY_BROKER_URL: redis://user:pass@dev-backend-redis:6379/1
+    # the operator keeps the readwrite service pointed at the current primary
+    REDIS_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0
+    DJANGO_CELERY_BROKER_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0
     AWS_S3_ENDPOINT_URL: http://dev-backend-minio.impress.svc.cluster.local:9000
     AWS_S3_ACCESS_KEY_ID: dinum
     AWS_S3_SECRET_ACCESS_KEY: password
@@ -212,8 +213,9 @@ yhub:
     # its own logical database on the dev-backend postgres: the init-db job
     # creates it, the backend never touches it
     POSTGRES: postgres://dinum:pass@dev-backend-postgres:5432/yhub
-    # a redis database of its own too — the backend cache and celery live in /1
-    REDIS: redis://user:pass@dev-backend-redis:6379/2
+    # a valkey instance of its own too — the backend cache and celery live on
+    # valkey-docs
+    REDIS: redis://user:password-1@rfr-valkey-yhub-readwrite:6379/0
     REDIS_PREFIX: yhub
     COLLABORATION_BACKEND_BASE_URL: https://docs.127.0.0.1.nip.io
     COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io

src/helm/env.d/dev/values.valkey.yaml.gotmpl

@@ -0,0 +1,3 @@
+# Both valkey-docs and valkey-yhub
+storage:
+  capacity: 1Gi

src/helm/env.d/feature/values.dev-backend.yaml.gotmpl

@@ -8,11 +8,7 @@ postgres:
   database: docs
   size: 5Gi
 redis:
-  enabled: true
-  name: redis
-  image: redis:8.2-alpine
-  username: user
-  password: pass
+  enabled: false
 minio:
   enabled: true
   image: quay.io/minio/minio

src/helm/env.d/feature/values.impress.yaml.gotmpl

@@ -62,8 +62,9 @@ backend:
         name: dev-backend-postgres
         key: password
     DB_PORT: 5432
-    REDIS_URL: redis://user:pass@dev-backend-redis:6379/1
-    DJANGO_CELERY_BROKER_URL: redis://user:pass@dev-backend-redis:6379/1
+    # the operator keeps the readwrite service pointed at the current primary
+    REDIS_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0
+    DJANGO_CELERY_BROKER_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0
     AWS_S3_ENDPOINT_URL: http://dev-backend-minio.{{ .Namespace }}.svc.cluster.local:9000
     AWS_S3_ACCESS_KEY_ID: dinum
     AWS_S3_SECRET_ACCESS_KEY: password
@@ -181,7 +182,9 @@ yhub:
     # its own logical database on the dev-backend postgres, created by the
     # init-db job; redis /2, the backend cache and celery live in /1
     POSTGRES: postgres://dinum:pass@dev-backend-postgres:5432/yhub
-    REDIS: redis://user:pass@dev-backend-redis:6379/2
+    # a valkey instance of its own too — the backend cache and celery live on
+    # valkey-docs
+    REDIS: redis://user:password-1@rfr-valkey-yhub-readwrite:6379/0
     REDIS_PREFIX: yhub
     COLLABORATION_BACKEND_BASE_URL: https://{{ .Values.feature }}-docs.{{ .Values.domain }}
     COLLABORATION_SERVER_ORIGIN: https://{{ .Values.feature }}-docs.{{ .Values.domain }}

src/helm/env.d/feature/values.valkey.yaml.gotmpl

@@ -0,0 +1,3 @@
+# Both valkey-docs and valkey-yhub
+storage:
+  capacity: 5Gi

src/helm/extra/valkey/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: valkey
+description: A Valkey primary with its replicas and Sentinels, managed by the valkey-operator, for local development
+type: application
+version: 0.1.0

src/helm/extra/valkey/templates/_helpers.tpl

@@ -0,0 +1,16 @@
+{{/*
+Spread the pods of one role across nodes when possible, instead of the
+operator's hard anti-affinity. Takes a dict with the instance `name` and the
+pod `type` (failover or sentinel).
+*/}}
+{{- define "valkey.softAntiAffinity" -}}
+podAntiAffinity:
+  preferredDuringSchedulingIgnoredDuringExecution:
+    - weight: 100
+      podAffinityTerm:
+        topologyKey: kubernetes.io/hostname
+        labelSelector:
+          matchLabels:
+            buf.red/name: {{ .name }}
+            buf.red/type: {{ .type }}
+{{- end }}

src/helm/extra/valkey/templates/user.yaml

@@ -0,0 +1,27 @@
+{{- with .Values.user }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $.Release.Name }}-{{ .username }}
+  namespace: {{ $.Release.Namespace }}
+type: Opaque
+stringData:
+  password: {{ .password | quote }}
+---
+apiVersion: valkey.buf.red/v1alpha1
+kind: User
+metadata:
+  name: {{ $.Release.Name }}-{{ .username }}
+  namespace: {{ $.Release.Namespace }}
+  annotations:
+    # see valkey.yaml
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  accountType: custom
+  arch: failover
+  instanceName: {{ $.Release.Name }}
+  username: {{ .username | quote }}
+  passwordSecrets:
+    - {{ $.Release.Name }}-{{ .username }}
+  aclRules: {{ .aclRules | quote }}
+{{- end }}

src/helm/extra/valkey/templates/valkey.yaml

@@ -0,0 +1,49 @@
+apiVersion: rds.valkey.buf.red/v1alpha1
+kind: Valkey
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    # argocd dry-runs the whole sync first, before the operator CRDs exist on
+    # a first deployment
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  version: {{ .Values.version | quote }}
+  arch: failover
+  replicas:
+    shards: 1
+    replicasOfShard: {{ .Values.replicas }}
+  resources:
+    {{- toYaml .Values.resources | nindent 4 }}
+  {{- with .Values.storage }}
+  storage:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.customConfigs }}
+  customConfigs:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- if .Values.softAntiAffinity }}
+  customAffinity:
+    {{- include "valkey.softAntiAffinity" (dict "name" .Release.Name "type" "failover") | nindent 4 }}
+  {{- end }}
+  sentinel:
+    replicas: {{ .Values.sentinel.replicas }}
+    {{- with .Values.sentinel.monitorConfig }}
+    monitorConfig:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    resources:
+      {{- toYaml .Values.sentinel.resources | nindent 6 }}
+    {{- if .Values.softAntiAffinity }}
+    affinity:
+      {{- include "valkey.softAntiAffinity" (dict "name" .Release.Name "type" "sentinel") | nindent 6 }}
+    {{- end }}
+    access:
+      serviceType: ClusterIP
+      ipFamilyPrefer: {{ .Values.ipFamily }}
+  access:
+    serviceType: ClusterIP
+    ipFamilyPrefer: {{ .Values.ipFamily }}
+  exporter:
+    disable: {{ .Values.exporter.disable }}

src/helm/extra/valkey/values.yaml

@@ -0,0 +1,61 @@
+# The Valkey resource is named after the release: clients reach the current
+# primary through the `rfr-<release>-readwrite` service the operator keeps
+# pointed at it.
+
+# One of the versions the operator supports
+version: "9.0"
+
+# Valkey pods: one primary, the others replicate it
+replicas: 2
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 128Mi
+  limits:
+    memory: 256Mi
+
+# The volume of each Valkey pod, passed to the operator as is: capacity,
+# storageClassName (the cluster default when unset), retainAfterDeleted.
+# Without storage the data lives in an emptyDir and is gone with the pod
+storage:
+  capacity: 1Gi
+
+customConfigs:
+  # only keys with a TTL are evicted: celery queues and yhub streams have none
+  maxmemory-policy: volatile-lru
+  appendonly: "yes"
+  appendfsync: everysec
+
+# The operator requires every Valkey pod, and every Sentinel pod, on a node of
+# its own, which a one-node kind cluster cannot schedule: only prefer it
+softAntiAffinity: true
+
+# The IP family of the services. The operator compares the services it
+# generates with the live ones: left empty, it never matches the family the API
+# server filled in, and every reconcile updates a service and deletes its pod
+ipFamily: IPv4
+
+sentinel:
+  replicas: 3
+  monitorConfig:
+    down-after-milliseconds: "5000"
+    failover-timeout: "60000"
+    parallel-syncs: "1"
+  resources:
+    requests:
+      cpu: 20m
+      memory: 32Mi
+    limits:
+      memory: 64Mi
+
+# The account the applications connect with. The operator refuses a password
+# shorter than 8 characters, or without a letter, a digit and one of
+# ~!@#$%^&*()-_=+?
+user:
+  username: user
+  password: password-1
+  aclRules: "+@all -acl -flushall -flushdb -keys ~* &*"
+
+exporter:
+  disable: true

src/helm/helmfile.yaml.gotmpl

@@ -22,6 +22,37 @@ releases:
     values:
       - env.d/{{ .Environment.Name }}/values.dev-backend.yaml.gotmpl
 
+  # One Valkey instance for the backend (cache, sessions, celery) and
+  # one for yhub, each a primary, a replica and three Sentinels
+  - name: valkey-operator
+    namespace: {{ .Namespace }}
+    chart: git::https://github.com/chideat/valkey-operator.git@charts/valkey-operator?ref=v2.0.1
+    # webhooks off: no cert-manager needed, and a User is not refused for being
+    # created before the operator made the Failover of its instance. The chart
+    # alone cannot turn them off: the ytt overlay
+    # extra/valkey-operator/disable-webhooks.ytt.yaml does the rest in argocd,
+    # bin/Tiltfile in the local stack
+    values:
+      - webhook:
+          enabled: false
+        certManager:
+          enabled: false
+
+  - name: valkey-docs
+    namespace: {{ .Namespace }}
+    chart: ./extra/valkey
+    needs:
+      - valkey-operator
+    values:
+      - env.d/{{ .Environment.Name }}/values.valkey.yaml.gotmpl
+
+  - name: valkey-yhub
+    namespace: {{ .Namespace }}
+    chart: ./extra/valkey
+    needs:
+      - valkey-operator
+    values:
+      - env.d/{{ .Environment.Name }}/values.valkey.yaml.gotmpl
 
   - name: impress
     version: {{ .Values.version }}

src/helm/overlays/patch.yaml

@@ -0,0 +1,21 @@
+#! ytt overlay for the valkey-operator chart, applied by argocd on the rendered
+#! helmfile. The chart's webhook.enabled=false only drops the webhook
+#! configuration: the operator still serves its webhooks, and crashes on the
+#! missing certificate unless ENABLE_WEBHOOKS=false, an env var the chart cannot
+#! set. Fails when the operator Deployment or its manager container is missing,
+#! rather than deploying an operator that crashes. bin/Tiltfile does the same
+#! for the local stack.
+#@ load("@ytt:overlay", "overlay")
+
+#@overlay/match by=overlay.subset({"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": "valkey-operator"}})
+---
+spec:
+  template:
+    spec:
+      containers:
+      #@overlay/match by="name"
+      - name: manager
+        env:
+        #@overlay/append
+        - name: ENABLE_WEBHOOKS
+          value: "false"