diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fcd58b49..5e8a1482c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,7 @@ and this project adheres to - 💥(backend) move the resource server JWKS from `/api/{version}/jwks` to `/external_api/{version}/jwks` - 🔧(collaboration) adapt docker stack for development purpose -- 🔧(tilt) run two valkey sentinel instances through valkey-operator +- 🔧(helm) run a valkey for the backend and one for yhub in dev and feature ### Fixed diff --git a/bin/Tiltfile b/bin/Tiltfile index 4c7c28b67..86e824cc6 100644 --- a/bin/Tiltfile +++ b/bin/Tiltfile @@ -63,37 +63,8 @@ k8s_resource('impress-docs-celery-worker', resource_deps=['valkey-docs']) k8s_resource('impress-docs-yhub', resource_deps=['valkey-yhub']) k8s_resource('impress-docs-yhub-worker', resource_deps=['valkey-yhub']) -# the operator needs its CRDs, which land in uncategorized -k8s_resource('valkey-operator', resource_deps=['uncategorized']) -# The pods of a Valkey instance are created by the operator: group them with the -# objects they come from, and wait for them -for instance in ['valkey-docs', 'valkey-yhub']: - k8s_resource( - new_name=instance, - objects=['%s:valkey' % instance, '%s-user:user' % instance, '%s-user:secret' % instance], - extra_pod_selectors=[{'buf.red/name': instance}], - pod_readiness='wait', - resource_deps=['valkey-operator'], - ) - -# without --include-crds, the CRDs of the valkey-operator chart are left out -manifests = decode_yaml_stream(local('cd ../src/helm && helmfile -n impress -e dev template --include-crds .')) -for manifest in manifests: - if not manifest: - continue - # tilt down leaves the operator and its CRDs in place: deleted along with the - # Valkey resources, nothing would remain to clear their finalizers, and the - # stuck resources would block the CRDs of the next tilt up - if ((manifest['kind'] == 'CustomResourceDefinition' and manifest['spec']['group'].endswith('valkey.buf.red')) or - manifest['metadata'].get('labels', {}).get('app.kubernetes.io/instance') == 'valkey-operator'): - manifest['metadata'].setdefault('annotations', {})['tilt.dev/down-policy'] = 'keep' - # The chart's webhook.enabled=false only drops the webhook configuration: the - # operator still serves its webhooks and crashes on the missing certificate - # unless ENABLE_WEBHOOKS=false, which the chart cannot set (argocd gets it - # from src/helm/extra/valkey-operator/disable-webhooks.ytt.yaml) - if manifest['kind'] == 'Deployment' and manifest['metadata']['name'] == 'valkey-operator': - manifest['spec']['template']['spec']['containers'][0]['env'].append({'name': 'ENABLE_WEBHOOKS', 'value': 'false'}) -k8s_yaml(encode_yaml_stream(manifests)) +# the valkey chart leaves its objects without a namespace +k8s_yaml(namespace_inject(local('cd ../src/helm && helmfile -n impress -e dev template .'), 'impress')) migration = ''' set -eu diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index 24ed8d28a..af89e7943 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -62,9 +62,8 @@ backend: name: dev-backend-postgres key: password DB_PORT: 5432 - # the operator keeps the readwrite service pointed at the current primary - REDIS_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0 - DJANGO_CELERY_BROKER_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0 + REDIS_URL: redis://default:pass@valkey-docs:6379/0 + DJANGO_CELERY_BROKER_URL: redis://default:pass@valkey-docs:6379/0 AWS_S3_ENDPOINT_URL: http://dev-backend-minio.impress.svc.cluster.local:9000 AWS_S3_ACCESS_KEY_ID: dinum AWS_S3_SECRET_ACCESS_KEY: password @@ -215,7 +214,7 @@ yhub: POSTGRES: postgres://dinum:pass@dev-backend-postgres:5432/yhub # a valkey instance of its own too — the backend cache and celery live on # valkey-docs - REDIS: redis://user:password-1@rfr-valkey-yhub-readwrite:6379/0 + REDIS: redis://default:pass@valkey-yhub:6379/0 REDIS_PREFIX: yhub COLLABORATION_BACKEND_BASE_URL: https://docs.127.0.0.1.nip.io COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io diff --git a/src/helm/env.d/dev/values.valkey.yaml.gotmpl b/src/helm/env.d/dev/values.valkey.yaml.gotmpl index 5b8179e8c..5791edd50 100644 --- a/src/helm/env.d/dev/values.valkey.yaml.gotmpl +++ b/src/helm/env.d/dev/values.valkey.yaml.gotmpl @@ -1,3 +1,12 @@ -# Both valkey-docs and valkey-yhub -storage: - capacity: 1Gi +valkeyConfig: | + # only keys with a TTL are evicted: celery queues and yhub streams have none + maxmemory-policy volatile-lru + appendonly yes + appendfsync everysec + +auth: + enabled: true + # an ACL line rather than aclUsers, whose passwords make the chart render a + # helm test pod that Tilt would run as a plain pod + aclConfig: | + user default on >pass ~* &* +@all diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index 3d93e37e2..1e0d42e61 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -62,9 +62,8 @@ backend: name: dev-backend-postgres key: password DB_PORT: 5432 - # the operator keeps the readwrite service pointed at the current primary - REDIS_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0 - DJANGO_CELERY_BROKER_URL: redis://user:password-1@rfr-valkey-docs-readwrite:6379/0 + REDIS_URL: redis://default:pass@valkey-docs:6379/0 + DJANGO_CELERY_BROKER_URL: redis://default:pass@valkey-docs:6379/0 AWS_S3_ENDPOINT_URL: http://dev-backend-minio.{{ .Namespace }}.svc.cluster.local:9000 AWS_S3_ACCESS_KEY_ID: dinum AWS_S3_SECRET_ACCESS_KEY: password @@ -184,7 +183,7 @@ yhub: POSTGRES: postgres://dinum:pass@dev-backend-postgres:5432/yhub # a valkey instance of its own too — the backend cache and celery live on # valkey-docs - REDIS: redis://user:password-1@rfr-valkey-yhub-readwrite:6379/0 + REDIS: redis://default:pass@valkey-yhub:6379/0 REDIS_PREFIX: yhub COLLABORATION_BACKEND_BASE_URL: https://{{ .Values.feature }}-docs.{{ .Values.domain }} COLLABORATION_SERVER_ORIGIN: https://{{ .Values.feature }}-docs.{{ .Values.domain }} diff --git a/src/helm/env.d/feature/values.valkey.yaml.gotmpl b/src/helm/env.d/feature/values.valkey.yaml.gotmpl index c2ca72123..5791edd50 100644 --- a/src/helm/env.d/feature/values.valkey.yaml.gotmpl +++ b/src/helm/env.d/feature/values.valkey.yaml.gotmpl @@ -1,3 +1,12 @@ -# Both valkey-docs and valkey-yhub -storage: - capacity: 5Gi +valkeyConfig: | + # only keys with a TTL are evicted: celery queues and yhub streams have none + maxmemory-policy volatile-lru + appendonly yes + appendfsync everysec + +auth: + enabled: true + # an ACL line rather than aclUsers, whose passwords make the chart render a + # helm test pod that Tilt would run as a plain pod + aclConfig: | + user default on >pass ~* &* +@all diff --git a/src/helm/extra/valkey/Chart.yaml b/src/helm/extra/valkey/Chart.yaml deleted file mode 100644 index 963520001..000000000 --- a/src/helm/extra/valkey/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -name: valkey -description: A Valkey primary with its replicas and Sentinels, managed by the valkey-operator, for local development -type: application -version: 0.1.0 diff --git a/src/helm/extra/valkey/templates/_helpers.tpl b/src/helm/extra/valkey/templates/_helpers.tpl deleted file mode 100644 index e8b895849..000000000 --- a/src/helm/extra/valkey/templates/_helpers.tpl +++ /dev/null @@ -1,16 +0,0 @@ -{{/* -Spread the pods of one role across nodes when possible, instead of the -operator's hard anti-affinity. Takes a dict with the instance `name` and the -pod `type` (failover or sentinel). -*/}} -{{- define "valkey.softAntiAffinity" -}} -podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - topologyKey: kubernetes.io/hostname - labelSelector: - matchLabels: - buf.red/name: {{ .name }} - buf.red/type: {{ .type }} -{{- end }} diff --git a/src/helm/extra/valkey/templates/user.yaml b/src/helm/extra/valkey/templates/user.yaml deleted file mode 100644 index 09fd34b57..000000000 --- a/src/helm/extra/valkey/templates/user.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{- with .Values.user }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ $.Release.Name }}-{{ .username }} - namespace: {{ $.Release.Namespace }} -type: Opaque -stringData: - password: {{ .password | quote }} ---- -apiVersion: valkey.buf.red/v1alpha1 -kind: User -metadata: - name: {{ $.Release.Name }}-{{ .username }} - namespace: {{ $.Release.Namespace }} - annotations: - # see valkey.yaml - argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true -spec: - accountType: custom - arch: failover - instanceName: {{ $.Release.Name }} - username: {{ .username | quote }} - passwordSecrets: - - {{ $.Release.Name }}-{{ .username }} - aclRules: {{ .aclRules | quote }} -{{- end }} diff --git a/src/helm/extra/valkey/templates/valkey.yaml b/src/helm/extra/valkey/templates/valkey.yaml deleted file mode 100644 index 644d709db..000000000 --- a/src/helm/extra/valkey/templates/valkey.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: rds.valkey.buf.red/v1alpha1 -kind: Valkey -metadata: - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - annotations: - # argocd dry-runs the whole sync first, before the operator CRDs exist on - # a first deployment - argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true -spec: - version: {{ .Values.version | quote }} - arch: failover - replicas: - shards: 1 - replicasOfShard: {{ .Values.replicas }} - resources: - {{- toYaml .Values.resources | nindent 4 }} - {{- with .Values.storage }} - storage: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.customConfigs }} - customConfigs: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if .Values.softAntiAffinity }} - customAffinity: - {{- include "valkey.softAntiAffinity" (dict "name" .Release.Name "type" "failover") | nindent 4 }} - {{- end }} - sentinel: - replicas: {{ .Values.sentinel.replicas }} - {{- with .Values.sentinel.monitorConfig }} - monitorConfig: - {{- toYaml . | nindent 6 }} - {{- end }} - resources: - {{- toYaml .Values.sentinel.resources | nindent 6 }} - {{- if .Values.softAntiAffinity }} - affinity: - {{- include "valkey.softAntiAffinity" (dict "name" .Release.Name "type" "sentinel") | nindent 6 }} - {{- end }} - access: - serviceType: ClusterIP - ipFamilyPrefer: {{ .Values.ipFamily }} - access: - serviceType: ClusterIP - ipFamilyPrefer: {{ .Values.ipFamily }} - exporter: - disable: {{ .Values.exporter.disable }} diff --git a/src/helm/extra/valkey/values.yaml b/src/helm/extra/valkey/values.yaml deleted file mode 100644 index 23473f96a..000000000 --- a/src/helm/extra/valkey/values.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# The Valkey resource is named after the release: clients reach the current -# primary through the `rfr--readwrite` service the operator keeps -# pointed at it. - -# One of the versions the operator supports -version: "9.0" - -# Valkey pods: one primary, the others replicate it -replicas: 2 - -resources: - requests: - cpu: 50m - memory: 128Mi - limits: - memory: 256Mi - -# The volume of each Valkey pod, passed to the operator as is: capacity, -# storageClassName (the cluster default when unset), retainAfterDeleted. -# Without storage the data lives in an emptyDir and is gone with the pod -storage: - capacity: 1Gi - -customConfigs: - # only keys with a TTL are evicted: celery queues and yhub streams have none - maxmemory-policy: volatile-lru - appendonly: "yes" - appendfsync: everysec - -# The operator requires every Valkey pod, and every Sentinel pod, on a node of -# its own, which a one-node kind cluster cannot schedule: only prefer it -softAntiAffinity: true - -# The IP family of the services. The operator compares the services it -# generates with the live ones: left empty, it never matches the family the API -# server filled in, and every reconcile updates a service and deletes its pod -ipFamily: IPv4 - -sentinel: - replicas: 3 - monitorConfig: - down-after-milliseconds: "5000" - failover-timeout: "60000" - parallel-syncs: "1" - resources: - requests: - cpu: 20m - memory: 32Mi - limits: - memory: 64Mi - -# The account the applications connect with. The operator refuses a password -# shorter than 8 characters, or without a letter, a digit and one of -# ~!@#$%^&*()-_=+? -user: - username: user - password: password-1 - aclRules: "+@all -acl -flushall -flushdb -keys ~* &*" - -exporter: - disable: true diff --git a/src/helm/helmfile.yaml.gotmpl b/src/helm/helmfile.yaml.gotmpl index a60bd3e00..cfb4c538c 100644 --- a/src/helm/helmfile.yaml.gotmpl +++ b/src/helm/helmfile.yaml.gotmpl @@ -12,6 +12,8 @@ environments: repositories: - name: dev-backends url: https://suitenumerique.github.io/helm-dev-backend +- name: valkey + url: https://valkey.io/valkey-helm/ --- releases: @@ -22,35 +24,19 @@ releases: values: - env.d/{{ .Environment.Name }}/values.dev-backend.yaml.gotmpl - # One Valkey instance for the backend (cache, sessions, celery) and - # one for yhub, each a primary, a replica and three Sentinels - - name: valkey-operator - namespace: {{ .Namespace }} - chart: git::https://github.com/chideat/valkey-operator.git@charts/valkey-operator?ref=v2.0.1 - # webhooks off: no cert-manager needed, and a User is not refused for being - # created before the operator made the Failover of its instance. The chart - # alone cannot turn them off: the ytt overlay - # extra/valkey-operator/disable-webhooks.ytt.yaml does the rest in argocd, - # bin/Tiltfile in the local stack - values: - - webhook: - enabled: false - certManager: - enabled: false - + # One Valkey instance for the backend (cache, sessions, celery) and one for + # yhub, each a single standalone pod - name: valkey-docs namespace: {{ .Namespace }} - chart: ./extra/valkey - needs: - - valkey-operator + chart: valkey/valkey + version: 0.12.0 values: - env.d/{{ .Environment.Name }}/values.valkey.yaml.gotmpl - name: valkey-yhub namespace: {{ .Namespace }} - chart: ./extra/valkey - needs: - - valkey-operator + chart: valkey/valkey + version: 0.12.0 values: - env.d/{{ .Environment.Name }}/values.valkey.yaml.gotmpl diff --git a/src/helm/overlays/patch.yaml b/src/helm/overlays/patch.yaml deleted file mode 100644 index 313adad89..000000000 --- a/src/helm/overlays/patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -#! ytt overlay for the valkey-operator chart, applied by argocd on the rendered -#! helmfile. The chart's webhook.enabled=false only drops the webhook -#! configuration: the operator still serves its webhooks, and crashes on the -#! missing certificate unless ENABLE_WEBHOOKS=false, an env var the chart cannot -#! set. Fails when the operator Deployment or its manager container is missing, -#! rather than deploying an operator that crashes. bin/Tiltfile does the same -#! for the local stack. -#@ load("@ytt:overlay", "overlay") - -#@overlay/match by=overlay.subset({"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": "valkey-operator"}}) ---- -spec: - template: - spec: - containers: - #@overlay/match by="name" - - name: manager - env: - #@overlay/append - - name: ENABLE_WEBHOOKS - value: "false"