From c7102bf0a0f498e43e20ed4a59a2296c98df3ad7 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 23 Sep 2026 09:46:39 +0200 Subject: [PATCH] =?UTF-8?q?=20=E2=9C=A8(loadtest)=20add=20the=20grafana=20?= =?UTF-8?q?dashboards=20of=20the=20load-test=20campaign?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To see what is happening in the cluster we added grafana dashboards using all the metrics introduced previsouly --- CHANGELOG.md | 4 + bin/Tiltfile | 18 +- documentation/metrics.md | 14 +- documentation/stress-test-plan.md | 35 +- src/helm/env.d/dev/prometheus/django.html | 201 --- src/helm/env.d/dev/prometheus/django.rules | 108 -- .../env.d/dev/values.prometheus.yaml.gotmpl | 83 +- src/helm/env.d/dev/values.valkey.yaml.gotmpl | 16 + src/helm/helmfile.yaml.gotmpl | 10 + src/loadtest/dashboards/README.md | 88 + src/loadtest/dashboards/backend.json | 1101 ++++++++++++ src/loadtest/dashboards/collaboration.json | 1234 +++++++++++++ src/loadtest/dashboards/django.json | 1588 +++++++++++++++++ src/loadtest/dashboards/generate.py | 234 +++ src/loadtest/dashboards/users.json | 625 +++++++ src/loadtest/dashboards/valkey.json | 1533 ++++++++++++++++ 16 files changed, 6521 insertions(+), 371 deletions(-) delete mode 100644 src/helm/env.d/dev/prometheus/django.html delete mode 100644 src/helm/env.d/dev/prometheus/django.rules create mode 100644 src/loadtest/dashboards/README.md create mode 100644 src/loadtest/dashboards/backend.json create mode 100644 src/loadtest/dashboards/collaboration.json create mode 100644 src/loadtest/dashboards/django.json create mode 100644 src/loadtest/dashboards/generate.py create mode 100644 src/loadtest/dashboards/users.json create mode 100644 src/loadtest/dashboards/valkey.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 505b5d450..5c3083d6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ and this project adheres to - ✨(loadtest) add k6 scenarios for the page-open sequence and the heavy endpoints - ✨(loadtest) add browser canaries measuring what a user feels under load +- ✨(loadtest) add the grafana dashboards of the load-test campaign, valkey + included +- 🔧(helm) run grafana with those dashboards in the dev cluster, in place of + the prometheus console - ✨(backend) measure the calls to yhub and to the converters, the database pool and the celery queue - ✨(backend) add a `LoadTest` configuration and its `loadtest` application, diff --git a/bin/Tiltfile b/bin/Tiltfile index ffdbe5f2e..55cd7f255 100644 --- a/bin/Tiltfile +++ b/bin/Tiltfile @@ -84,17 +84,27 @@ k8s_resource('prometheus-operator', objects=[ ]) k8s_resource('prometheus-prometheus', objects=[ 'prometheus-prometheus:ingress', - # the example console of django-prometheus and the rules feeding it - 'django-console:configmap', - 'django:prometheusrule', ], resource_deps=['prometheus-operator'], links=[ 'https://docs-prometheus.127.0.0.1.nip.io/targets', - 'https://docs-prometheus.127.0.0.1.nip.io/consoles/django.html', +]) +# Grafana, with the dashboards of the load-test campaign +# (src/loadtest/dashboards) as ConfigMaps its sidecar loads +k8s_resource('prometheus-grafana', objects=[ + 'prometheus-grafana:ingress', + 'dashboard-users:configmap', + 'dashboard-collaboration:configmap', + 'dashboard-backend:configmap', + 'dashboard-django:configmap', + 'dashboard-valkey:configmap', +], resource_deps=['prometheus-prometheus'], links=[ + 'https://docs-grafana.127.0.0.1.nip.io/dashboards', ]) k8s_resource(new_name='impress-docs-monitors', objects=[ 'impress-docs-backend:servicemonitor', 'impress-docs-yhub:servicemonitor', 'impress-docs-yhub-worker:servicemonitor', + 'valkey-docs:servicemonitor', + 'valkey-yhub:servicemonitor', ], resource_deps=['prometheus-operator']) # the valkey chart leaves its objects without a namespace, and the CRDs of the diff --git a/documentation/metrics.md b/documentation/metrics.md index cd630e9d1..d08770ffa 100644 --- a/documentation/metrics.md +++ b/documentation/metrics.md @@ -220,14 +220,12 @@ on with a token in the `docs-metrics` Secret, and `serviceMonitor.enabled` builds the three monitors. The targets are at https://docs-prometheus.127.0.0.1.nip.io/targets. -That Prometheus also serves the -[example console of django-prometheus](https://github.com/django-commons/django-prometheus/tree/master/examples/prometheus) -at https://docs-prometheus.127.0.0.1.nip.io/consoles/django.html: requests per -second, by view, median and tail latency, model writes and database queries, -drawn from its recording rules. Both files are in -`src/helm/env.d/dev/prometheus/`, the rules verbatim and the console with its -job renamed to `backend`. Console templates draw with the classic UI, which -Prometheus 3 removed, so the dev Prometheus is the last 2.x release. +Next to it, a Grafana at https://docs-grafana.127.0.0.1.nip.io (admin / +admin) with the dashboards of the load-test campaign, from +`src/loadtest/dashboards/` (see its README): what the users feel, the +collaboration server, the backend, and the generic +[Django dashboard](https://grafana.com/grafana/dashboards/17658-django/) +of the community for the per-view detail of django-prometheus. ### A Prometheus outside of the cluster diff --git a/documentation/stress-test-plan.md b/documentation/stress-test-plan.md index bcb880f62..711ac4db1 100644 --- a/documentation/stress-test-plan.md +++ b/documentation/stress-test-plan.md @@ -295,17 +295,32 @@ Rules for the load clients: ### 0.5 Dashboards -One board per question, built before the first run: +**Implemented** (2026-09-22): `src/loadtest/dashboards/` (see its README), +loaded into the Grafana of the dev cluster +(`src/helm/env.d/dev/values.prometheus.yaml.gotmpl`, Grafana 13, Prometheus +3.14, in place of the django-prometheus console). Every panel query was +checked against live data from the backend, yhub, the swarm, k6 and the canary. -- users: `canary_page_open_seconds`, `canary_editor_ready_seconds`, - `canary_propagation_seconds`, `canary_failures_total{step}` (from - `src/loadtest/canary/`); -- yhub: sockets and rooms per pod, event-loop lag, auth duration, backend - call duration and inflight, pending tasks, task duration, seeds; -- Django: latency and rate per view, yhub client latency, pool waiting, - Celery queue length; -- stores: Postgres connections and top queries, Valkey memory, commands and - evictions. +- users: `users.json` — the canary (page open, editor ready, keystroke to the + other screen, failures by step), plus the swarm's connect and propagation + and k6's latency and failures; +- yhub: `collaboration.json` — sockets and rooms per replica, event-loop lag, + auth duration and results, backend calls (duration, in flight, failures), + compaction backlog and duration, seeds; then the swarm's view; +- Django: `backend.json` — requests and latency by view, 5xx, calls to yhub + and the converters, the psycopg pool, queries, Celery queue length; then + k6's view. `django.json` is the community Django dashboard (17658) for the + per-view detail; +- Valkey: `valkey.json` — the two instances through the operator's + `redis_exporter`: memory against `maxmemory`, evictions, commands and their + latency, network, CPU, the yhub streams, replication, Sentinel. The streams + row needs `--check-streams`, which the operator's exporter spec cannot pass + (see the README); +- Postgres: not here, the exporter belongs to the team running it. The README + lists the queries the campaign needs from it. + +Pod CPU and memory come from the cluster's cAdvisor, not from these boards: +the backend exports no process metrics in multiprocess mode. ## 1. Tooling diff --git a/src/helm/env.d/dev/prometheus/django.html b/src/helm/env.d/dev/prometheus/django.html deleted file mode 100644 index ac48b362f..000000000 --- a/src/helm/env.d/dev/prometheus/django.html +++ /dev/null @@ -1,201 +0,0 @@ -{{/* - The example console of django-prometheus, from - https://github.com/django-commons/django-prometheus/blob/master/examples/prometheus/consoles/django.html - with two changes: the job is `backend` here (the ServiceMonitor names it after - the component), and the connection errors graph read a metric that does not - exist (`erros`). The graphs come from the recording rules of django.rules. -*/}} -{{template "head" .}} - -{{template "prom_right_table_head"}} - - Django - {{ template "prom_query_drilldown" (args "sum(up{job='backend'})") }} - / {{ template "prom_query_drilldown" (args "count(up{job='backend'})") }} - - - - avg CPU - {{ template "prom_query_drilldown" (args "avg by(job)(rate(process_cpu_seconds_total{job='backend'}[5m]))" "s/s" "humanizeNoSmallPrefix") }} - - - - avg Memory - {{ template "prom_query_drilldown" (args "avg by(job)(process_resident_memory_bytes{job='backend'})" "B" "humanize1024") }} - - -{{template "prom_right_table_tail"}} - - -{{template "prom_content_head" .}} -

Django

- -

Requests

-

Total

-
- - -

By view

-
- - -

Latency (median)

-
- - -

Latency (99.9th percentile)

-
- - -

Models

-

Insertions/s

-
- - -

Updates/s

-
- - -

Deletions/s

-
- - -

Database

-

Connections/s

-
- - -

Connections errors/s

-
- - -

Queries/s

-
- - -

Errors/s

-
- - -{{template "prom_content_tail" .}} - -{{template "tail"}} diff --git a/src/helm/env.d/dev/prometheus/django.rules b/src/helm/env.d/dev/prometheus/django.rules deleted file mode 100644 index 54209951e..000000000 --- a/src/helm/env.d/dev/prometheus/django.rules +++ /dev/null @@ -1,108 +0,0 @@ -# The example recording rules of django-prometheus, verbatim, from -# https://github.com/django-commons/django-prometheus/blob/master/examples/prometheus/django.rules -# They are what the graphs of django.html read. -groups: -- name: django.rules - rules: - - record: job:django_http_requests_before_middlewares_total:sum_rate30s - expr: sum(rate(django_http_requests_before_middlewares_total[30s])) BY (job) - - record: job:django_http_requests_unknown_latency_total:sum_rate30s - expr: sum(rate(django_http_requests_unknown_latency_total[30s])) BY (job) - - record: job:django_http_ajax_requests_total:sum_rate30s - expr: sum(rate(django_http_ajax_requests_total[30s])) BY (job) - - record: job:django_http_responses_before_middlewares_total:sum_rate30s - expr: sum(rate(django_http_responses_before_middlewares_total[30s])) BY (job) - - record: job:django_http_requests_unknown_latency_including_middlewares_total:sum_rate30s - expr: sum(rate(django_http_requests_unknown_latency_including_middlewares_total[30s])) - BY (job) - - record: job:django_http_requests_body_total_bytes:sum_rate30s - expr: sum(rate(django_http_requests_body_total_bytes[30s])) BY (job) - - record: job:django_http_responses_streaming_total:sum_rate30s - expr: sum(rate(django_http_responses_streaming_total[30s])) BY (job) - - record: job:django_http_responses_body_total_bytes:sum_rate30s - expr: sum(rate(django_http_responses_body_total_bytes[30s])) BY (job) - - record: job:django_http_requests_total:sum_rate30s - expr: sum(rate(django_http_requests_total_by_method[30s])) BY (job) - - record: job:django_http_requests_total_by_method:sum_rate30s - expr: sum(rate(django_http_requests_total_by_method[30s])) BY (job, method) - - record: job:django_http_requests_total_by_transport:sum_rate30s - expr: sum(rate(django_http_requests_total_by_transport[30s])) BY (job, transport) - - record: job:django_http_requests_total_by_view:sum_rate30s - expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) BY (job, - view) - - record: job:django_http_requests_total_by_view_transport_method:sum_rate30s - expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) BY (job, - view, transport, method) - - record: job:django_http_responses_total_by_templatename:sum_rate30s - expr: sum(rate(django_http_responses_total_by_templatename[30s])) BY (job, templatename) - - record: job:django_http_responses_total_by_status:sum_rate30s - expr: sum(rate(django_http_responses_total_by_status[30s])) BY (job, status) - - record: job:django_http_responses_total_by_status_name_method:sum_rate30s - expr: sum(rate(django_http_responses_total_by_status_name_method[30s])) BY (job, - status, name, method) - - record: job:django_http_responses_total_by_charset:sum_rate30s - expr: sum(rate(django_http_responses_total_by_charset[30s])) BY (job, charset) - - record: job:django_http_exceptions_total_by_type:sum_rate30s - expr: sum(rate(django_http_exceptions_total_by_type[30s])) BY (job, type) - - record: job:django_http_exceptions_total_by_view:sum_rate30s - expr: sum(rate(django_http_exceptions_total_by_view[30s])) BY (job, view) - - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s - expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "50" - - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s - expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "95" - - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s - expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "99" - - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s - expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "99.9" - - record: job:django_http_requests_latency_seconds:quantile_rate30s - expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "50" - - record: job:django_http_requests_latency_seconds:quantile_rate30s - expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "95" - - record: job:django_http_requests_latency_seconds:quantile_rate30s - expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "99" - - record: job:django_http_requests_latency_seconds:quantile_rate30s - expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "99.9" - - record: job:django_model_inserts_total:sum_rate1m - expr: sum(rate(django_model_inserts_total[1m])) BY (job, model) - - record: job:django_model_updates_total:sum_rate1m - expr: sum(rate(django_model_updates_total[1m])) BY (job, model) - - record: job:django_model_deletes_total:sum_rate1m - expr: sum(rate(django_model_deletes_total[1m])) BY (job, model) - - record: job:django_db_new_connections_total:sum_rate30s - expr: sum(rate(django_db_new_connections_total[30s])) BY (alias, vendor) - - record: job:django_db_new_connection_errors_total:sum_rate30s - expr: sum(rate(django_db_new_connection_errors_total[30s])) BY (alias, vendor) - - record: job:django_db_execute_total:sum_rate30s - expr: sum(rate(django_db_execute_total[30s])) BY (alias, vendor) - - record: job:django_db_execute_many_total:sum_rate30s - expr: sum(rate(django_db_execute_many_total[30s])) BY (alias, vendor) - - record: job:django_db_errors_total:sum_rate30s - expr: sum(rate(django_db_errors_total[30s])) BY (alias, vendor, type) - - record: job:django_migrations_applied_total:max - expr: max(django_migrations_applied_total) BY (job, connection) - - record: job:django_migrations_unapplied_total:max - expr: max(django_migrations_unapplied_total) BY (job, connection) diff --git a/src/helm/env.d/dev/values.prometheus.yaml.gotmpl b/src/helm/env.d/dev/values.prometheus.yaml.gotmpl index 448a9243e..90225cdb9 100644 --- a/src/helm/env.d/dev/values.prometheus.yaml.gotmpl +++ b/src/helm/env.d/dev/values.prometheus.yaml.gotmpl @@ -1,15 +1,11 @@ -# A Prometheus scraping the dev cluster, to see the metrics of the backend and -# of yhub through the ServiceMonitors the impress chart builds. Trimmed down to -# the Prometheus Operator, its CRDs and one Prometheus: no Grafana, no -# Alertmanager, no exporter, and none of the monitors and rules of the -# kubernetes control plane, which a kind cluster has nothing to show for. -# UI: https://docs-prometheus.127.0.0.1.nip.io (Status > Targets lists the pods) -# -# It also serves the example console of django-prometheus, the graphs of -# https://docs-prometheus.127.0.0.1.nip.io/consoles/django.html, fed by its -# recording rules: both are in the prometheus/ directory next to this file. -# Console templates draw with the classic UI, which Prometheus 3 removed along -# with the console libraries, so this Prometheus is the last 2.x release. +# A Prometheus and a Grafana for the dev cluster, to see the metrics of the +# backend and of yhub through the ServiceMonitors the impress chart builds, and +# the dashboards of the load-test campaign (src/loadtest/dashboards). Trimmed +# down to the Prometheus Operator, its CRDs, one Prometheus and one Grafana: no +# Alertmanager, no exporter, and none of the monitors, rules and dashboards of +# the kubernetes control plane, which a kind cluster has nothing to show for. +# Prometheus: https://docs-prometheus.127.0.0.1.nip.io (Status > Targets) +# Grafana: https://docs-grafana.127.0.0.1.nip.io (admin / admin) fullnameOverride: prometheus defaultRules: @@ -17,7 +13,25 @@ defaultRules: alertmanager: enabled: false grafana: - enabled: false + enabled: true + adminPassword: admin + # none of the kubernetes dashboards of the chart: the boards of the campaign + # only, loaded from the ConfigMaps below by the sidecar + defaultDashboardsEnabled: false + sidecar: + dashboards: + folderAnnotation: grafana_folder + provider: + foldersFromFilesStructure: true + ingress: + enabled: true + hosts: + - docs-grafana.127.0.0.1.nip.io + path: / + tls: + - secretName: docs-tls + hosts: + - docs-grafana.127.0.0.1.nip.io nodeExporter: enabled: false kubeStateMetrics: @@ -57,19 +71,6 @@ prometheus: serviceMonitor: selfMonitor: false prometheusSpec: - image: - tag: v2.55.1 - # the console, in the directory the operator points --web.console.templates - # at (/etc/prometheus/consoles is a symlink to it in the image). Only the - # console libraries of the image are left next to it, which is all it needs - volumes: - - name: django-console - configMap: - name: django-console - volumeMounts: - - name: django-console - mountPath: /usr/share/prometheus/consoles - readOnly: true # every ServiceMonitor and PodMonitor of the namespace, whatever its labels serviceMonitorSelectorNilUsesHelmValues: false podMonitorSelectorNilUsesHelmValues: false @@ -107,23 +108,25 @@ extraManifests: name: docs-metrics stringData: PROMETHEUS_API_KEY: ThisIsAnExampleMetricsTokenForDevPurposeOnly - # The recording rules the console reads, selected like the monitors are - - apiVersion: monitoring.coreos.com/v1 - kind: PrometheusRule - metadata: - name: django - spec: - {{- readFile "prometheus/django.rules" | nindent 6 }} - # The console itself. The chart passes every extra manifest through helm's - # tpl, which would render the actions of the console template: each opening - # delimiter is escaped so that tpl writes it back as is (the closing one is - # spelled in two halves so that it does not end this action) + # The dashboards of the load-test campaign, one ConfigMap each, which the + # sidecar of Grafana loads by their label into the folder of the annotation. + # The chart passes every extra manifest through helm's tpl, which would + # render the legend templates of a dashboard (a label name between double + # braces): each opening delimiter is escaped so that tpl writes it back as is + # (the closing one is spelled in two halves so that it does not end this + # action). This file is a template too, hence no brace in this comment. {{- $open := "{{" }} {{- $escaped := printf "%s %q %s" $open $open (print "}" "}") }} + {{- range $name := list "users" "collaboration" "backend" "django" "valkey" }} - apiVersion: v1 kind: ConfigMap metadata: - name: django-console + name: dashboard-{{ $name }} + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: Docs data: - django.html: | - {{- readFile "prometheus/django.html" | replace $open $escaped | nindent 8 }} + {{ $name }}.json: | + {{- readFile (printf "../../../loadtest/dashboards/%s.json" $name) | replace $open $escaped | nindent 8 }} + {{- end }} diff --git a/src/helm/env.d/dev/values.valkey.yaml.gotmpl b/src/helm/env.d/dev/values.valkey.yaml.gotmpl index 5791edd50..91e04bf3e 100644 --- a/src/helm/env.d/dev/values.valkey.yaml.gotmpl +++ b/src/helm/env.d/dev/values.valkey.yaml.gotmpl @@ -10,3 +10,19 @@ auth: # helm test pod that Tilt would run as a plain pod aclConfig: | user default on >pass ~* &* +@all + +# redis_exporter next to the pod, for the Prometheus of the prometheus release +# (values.prometheus.yaml.gotmpl) and the valkey board of src/loadtest/dashboards. +# The chart hands the exporter the password of the instance itself. The check +# of the yhub streams (their length, the pending entries of the worker group) +# is what the board's streams row reads; on valkey-docs it matches nothing and +# costs nothing. The ServiceMonitor is only built with the operator's CRDs, i.e. +# with the prometheus release (`monitoring` in the helmfile environment). +metrics: + enabled: true + exporter: + extraEnvs: + REDIS_EXPORTER_CHECK_STREAMS: "yhub:*" + serviceMonitor: + enabled: {{ .Values | get "monitoring" false }} + interval: 15s diff --git a/src/helm/helmfile.yaml.gotmpl b/src/helm/helmfile.yaml.gotmpl index 5cce13c57..a3391a67f 100644 --- a/src/helm/helmfile.yaml.gotmpl +++ b/src/helm/helmfile.yaml.gotmpl @@ -34,6 +34,11 @@ releases: namespace: {{ .Namespace }} chart: valkey/valkey version: 0.12.0 + {{- if .Values | get "monitoring" false }} + # its ServiceMonitor needs the CRDs of the operator + needs: + - {{ .Namespace }}/prometheus + {{- end }} values: - env.d/{{ .Environment.Name }}/values.valkey.yaml.gotmpl @@ -41,6 +46,11 @@ releases: namespace: {{ .Namespace }} chart: valkey/valkey version: 0.12.0 + {{- if .Values | get "monitoring" false }} + # its ServiceMonitor needs the CRDs of the operator + needs: + - {{ .Namespace }}/prometheus + {{- end }} values: - env.d/{{ .Environment.Name }}/values.valkey.yaml.gotmpl diff --git a/src/loadtest/dashboards/README.md b/src/loadtest/dashboards/README.md new file mode 100644 index 000000000..2c8b24189 --- /dev/null +++ b/src/loadtest/dashboards/README.md @@ -0,0 +1,88 @@ +# Dashboards of the load-test campaign + +Grafana dashboards over the metrics of the campaign: the backend's and yhub's +`/metrics`, and what the load generators export (`../swarm`, `../k6`, +`../canary`). One board per question of `documentation/stress-test-plan.md` +(section 0.5): what the users feel, what the collaboration server does, what +the backend does. Each is laid out so that a saturation reads left to right: +the symptom the clients see, the server-side cause, the resource that ran out. + +| File | Board | For | +| ---- | ----- | --- | +| `users.json` | Docs load test — users | The canary: page open, editor ready, keystroke to the other screen, failures by step. Plus the swarm's and k6's client-side numbers. The board that says "the users noticed"; the others say why | +| `collaboration.json` | Docs load test — collaboration server | yhub per replica: sockets, rooms, event-loop lag, auth duration, calls to the backend (duration, in flight, failures), compaction backlog and duration, seeds. Then the swarm: clients by state, connect and sync, propagation, reconnects, refused upgrades, close codes, traffic | +| `backend.json` | Docs load test — backend | Django: requests and latency by view, 5xx, calls to yhub and the converters (duration, in flight, failures), the psycopg pool (waiting, queued, wait time), queries, Celery queue length. Then k6: rate and p95 by endpoint, failures, VUs, dropped iterations | +| `valkey.json` | Docs load test — valkey | The two Valkey instances through `redis_exporter`, one target per pod: health and role, memory against `maxmemory`, evictions, commands and their latency, network, CPU, keys, the yhub streams, replication, Sentinel quorum and master status, persistence | +| `django.json` | Django | The community [Django dashboard 17658](https://grafana.com/grafana/dashboards/17658-django/) (revision 2), for the per-view detail of django-prometheus | + +A `hostname` variable on the collaboration and backend boards narrows the +panels to one replica: the label every sample carries (`documentation/metrics.md`). + +## Where they run + +The dev cluster (`monitoring: true`, `src/helm/env.d/dev/values.prometheus.yaml.gotmpl`) +loads them into its Grafana, https://docs-grafana.127.0.0.1.nip.io, from +ConfigMaps built from these files. Anywhere else, import the JSON files as they +are: every panel goes through the `datasource` variable of the board, which +defaults to the Grafana's default Prometheus and can be switched at the top of +the board. No uid is hardcoded. + +The k6 panels expect k6's Prometheus remote write +(`-o experimental-prometheus-rw`, see `../k6/README.md`): its trends arrive as +gauges, `k6_http_req_duration_p99` by default; set +`K6_PROMETHEUS_RW_TREND_STATS=p(95),p(99)` for a p95 too. + +## What stays empty, and why + +- On the valkey board, `maxmemory` panels when no limit is set, the streams + row without `--check-streams`, the Sentinel row without the sentinel pods + scraped (see Stores). +- Pod memory and CPU: not on these boards. The backend exports no process + metrics (prometheus_client's multiprocess mode has none); take them from + the cluster's cAdvisor. yhub does export `process_resident_memory_bytes`. +- Cache hit ratio, on the Django board: django-prometheus only counts cache + calls through its own cache backends, which the backend does not use. +- The canary panels are empty until a canary runs; the k6 ones until k6 runs + with the remote write. + +## Changes to the community Django dashboard + +`django.json` is revision 2 of dashboard 17658 with two edits, so that the +sidecar can load it as it is: the `${DS_PROMETHEUS}` import input is replaced +by the board's own `${datasource}` variable, and its `app` label filter (which nothing +here sets) by `job`, the label the chart's ServiceMonitor names the component +with (`backend`). Take a new revision from grafana.com the same way. + +## Stores + +Valkey is `valkey.json`. The team running it (chideat/valkey-operator) enables +the operator's exporter, which is `redis_exporter` as a sidecar of every valkey +pod, so the board has one target per pod: the `job` variable picks the +instance, `instance` the pod. Two things about that exporter: + +- the operator's `exporter` spec sets an image, resources and a security + context, and nothing else: no arguments, no environment. The streams row + (`redis_stream_*`) needs `--check-streams` (or + `REDIS_EXPORTER_CHECK_STREAMS=yhub:*`), which the operator cannot pass. + What it does let through is the image (`exporter.image`), and it starts it + as `/redis_exporter `: an image where `/redis_exporter` is a + two-line wrapper setting that variable and exec'ing the real binary gets the + streams row with the operator's own command line (checked against + redis_exporter v1.92.0). Otherwise, a second exporter run with the flag + against valkey-yhub. The dev cluster's chart can pass the variable, and does; +- the Sentinel row needs the sentinel pods scraped as well. + +Postgres is not here: that exporter belongs to the team running it. What the +campaign needs from it, per instance: + +- Postgres (`postgres_exporter` or pghero): connections by state and by + application name, transactions and tuples per second, the slowest queries + (`pg_stat_statements` by total and mean time), replication lag, and the + Patroni leader. On the `yhub` database as well as the backend's. + +## Editing + +The three `docs-loadtest-*` boards are generated by `generate.py` +(`python3 generate.py .`): change the generator and regenerate, rather than the +JSON. A legend template such as `{{hostname}}` is fine in the JSON; +the dev values escape it for helm's `tpl` when building the ConfigMaps. diff --git a/src/loadtest/dashboards/backend.json b/src/loadtest/dashboards/backend.json new file mode 100644 index 000000000..adb7b203d --- /dev/null +++ b/src/loadtest/dashboards/backend.json @@ -0,0 +1,1101 @@ +{ + "uid": "docs-loadtest-backend", + "title": "Docs load test \u2014 backend", + "description": "The Django backend under load: what it spends on the other services, its database pool, its queue, and what k6 sees from the outside. The generic Django board (17658) has the per-view detail.", + "tags": [ + "docs", + "loadtest" + ], + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "time": { + "from": "now-1h", + "to": "now" + }, + "refresh": "15s", + "timezone": "browser", + "templating": { + "list": [ + { + "name": "datasource", + "label": "Prometheus", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "refresh": 1 + }, + { + "name": "hostname", + "label": "backend replica", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(django_http_requests_total_by_view_transport_method_total, hostname)", + "refId": "hostname" + }, + "definition": "label_values(django_http_requests_total_by_view_transport_method_total, hostname)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "Requests", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 0 + }, + "panels": [], + "id": 1 + }, + { + "type": "timeseries", + "title": "Requests by view", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(django_http_requests_total_by_view_transport_method_total{hostname=~\"$hostname\"}[$__rate_interval])) by (view)", + "legendFormat": "{{view}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 1 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 2 + }, + { + "type": "timeseries", + "title": "Latency p95 by view", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, view))", + "legendFormat": "{{view}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 1 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 3 + }, + { + "type": "timeseries", + "title": "Responses by status", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(django_http_responses_total_by_status_view_method_total{hostname=~\"$hostname\"}[$__rate_interval])) by (status)", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 9 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 4 + }, + { + "type": "timeseries", + "title": "Errors (5xx) by view", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(django_http_responses_total_by_status_view_method_total{hostname=~\"$hostname\", status=~\"5..\"}[$__rate_interval])) by (view)", + "legendFormat": "{{view}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 9 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 5 + }, + { + "type": "row", + "title": "Calls to the other services", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 17 + }, + "panels": [], + "id": 6 + }, + { + "type": "timeseries", + "title": "Outgoing calls p95", + "description": "yhub, and the converters. Made inside requests (duplicate, formatted-content, import) and from the Celery tasks.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(docs_outgoing_request_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, service, operation))", + "legendFormat": "{{service}} {{operation}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 18 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7 + }, + { + "type": "timeseries", + "title": "Outgoing calls in flight", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(docs_outgoing_requests_inflight{hostname=~\"$hostname\"}) by (service, operation)", + "legendFormat": "{{service}} {{operation}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 18 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8 + }, + { + "type": "timeseries", + "title": "Outgoing calls failed", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(docs_outgoing_request_duration_seconds_count{hostname=~\"$hostname\", status=~\"timeout|error|5..\"}[$__rate_interval])) by (service, operation, status)", + "legendFormat": "{{service}} {{operation}} {{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 26 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 9 + }, + { + "type": "timeseries", + "title": "Outgoing calls rate", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(docs_outgoing_request_duration_seconds_count{hostname=~\"$hostname\"}[$__rate_interval])) by (service, operation)", + "legendFormat": "{{service}} {{operation}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 26 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 10 + }, + { + "type": "row", + "title": "Database pool (DB_PSYCOPG_POOL_ENABLED)", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 34 + }, + "panels": [], + "id": 11 + }, + { + "type": "timeseries", + "title": "Requests waiting for a connection", + "description": "The application waiting for connections, before Postgres shows anything. What was missing in the 2026-08-18 and 2026-09-07 outages.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(docs_db_pool_requests_waiting{hostname=~\"$hostname\"}) by (hostname)", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 35 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 12 + }, + { + "type": "timeseries", + "title": "Queued requests and time spent waiting", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(docs_db_pool_requests_queued_total{hostname=~\"$hostname\"}[$__rate_interval]))", + "legendFormat": "queued/s", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(docs_db_pool_requests_wait_seconds_total{hostname=~\"$hostname\"}[$__rate_interval]))", + "legendFormat": "wait s/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 35 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 13 + }, + { + "type": "timeseries", + "title": "Pool size and idle", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(docs_db_pool_size{hostname=~\"$hostname\"})", + "legendFormat": "size", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(docs_db_pool_available{hostname=~\"$hostname\"})", + "legendFormat": "idle", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 43 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 14 + }, + { + "type": "timeseries", + "title": "Connections opened to Postgres", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(docs_db_pool_connections_total{hostname=~\"$hostname\"}[$__rate_interval]))", + "legendFormat": "opened/s", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(docs_db_pool_connections_errors_total{hostname=~\"$hostname\"}[$__rate_interval]))", + "legendFormat": "errors/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 43 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 15 + }, + { + "type": "row", + "title": "Queries and Celery", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 51 + }, + "panels": [], + "id": 16 + }, + { + "type": "timeseries", + "title": "Query duration p95", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(django_db_query_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le))", + "legendFormat": "p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 52 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 17 + }, + { + "type": "timeseries", + "title": "Queries", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(django_db_execute_total{hostname=~\"$hostname\"}[$__rate_interval]))", + "legendFormat": "queries/s", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(django_db_errors_total{hostname=~\"$hostname\"}[$__rate_interval]))", + "legendFormat": "errors/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 52 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 18 + }, + { + "type": "timeseries", + "title": "Celery queue length", + "description": "One queue for the whole deployment, reported by every replica: max, not sum. The delete, restore and access cascades land on it.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(docs_celery_queue_length) by (queue)", + "legendFormat": "{{queue}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 60 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 19 + }, + { + "type": "timeseries", + "title": "Uvicorn workers alive", + "description": "Replicas answering scrapes. The memory and CPU of the pods come from the cluster (cAdvisor), not from the application: prometheus_client exports no process metrics in multiprocess mode.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count(count(django_http_requests_total_by_view_transport_method_total{hostname=~\"$hostname\"}) by (hostname))", + "legendFormat": "replicas", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 60 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 20 + }, + { + "type": "row", + "title": "k6 \u2014 what the clients see", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 68 + }, + "panels": [], + "id": 21 + }, + { + "type": "timeseries", + "title": "k6 requests by endpoint", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(k6_http_reqs_total[$__rate_interval])) by (name)", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 69 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 22 + }, + { + "type": "timeseries", + "title": "k6 latency p99 by endpoint", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(k6_http_req_duration_p99{name!=\"\"}) by (name)", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 69 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 23 + }, + { + "type": "timeseries", + "title": "k6 failures", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(k6_http_req_failed_rate)", + "legendFormat": "requests failed", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "1 - max(k6_checks_rate)", + "legendFormat": "checks failed", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": {}, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 77 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 24 + }, + { + "type": "timeseries", + "title": "k6 virtual users and dropped iterations", + "description": "Dropped iterations: the arrival rate asked for could not be served with the VUs allocated, the API was too slow.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(k6_vus)", + "legendFormat": "VUs", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(k6_dropped_iterations_total[$__rate_interval]))", + "legendFormat": "dropped/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 77 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 25 + } + ] +} diff --git a/src/loadtest/dashboards/collaboration.json b/src/loadtest/dashboards/collaboration.json new file mode 100644 index 000000000..e676443fe --- /dev/null +++ b/src/loadtest/dashboards/collaboration.json @@ -0,0 +1,1234 @@ +{ + "uid": "docs-loadtest-collaboration", + "title": "Docs load test \u2014 collaboration server", + "description": "The collaboration server (yhub) under load, and what the swarm sees from the outside.", + "tags": [ + "docs", + "loadtest" + ], + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "time": { + "from": "now-1h", + "to": "now" + }, + "refresh": "15s", + "timezone": "browser", + "templating": { + "list": [ + { + "name": "datasource", + "label": "Prometheus", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "refresh": 1 + }, + { + "name": "hostname", + "label": "yhub replica", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(yhub_ws_connections, hostname)", + "refId": "hostname" + }, + "definition": "label_values(yhub_ws_connections, hostname)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "Connections", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 0 + }, + "panels": [], + "id": 1 + }, + { + "type": "timeseries", + "title": "Websocket connections per replica", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(yhub_ws_connections{hostname=~\"$hostname\"}) by (hostname)", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 1 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 2 + }, + { + "type": "timeseries", + "title": "Open documents per replica", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(yhub_rooms{hostname=~\"$hostname\"}) by (hostname)", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 1 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 3 + }, + { + "type": "timeseries", + "title": "Event loop lag p99 per replica", + "description": "One thread serves every socket of a replica: this is the first thing to move when it saturates.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(nodejs_eventloop_lag_p99_seconds{hostname=~\"$hostname\", job=~\"yhub.*\"}) by (hostname)", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 9 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 4 + }, + { + "type": "timeseries", + "title": "Memory per replica", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(process_resident_memory_bytes{hostname=~\"$hostname\", job=~\"yhub.*\"}) by (hostname)", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 9 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 5 + }, + { + "type": "row", + "title": "Admitting callers", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 17 + }, + "panels": [], + "id": 6 + }, + { + "type": "timeseries", + "title": "Auth duration p95", + "description": "What a websocket upgrade, a recheck, a REST call or a fallback poll costs: the backend calls it makes, and the legacy seed under soft migration.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(yhub_auth_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, phase, endpoint))", + "legendFormat": "{{phase}} {{endpoint}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 18 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7 + }, + { + "type": "timeseries", + "title": "Auth results", + "description": "unavailable: a 503 sent to the client, the backend did not answer.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(yhub_auth_duration_seconds_count{hostname=~\"$hostname\"}[$__rate_interval])) by (phase, result)", + "legendFormat": "{{phase}} {{result}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 18 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8 + }, + { + "type": "timeseries", + "title": "Backend calls p95 by route", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(yhub_backend_request_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, route))", + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 26 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 9 + }, + { + "type": "timeseries", + "title": "Backend calls in flight", + "description": "What piles up when the backend slows down: backendFetch has no timeout.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(yhub_backend_requests_inflight{hostname=~\"$hostname\"}) by (route)", + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 26 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 10 + }, + { + "type": "timeseries", + "title": "Backend calls not answered 2xx", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(yhub_backend_request_duration_seconds_count{hostname=~\"$hostname\", status!~\"2..\"}[$__rate_interval])) by (route, status)", + "legendFormat": "{{route}} {{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 34 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 11 + }, + { + "type": "row", + "title": "Persistence (worker)", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 42 + }, + "panels": [], + "id": 12 + }, + { + "type": "timeseries", + "title": "Compaction backlog", + "description": "One queue for the whole deployment, reported by every worker: max, not sum.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(yhub_worker_pending_tasks)", + "legendFormat": "pending", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 43 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 13 + }, + { + "type": "timeseries", + "title": "Compactions", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(yhub_worker_task_duration_seconds_count{hostname=~\"$hostname\"}[$__rate_interval])) by (result)", + "legendFormat": "{{result}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(yhub_doc_updates_total{hostname=~\"$hostname\"}[$__rate_interval]))", + "legendFormat": "with new content", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 43 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 14 + }, + { + "type": "timeseries", + "title": "Compaction duration p95", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(yhub_worker_task_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le))", + "legendFormat": "p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 51 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 15 + }, + { + "type": "timeseries", + "title": "Compactions in flight", + "description": "Against YHUB_TASK_CONCURRENCY per worker.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(yhub_worker_tasks_inflight{hostname=~\"$hostname\"}) by (hostname)", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 51 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 16 + }, + { + "type": "row", + "title": "Soft migration", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 59 + }, + "panels": [], + "id": 17 + }, + { + "type": "timeseries", + "title": "Seeds", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(yhub_seed_duration_seconds_count{hostname=~\"$hostname\"}[$__rate_interval])) by (result)", + "legendFormat": "{{result}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(yhub_seed_rejected_total{hostname=~\"$hostname\"}[$__rate_interval]))", + "legendFormat": "rejected", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 60 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 18 + }, + { + "type": "timeseries", + "title": "Seed duration p95", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(yhub_seed_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, result))", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 60 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 19 + }, + { + "type": "timeseries", + "title": "Seeds in flight", + "description": "Refused with a 503 past 20 per replica.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(yhub_seeds_inflight{hostname=~\"$hostname\"}) by (hostname)", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 68 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 20 + }, + { + "type": "row", + "title": "Swarm \u2014 what the clients see", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 76 + }, + "panels": [], + "id": 21 + }, + { + "type": "timeseries", + "title": "Swarm clients", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(swarm_clients) by (state)", + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 77 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 22 + }, + { + "type": "timeseries", + "title": "Connect and sync p95", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(swarm_connect_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "connect", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(swarm_sync_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "sync", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 77 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 23 + }, + { + "type": "timeseries", + "title": "Propagation", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.5, sum(rate(swarm_propagation_latency_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(swarm_propagation_latency_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum(rate(swarm_propagation_latency_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 85 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 24 + }, + { + "type": "timeseries", + "title": "Reconnects and refused upgrades", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(swarm_reconnects_total[$__rate_interval]))", + "legendFormat": "reconnects", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(swarm_upgrade_failures_total[$__rate_interval])) by (status)", + "legendFormat": "refused {{status}}", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 85 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 25 + }, + { + "type": "timeseries", + "title": "Socket closes by code", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(swarm_ws_closes_total[$__rate_interval])) by (code)", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 93 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 26 + }, + { + "type": "timeseries", + "title": "Websocket traffic", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(swarm_ws_bytes_total[$__rate_interval])) by (direction)", + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 93 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 27 + }, + { + "type": "timeseries", + "title": "Swarm event loop lag p99", + "description": "Past a point the load generator saturates before the server: its numbers stop meaning anything.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(nodejs_eventloop_lag_p99_seconds{job=~\"swarm.*\"})", + "legendFormat": "lag", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 101 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 28 + } + ] +} diff --git a/src/loadtest/dashboards/django.json b/src/loadtest/dashboards/django.json new file mode 100644 index 000000000..65360c5f7 --- /dev/null +++ b/src/loadtest/dashboards/django.json @@ -0,0 +1,1588 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Django metrics dashboard using django-prometheus metrics exporter", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 17658, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 13, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 40 + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(irate(django_http_requests_total_by_transport_total{job=~\"^$application$\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "range": true, + "refId": "A" + } + ], + "title": "Requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 20 + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(irate(django_http_responses_total_by_status_total{status=~\"2.+\",job=~\"^$application$\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "2XX Responses", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 16, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 20 + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(irate(django_http_responses_total_by_status_total{status=~\"4.+\",job=~\"^$application$\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "4XX Responses", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 0 + }, + "id": 37, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(max_over_time(django_db_errors_total{job=~\"^$application$\"}[$__range]))", + "format": "time_series", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DB Query Errors", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 0 + }, + "id": 38, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(max_over_time(django_db_new_connection_errors_total{job=~\"^$application$\"}[$__range]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DB Connection Errors", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.4 + }, + { + "color": "green", + "value": 0.7 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 31, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(django_cache_get_hits_total{job=~\"^$application$\"}) by (backend) / sum(django_cache_get_total{job=~\"^$application$\"}) by (backend)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cache Hit Ratio", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 6, + "y": 2 + }, + "id": 23, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 20 + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(irate(django_http_responses_total_by_status_total{status=~\"3.+\",job=~\"^$application$\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "3XX Responses", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 9, + "y": 2 + }, + "id": 17, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 20 + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(irate(django_http_responses_total_by_status_total{status=~\"5.+\",job=~\"^$application$\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "5XX Responses", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 20, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "reqps" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto", + "value": "Responses" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of Requests by Processing Time", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 22, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "reqps" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto", + "value": "Responses" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "bytes" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(django_http_responses_body_total_bytes_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of Responses by Size", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "linearThreshold": 1, + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 4, + "interval": "30s", + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(irate(django_http_requests_latency_seconds_by_view_method_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "50 quantile", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(irate(django_http_requests_latency_seconds_by_view_method_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "95 quantile", + "range": true, + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(irate(django_http_requests_latency_seconds_by_view_method_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99 quantile", + "range": true, + "refId": "C" + } + ], + "title": "Request Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 11, + "interval": "30s", + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(irate(django_http_responses_total_by_status_total{job=~\"^$application$\"}[$__rate_interval])) by(status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "Response Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.3 + }, + { + "color": "red", + "value": 0.6 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 25, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "limit": 20, + "values": true + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(20, (1 - (sum(max_over_time(django_http_requests_latency_seconds_by_view_method_bucket{job=~\"^$application$\",le=\"$threshold\"}[$__range]) / ignoring(le) max_over_time(django_http_requests_latency_seconds_by_view_method_count{job=~\"^$application$\"}[$__range])) by (method, view) / count(present_over_time(django_http_requests_latency_seconds_by_view_method_count{job=~\"^$application$\"}[$__range])) by (method, view))) > 0.0099)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{method}} {{view}}", + "range": false, + "refId": "A" + } + ], + "title": "Top 20 Views by Response Time (> $threshold)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 2, + "interval": "30s", + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(10, sum(irate(django_http_requests_total_by_view_transport_method_total{job=~\"^$application$\",view!~\"prometheus-django-metrics|healthcheck|.*shop_product_import_1c_status\"}[$__rate_interval])) by(method, view) > 0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{method}} /{{view}}", + "range": true, + "refId": "A" + } + ], + "title": "Top Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(django_db_execute_total{job=~\"^$application$\"}[$__rate_interval])) by (vendor)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Database Total Queries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 33, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "ops" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(django_db_query_duration_seconds_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Database Query Duration", + "type": "heatmap" + } + ], + "refresh": "30s", + "schemaVersion": 37, + "style": "dark", + "tags": [ + "Web", + "Django", + "Database", + "Cache" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(python_info,app)", + "hide": 0, + "includeAll": true, + "label": "application", + "multi": false, + "name": "application", + "options": [], + "query": { + "query": "label_values(python_info,app)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": "1s", + "value": "1.0" + }, + "description": "Request duration threshold", + "hide": 0, + "includeAll": false, + "label": "threshold", + "multi": false, + "name": "threshold", + "options": [ + { + "selected": false, + "text": "100ms", + "value": "0.1" + }, + { + "selected": false, + "text": "250ms", + "value": "0.25" + }, + { + "selected": false, + "text": "500ms", + "value": "0.5" + }, + { + "selected": false, + "text": "750ms", + "value": "0.75" + }, + { + "selected": true, + "text": "1s", + "value": "1.0" + }, + { + "selected": false, + "text": "2.5s", + "value": "2.5" + }, + { + "selected": false, + "text": "5s", + "value": "5.0" + }, + { + "selected": false, + "text": "7.5s", + "value": "7.5" + }, + { + "selected": false, + "text": "10s", + "value": "10.0" + }, + { + "selected": false, + "text": "25s", + "value": "25.0" + } + ], + "query": "100ms : 0.1, 250ms : 0.25, 500ms : 0.5, 750ms : 0.75, 1s : 1.0, 2.5s : 2.5, 5s : 5.0, 7.5s : 7.5, 10s : 10.0, 25s : 25.0", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Django", + "uid": "O6v4rMpizda", + "version": 139, + "weekStart": "" +} diff --git a/src/loadtest/dashboards/generate.py b/src/loadtest/dashboards/generate.py new file mode 100644 index 000000000..28c824a9c --- /dev/null +++ b/src/loadtest/dashboards/generate.py @@ -0,0 +1,234 @@ +"""Generate the docs-loadtest-* dashboards: python3 generate.py .""" +import json, sys + +# Every panel and query goes through the `datasource` variable, never a fixed +# uid: the boards are imported in Grafanas whose Prometheus is named anything +DS = {"type": "prometheus", "uid": "${datasource}"} +OUT = sys.argv[1] + +def target(expr, legend="", ref="A"): + return {"datasource": DS, "expr": expr, "legendFormat": legend, "refId": ref} + +def panel(title, targets, unit="short", kind="timeseries", w=12, h=8, desc="", opts=None, stack=False, mn=None, mx=None): + fc = {"defaults": {"unit": unit, "custom": {}}, "overrides": []} + if stack: fc["defaults"]["custom"]["stacking"] = {"mode": "normal"} + if mn is not None: fc["defaults"]["min"] = mn + if mx is not None: fc["defaults"]["max"] = mx + p = {"type": kind, "title": title, "description": desc, "datasource": DS, + "targets": [dict(t, refId=chr(65 + i)) for i, t in enumerate(targets)], + "fieldConfig": fc, "gridPos": {"w": w, "h": h}, "options": opts or {}} + if kind == "timeseries": + p["options"] = {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": True}, + "tooltip": {"mode": "multi", "sort": "desc"}, **(opts or {})} + if kind == "stat": + p["options"] = {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "graphMode": "area", **(opts or {})} + return p + +def row(title): + return {"type": "row", "title": title, "collapsed": False, "gridPos": {"w": 24, "h": 1}, "panels": []} + +def layout(panels): + """Assign y/x from panel order: rows take a full line, panels pack left to right.""" + x = y = 0 + line_h = 0 + out = [] + for i, p in enumerate(panels): + w, h = p["gridPos"]["w"], p["gridPos"]["h"] + if p["type"] == "row" or x + w > 24: + y += line_h + x, line_h = 0, 0 + p["gridPos"].update({"x": x, "y": y}) + p["id"] = i + 1 + x += w + line_h = max(line_h, h) + if p["type"] == "row": + y += 1 + x, line_h = 0, 0 + out.append(p) + return out + +def dashboard(uid, title, desc, panels, variables=(), tags=("docs", "loadtest")): + templating = [{"name": "datasource", "label": "Prometheus", "type": "datasource", "query": "prometheus", "current": {}, "hide": 0, "refresh": 1}] + for name, label, query in variables: + templating.append({"name": name, "label": label, "type": "query", "datasource": DS, "query": {"query": query, "refId": name}, + "definition": query, "refresh": 2, "includeAll": True, "multi": True, "allValue": ".*", + "current": {"text": "All", "value": "$__all"}, "sort": 1}) + return {"uid": uid, "title": title, "description": desc, "tags": list(tags), "schemaVersion": 39, "version": 1, + "editable": True, "graphTooltip": 1, "time": {"from": "now-1h", "to": "now"}, "refresh": "15s", + "timezone": "browser", "templating": {"list": templating}, "annotations": {"list": []}, + "panels": layout(panels)} + +def q(quantile, metric, by="", rng="$__rate_interval", sel=""): + by_clause = f" by (le{', ' + by if by else ''})" + return f"histogram_quantile({quantile}, sum(rate({metric}_bucket{{{sel}}}[{rng}])){by_clause})" + +H = 'hostname=~"$hostname"' + +# ---------------------------------------------------------------- users +users = dashboard("docs-loadtest-users", "Docs load test — users", + "What a user feels during a run: the browser canaries (src/loadtest/canary). The other boards say why.", + [ + row("Canary — real browsers"), + panel("Page open", [target(q(0.5, "canary_page_open_seconds"), "p50"), target(q(0.95, "canary_page_open_seconds"), "p95")], "s", + desc="From the navigation to the document page being visible."), + panel("Editor ready", [target(q(0.5, "canary_editor_ready_seconds"), "p50"), target(q(0.95, "canary_editor_ready_seconds"), "p95")], "s", + desc="From the navigation to the editor accepting input: the collaboration provider synced. The number to compare against the 2 s target of the plan."), + panel("Keystroke to the other screen", [target(q(0.5, "canary_propagation_seconds"), "p50"), target(q(0.95, "canary_propagation_seconds"), "p95")], "s", + desc="From a keystroke in one browser to the text showing in another browser on the same document. Target of the plan: p95 under 500 ms."), + panel("Iterations", [target("sum(rate(canary_iterations_total[$__rate_interval])) by (result)", "{{result}}")], "ops", stack=True), + panel("Failures by step", [target("sum(increase(canary_failures_total[$__rate_interval])) by (step)", "{{step}}")], "short", + desc="page-open: the page never showed. editor-ready: the provider never synced. propagation: the text never reached the other browser."), + panel("Console errors", [target("sum(rate(canary_console_errors_total[$__rate_interval]))", "errors/s")], "ops", w=6), + panel("Pairs running", [target("sum(canary_pairs)", "pairs")], "short", kind="stat", w=6), + row("Swarm and k6, as a user would feel them"), + panel("Swarm: time to connect and to first sync", [target(q(0.95, "swarm_connect_duration_seconds"), "connect p95"), target(q(0.95, "swarm_sync_duration_seconds"), "sync p95")], "s", + desc="What a client of the swarm waits before the document is usable."), + panel("Swarm: edit propagation", [target(q(0.5, "swarm_propagation_latency_seconds"), "p50"), target(q(0.95, "swarm_propagation_latency_seconds"), "p95"), target(q(0.99, "swarm_propagation_latency_seconds"), "p99")], "s"), + panel("k6: page-open requests, p99 by endpoint", [target('max(k6_http_req_duration_p99{name!=""}) by (name)', "{{name}}")], "ms", + desc="k6 exports its trends as gauges over the remote-write interval, p99 by default (K6_PROMETHEUS_RW_TREND_STATS adds p95)."), + panel("k6: failed requests", [target("max(k6_http_req_failed_rate)", "failed"), target("1 - max(k6_checks_rate)", "checks failed")], "percentunit", mn=0, mx=1), + ]) + +# --------------------------------------------------------- collaboration +collab = dashboard("docs-loadtest-collaboration", "Docs load test — collaboration server", + "The collaboration server (yhub) under load, and what the swarm sees from the outside.", + [ + row("Connections"), + panel("Websocket connections per replica", [target(f"sum(yhub_ws_connections{{{H}}}) by (hostname)", "{{hostname}}")], "short", stack=True), + panel("Open documents per replica", [target(f"sum(yhub_rooms{{{H}}}) by (hostname)", "{{hostname}}")], "short"), + panel("Event loop lag p99 per replica", [target(f"max(nodejs_eventloop_lag_p99_seconds{{{H}, job=~\"yhub.*\"}}) by (hostname)", "{{hostname}}")], "s", + desc="One thread serves every socket of a replica: this is the first thing to move when it saturates."), + panel("Memory per replica", [target(f"max(process_resident_memory_bytes{{{H}, job=~\"yhub.*\"}}) by (hostname)", "{{hostname}}")], "bytes"), + row("Admitting callers"), + panel("Auth duration p95", [target(q(0.95, "yhub_auth_duration_seconds", "phase, endpoint", sel=H), "{{phase}} {{endpoint}}")], "s", + desc="What a websocket upgrade, a recheck, a REST call or a fallback poll costs: the backend calls it makes, and the legacy seed under soft migration."), + panel("Auth results", [target(f"sum(rate(yhub_auth_duration_seconds_count{{{H}}}[$__rate_interval])) by (phase, result)", "{{phase}} {{result}}")], "ops", stack=True, + desc="unavailable: a 503 sent to the client, the backend did not answer."), + panel("Backend calls p95 by route", [target(q(0.95, "yhub_backend_request_duration_seconds", "route", sel=H), "{{route}}")], "s"), + panel("Backend calls in flight", [target(f"sum(yhub_backend_requests_inflight{{{H}}}) by (route)", "{{route}}")], "short", stack=True, + desc="What piles up when the backend slows down: backendFetch has no timeout."), + panel("Backend calls not answered 2xx", [target(f'sum(rate(yhub_backend_request_duration_seconds_count{{{H}, status!~"2.."}}[$__rate_interval])) by (route, status)', "{{route}} {{status}}")], "ops"), + row("Persistence (worker)"), + panel("Compaction backlog", [target("max(yhub_worker_pending_tasks)", "pending")], "short", + desc="One queue for the whole deployment, reported by every worker: max, not sum."), + panel("Compactions", [target(f"sum(rate(yhub_worker_task_duration_seconds_count{{{H}}}[$__rate_interval])) by (result)", "{{result}}"), target(f"sum(rate(yhub_doc_updates_total{{{H}}}[$__rate_interval]))", "with new content")], "ops"), + panel("Compaction duration p95", [target(q(0.95, "yhub_worker_task_duration_seconds", sel=H), "p95")], "s"), + panel("Compactions in flight", [target(f"sum(yhub_worker_tasks_inflight{{{H}}}) by (hostname)", "{{hostname}}")], "short", stack=True, + desc="Against YHUB_TASK_CONCURRENCY per worker."), + row("Soft migration"), + panel("Seeds", [target(f"sum(rate(yhub_seed_duration_seconds_count{{{H}}}[$__rate_interval])) by (result)", "{{result}}"), target(f"sum(rate(yhub_seed_rejected_total{{{H}}}[$__rate_interval]))", "rejected")], "ops"), + panel("Seed duration p95", [target(q(0.95, "yhub_seed_duration_seconds", "result", sel=H), "{{result}}")], "s"), + panel("Seeds in flight", [target(f"sum(yhub_seeds_inflight{{{H}}}) by (hostname)", "{{hostname}}")], "short", desc="Refused with a 503 past 20 per replica."), + row("Swarm — what the clients see"), + panel("Swarm clients", [target("sum(swarm_clients) by (state)", "{{state}}")], "short", stack=True), + panel("Connect and sync p95", [target(q(0.95, "swarm_connect_duration_seconds"), "connect"), target(q(0.95, "swarm_sync_duration_seconds"), "sync")], "s"), + panel("Propagation", [target(q(0.5, "swarm_propagation_latency_seconds"), "p50"), target(q(0.95, "swarm_propagation_latency_seconds"), "p95"), target(q(0.99, "swarm_propagation_latency_seconds"), "p99")], "s"), + panel("Reconnects and refused upgrades", [target("sum(rate(swarm_reconnects_total[$__rate_interval]))", "reconnects"), target("sum(rate(swarm_upgrade_failures_total[$__rate_interval])) by (status)", "refused {{status}}")], "ops"), + panel("Socket closes by code", [target("sum(rate(swarm_ws_closes_total[$__rate_interval])) by (code)", "{{code}}")], "ops"), + panel("Websocket traffic", [target("sum(rate(swarm_ws_bytes_total[$__rate_interval])) by (direction)", "{{direction}}")], "Bps"), + panel("Swarm event loop lag p99", [target('max(nodejs_eventloop_lag_p99_seconds{job=~"swarm.*"})', "lag")], "s", + desc="Past a point the load generator saturates before the server: its numbers stop meaning anything."), + ], variables=[("hostname", "yhub replica", "label_values(yhub_ws_connections, hostname)")]) + +# --------------------------------------------------------------- backend +backend = dashboard("docs-loadtest-backend", "Docs load test — backend", + "The Django backend under load: what it spends on the other services, its database pool, its queue, and what k6 sees from the outside. The generic Django board (17658) has the per-view detail.", + [ + row("Requests"), + panel("Requests by view", [target(f"sum(rate(django_http_requests_total_by_view_transport_method_total{{{H}}}[$__rate_interval])) by (view)", "{{view}}")], "reqps", stack=True), + panel("Latency p95 by view", [target(q(0.95, "django_http_requests_latency_seconds_by_view_method", "view", sel=H), "{{view}}")], "s"), + panel("Responses by status", [target(f"sum(rate(django_http_responses_total_by_status_view_method_total{{{H}}}[$__rate_interval])) by (status)", "{{status}}")], "reqps", stack=True), + panel("Errors (5xx) by view", [target(f'sum(rate(django_http_responses_total_by_status_view_method_total{{{H}, status=~"5.."}}[$__rate_interval])) by (view)', "{{view}}")], "reqps"), + row("Calls to the other services"), + panel("Outgoing calls p95", [target(q(0.95, "docs_outgoing_request_duration_seconds", "service, operation", sel=H), "{{service}} {{operation}}")], "s", + desc="yhub, and the converters. Made inside requests (duplicate, formatted-content, import) and from the Celery tasks."), + panel("Outgoing calls in flight", [target(f"sum(docs_outgoing_requests_inflight{{{H}}}) by (service, operation)", "{{service}} {{operation}}")], "short", stack=True), + panel("Outgoing calls failed", [target(f'sum(rate(docs_outgoing_request_duration_seconds_count{{{H}, status=~"timeout|error|5.."}}[$__rate_interval])) by (service, operation, status)', "{{service}} {{operation}} {{status}}")], "ops"), + panel("Outgoing calls rate", [target(f"sum(rate(docs_outgoing_request_duration_seconds_count{{{H}}}[$__rate_interval])) by (service, operation)", "{{service}} {{operation}}")], "ops"), + row("Database pool (DB_PSYCOPG_POOL_ENABLED)"), + panel("Requests waiting for a connection", [target(f"sum(docs_db_pool_requests_waiting{{{H}}}) by (hostname)", "{{hostname}}")], "short", stack=True, + desc="The application waiting for connections, before Postgres shows anything. What was missing in the 2026-08-18 and 2026-09-07 outages."), + panel("Queued requests and time spent waiting", [target(f"sum(rate(docs_db_pool_requests_queued_total{{{H}}}[$__rate_interval]))", "queued/s"), target(f"sum(rate(docs_db_pool_requests_wait_seconds_total{{{H}}}[$__rate_interval]))", "wait s/s")], "short"), + panel("Pool size and idle", [target(f"sum(docs_db_pool_size{{{H}}})", "size"), target(f"sum(docs_db_pool_available{{{H}}})", "idle")], "short"), + panel("Connections opened to Postgres", [target(f"sum(rate(docs_db_pool_connections_total{{{H}}}[$__rate_interval]))", "opened/s"), target(f"sum(rate(docs_db_pool_connections_errors_total{{{H}}}[$__rate_interval]))", "errors/s")], "ops"), + row("Queries and Celery"), + panel("Query duration p95", [target(q(0.95, "django_db_query_duration_seconds", sel=H), "p95")], "s"), + panel("Queries", [target(f"sum(rate(django_db_execute_total{{{H}}}[$__rate_interval]))", "queries/s"), target(f"sum(rate(django_db_errors_total{{{H}}}[$__rate_interval]))", "errors/s")], "ops"), + panel("Celery queue length", [target("max(docs_celery_queue_length) by (queue)", "{{queue}}")], "short", + desc="One queue for the whole deployment, reported by every replica: max, not sum. The delete, restore and access cascades land on it."), + panel("Uvicorn workers alive", [target(f'count(count(django_http_requests_total_by_view_transport_method_total{{{H}}}) by (hostname))', "replicas")], "short", + desc="Replicas answering scrapes. The memory and CPU of the pods come from the cluster (cAdvisor), not from the application: prometheus_client exports no process metrics in multiprocess mode."), + row("k6 — what the clients see"), + panel("k6 requests by endpoint", [target('sum(rate(k6_http_reqs_total[$__rate_interval])) by (name)', "{{name}}")], "reqps", stack=True), + panel("k6 latency p99 by endpoint", [target('max(k6_http_req_duration_p99{name!=""}) by (name)', "{{name}}")], "ms"), + panel("k6 failures", [target("max(k6_http_req_failed_rate)", "requests failed"), target("1 - max(k6_checks_rate)", "checks failed")], "percentunit", mn=0, mx=1), + panel("k6 virtual users and dropped iterations", [target("max(k6_vus)", "VUs"), target("sum(rate(k6_dropped_iterations_total[$__rate_interval]))", "dropped/s")], "short", + desc="Dropped iterations: the arrival rate asked for could not be served with the VUs allocated, the API was too slow."), + ], variables=[("hostname", "backend replica", "label_values(django_http_requests_total_by_view_transport_method_total, hostname)")]) + +# ---------------------------------------------------------------- valkey +# redis_exporter (what chideat/valkey-operator runs next to every valkey pod), +# one target per pod: `job` names the instance, `instance` the pod. +J = 'job=~"$job", instance=~"$instance"' +valkey = dashboard("docs-loadtest-valkey", "Docs load test — valkey", + "The two Valkey instances (valkey-docs: backend cache, sessions, Celery; valkey-yhub: the collaboration server's streams), through redis_exporter. One target per pod.", + [ + row("Health"), + panel("Up", [target(f"min(redis_up{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "short", kind="stat", w=6, mn=0, mx=1), + panel("Role", [target(f"max(redis_instance_info{{{J}}}) by (job, instance, role)", "{{job}} {{instance}}: {{role}}")], "short", kind="stat", w=6, + opts={"textMode": "name", "colorMode": "none", "graphMode": "none"}, + desc="master or slave, as INFO says it. A change is a failover."), + panel("Uptime", [target(f"min(redis_uptime_in_seconds{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "s", kind="stat", w=6, + desc="A reset is a restart: what wiped the sessions on 2026-09-07."), + panel("Connected clients", [target(f"sum(redis_connected_clients{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "short", w=6), + row("Memory"), + panel("Memory used", [target(f"max(redis_memory_used_bytes{{{J}}}) by (job, instance)", "{{job}} {{instance}} used"), target(f"max(redis_memory_max_bytes{{{J}}} > 0) by (job, instance)", "{{job}} {{instance}} max")], "bytes", + desc="Against maxmemory (0 when there is none, then hidden). The yhub streams have no TTL and cannot be evicted under volatile-lru: this is the number that decides how many replicas and how much history valkey-yhub can hold."), + panel("Memory used, share of max", [target(f"max(redis_memory_used_bytes{{{J}}} / (redis_memory_max_bytes{{{J}}} > 0)) by (job, instance)", "{{job}} {{instance}}")], "percentunit", mn=0, mx=1), + panel("Evicted and expired keys", [target(f"sum(rate(redis_evicted_keys_total{{{J}}}[$__rate_interval])) by (job, instance)", "{{job}} {{instance}} evicted"), target(f"sum(rate(redis_expired_keys_total{{{J}}}[$__rate_interval])) by (job, instance)", "{{job}} {{instance}} expired")], "ops", + desc="Evictions on valkey-docs are sessions and cache entries thrown away under memory pressure."), + panel("Fragmentation ratio", [target(f"max(redis_mem_fragmentation_ratio{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "short"), + row("Traffic"), + panel("Commands", [target(f"sum(rate(redis_commands_processed_total{{{J}}}[$__rate_interval])) by (job, instance)", "{{job}} {{instance}}")], "ops"), + panel("Commands by command, top 10", [target(f"topk(10, sum(rate(redis_commands_total{{{J}}}[$__rate_interval])) by (job, cmd))", "{{job}} {{cmd}}")], "ops", stack=True), + panel("Command latency p99", [target(f"histogram_quantile(0.99, sum(rate(redis_commands_latencies_usec_bucket{{{J}}}[$__rate_interval])) by (le, job, instance)) / 1e6", "{{job}} {{instance}}")], "s", + desc="Server-side, from LATENCY HISTOGRAM: the time valkey spends on a command, not what the client waits."), + panel("Time spent in commands, top 10", [target(f"topk(10, sum(rate(redis_commands_duration_seconds_total{{{J}}}[$__rate_interval])) by (job, cmd))", "{{job}} {{cmd}}")], "short", stack=True, + desc="Seconds of command time per second: which commands the CPU goes to."), + panel("Network", [target(f"sum(rate(redis_net_input_bytes_total{{{J}}}[$__rate_interval])) by (job, instance)", "{{job}} {{instance}} in"), target(f"sum(rate(redis_net_output_bytes_total{{{J}}}[$__rate_interval])) by (job, instance)", "{{job}} {{instance}} out")], "Bps", + desc="On valkey-yhub the out side is the fan-out of every update to every replica of yhub."), + panel("CPU", [target(f"sum(rate(redis_cpu_sys_seconds_total{{{J}}}[$__rate_interval]) + rate(redis_cpu_user_seconds_total{{{J}}}[$__rate_interval])) by (job, instance)", "{{job}} {{instance}}")], "percentunit", + desc="One core is the ceiling: valkey runs its commands on one thread."), + panel("Blocked clients and rejected connections", [target(f"sum(redis_blocked_clients{{{J}}}) by (job, instance)", "{{job}} {{instance}} blocked"), target(f"sum(rate(redis_rejected_connections_total{{{J}}}[$__rate_interval])) by (job, instance)", "{{job}} {{instance}} rejected/s")], "short", + desc="Blocked: yhub replicas waiting on XREAD, which is normal. Rejected: maxclients reached."), + panel("Slow log length", [target(f"max(redis_slowlog_length{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "short"), + row("Keys"), + panel("Keys", [target(f"sum(redis_db_keys{{{J}}}) by (job, instance, db)", "{{job}} {{instance}} {{db}}"), target(f"sum(redis_db_keys_expiring{{{J}}}) by (job, instance, db)", "{{job}} {{instance}} {{db}} expiring")], "short", + desc="On valkey-docs, expiring keys are the sessions and the cache; on valkey-yhub the streams and locks."), + panel("Cache hit ratio", [target(f"sum(rate(redis_keyspace_hits_total{{{J}}}[$__rate_interval])) by (job, instance) / (sum(rate(redis_keyspace_hits_total{{{J}}}[$__rate_interval])) by (job, instance) + sum(rate(redis_keyspace_misses_total{{{J}}}[$__rate_interval])) by (job, instance))", "{{job}} {{instance}}")], "percentunit", mn=0, mx=1), + row("Streams (valkey-yhub, needs --check-streams on the exporter)"), + panel("Stream length", [target(f"max(redis_stream_length{{{J}}}) by (job, stream)", "{{job}} {{stream}}")], "short", + desc="yhub:worker is the compaction queue; the others are one per open document. Only exported when the exporter is started with --check-streams (REDIS_EXPORTER_CHECK_STREAMS=yhub:*)."), + panel("Entries added", [target(f"sum(rate(redis_stream_entries_added_total{{{J}}}[$__rate_interval])) by (job, stream)", "{{job}} {{stream}}")], "ops"), + panel("Consumer group pending and lag", [target(f"max(redis_stream_group_messages_pending{{{J}}}) by (job, stream, group)", "{{stream}} {{group}} pending"), target(f"max(redis_stream_group_lag{{{J}}}) by (job, stream, group)", "{{stream}} {{group}} lag")], "short", + desc="Pending: claimed by a worker and not acknowledged yet. Lag: not claimed by anyone yet."), + row("Replication and Sentinel"), + panel("Replicas per master", [target(f"max(redis_connected_slaves{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "short"), + panel("Replication offset", [target(f"max(redis_master_repl_offset{{{J}}}) by (job, instance)", "{{job}} {{instance}} master"), target(f"max(redis_slave_repl_offset{{{J}}}) by (job, instance)", "{{job}} {{instance}} replica")], "bytes", + desc="A replica falling behind its master is what turns a failover into data loss."), + panel("Replica link", [target(f"min(redis_master_link_up{{{J}}}) by (job, instance)", "{{job}} {{instance}} link up"), target(f"max(redis_master_last_io_seconds_ago{{{J}}}) by (job, instance)", "{{job}} {{instance}} last io (s)")], "short"), + panel("Sentinel: quorum", [target('min(redis_sentinel_master_ok_sentinels) by (job, master_name)', "{{job}} {{master_name}} sentinels ok"), target('min(redis_sentinel_master_ok_slaves) by (job, master_name)', "{{job}} {{master_name}} replicas ok"), target('min(redis_sentinel_master_ckquorum_status) by (job, master_name)', "{{job}} {{master_name}} quorum")], "short", + desc="From the sentinel pods, when they are scraped too. Sentinels ok below the quorum, or quorum status 0, means no failover is possible."), + panel("Sentinel: master status", [target('max(redis_sentinel_master_status) by (job, master_name, master_status)', "{{job}} {{master_name}} {{master_status}}")], "short", + desc="ok, or s_down / o_down when the sentinels have lost the master: the 2026-09-14 failover, seen from their side."), + row("Persistence"), + panel("Changes since last save", [target(f"max(redis_rdb_changes_since_last_save{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "short"), + panel("Last fork", [target(f"max(redis_latest_fork_seconds{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "s", + desc="A fork for a snapshot or an AOF rewrite stalls valkey for this long."), + ], variables=[("job", "instance (job)", "label_values(redis_up, job)"), ("instance", "pod", 'label_values(redis_up{job=~"$job"}, instance)')]) + +for name, d in [("users", users), ("collaboration", collab), ("backend", backend), ("valkey", valkey)]: + with open(f"{OUT}/{name}.json", "w") as f: + json.dump(d, f, indent=2) + f.write("\n") +print("generated") diff --git a/src/loadtest/dashboards/users.json b/src/loadtest/dashboards/users.json new file mode 100644 index 000000000..484ade18a --- /dev/null +++ b/src/loadtest/dashboards/users.json @@ -0,0 +1,625 @@ +{ + "uid": "docs-loadtest-users", + "title": "Docs load test \u2014 users", + "description": "What a user feels during a run: the browser canaries (src/loadtest/canary). The other boards say why.", + "tags": [ + "docs", + "loadtest" + ], + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "time": { + "from": "now-1h", + "to": "now" + }, + "refresh": "15s", + "timezone": "browser", + "templating": { + "list": [ + { + "name": "datasource", + "label": "Prometheus", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "refresh": 1 + } + ] + }, + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "Canary \u2014 real browsers", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 0 + }, + "panels": [], + "id": 1 + }, + { + "type": "timeseries", + "title": "Page open", + "description": "From the navigation to the document page being visible.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.5, sum(rate(canary_page_open_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(canary_page_open_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 1 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 2 + }, + { + "type": "timeseries", + "title": "Editor ready", + "description": "From the navigation to the editor accepting input: the collaboration provider synced. The number to compare against the 2 s target of the plan.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.5, sum(rate(canary_editor_ready_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(canary_editor_ready_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 1 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 3 + }, + { + "type": "timeseries", + "title": "Keystroke to the other screen", + "description": "From a keystroke in one browser to the text showing in another browser on the same document. Target of the plan: p95 under 500 ms.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.5, sum(rate(canary_propagation_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(canary_propagation_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 9 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 4 + }, + { + "type": "timeseries", + "title": "Iterations", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(canary_iterations_total[$__rate_interval])) by (result)", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 9 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 5 + }, + { + "type": "timeseries", + "title": "Failures by step", + "description": "page-open: the page never showed. editor-ready: the provider never synced. propagation: the text never reached the other browser.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(canary_failures_total[$__rate_interval])) by (step)", + "legendFormat": "{{step}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 17 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 6 + }, + { + "type": "timeseries", + "title": "Console errors", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(canary_console_errors_total[$__rate_interval]))", + "legendFormat": "errors/s", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 6, + "h": 8, + "x": 12, + "y": 17 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7 + }, + { + "type": "stat", + "title": "Pairs running", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(canary_pairs)", + "legendFormat": "pairs", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 6, + "h": 8, + "x": 18, + "y": 17 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 8 + }, + { + "type": "row", + "title": "Swarm and k6, as a user would feel them", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 25 + }, + "panels": [], + "id": 9 + }, + { + "type": "timeseries", + "title": "Swarm: time to connect and to first sync", + "description": "What a client of the swarm waits before the document is usable.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(swarm_connect_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "connect p95", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(swarm_sync_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "sync p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 26 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 10 + }, + { + "type": "timeseries", + "title": "Swarm: edit propagation", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.5, sum(rate(swarm_propagation_latency_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(swarm_propagation_latency_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum(rate(swarm_propagation_latency_seconds_bucket{}[$__rate_interval])) by (le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 26 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 11 + }, + { + "type": "timeseries", + "title": "k6: page-open requests, p99 by endpoint", + "description": "k6 exports its trends as gauges over the remote-write interval, p99 by default (K6_PROMETHEUS_RW_TREND_STATS adds p95).", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(k6_http_req_duration_p99{name!=\"\"}) by (name)", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 34 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 12 + }, + { + "type": "timeseries", + "title": "k6: failed requests", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(k6_http_req_failed_rate)", + "legendFormat": "failed", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "1 - max(k6_checks_rate)", + "legendFormat": "checks failed", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": {}, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 34 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 13 + } + ] +} diff --git a/src/loadtest/dashboards/valkey.json b/src/loadtest/dashboards/valkey.json new file mode 100644 index 000000000..d4082185b --- /dev/null +++ b/src/loadtest/dashboards/valkey.json @@ -0,0 +1,1533 @@ +{ + "uid": "docs-loadtest-valkey", + "title": "Docs load test \u2014 valkey", + "description": "The two Valkey instances (valkey-docs: backend cache, sessions, Celery; valkey-yhub: the collaboration server's streams), through redis_exporter. One target per pod.", + "tags": [ + "docs", + "loadtest" + ], + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "time": { + "from": "now-1h", + "to": "now" + }, + "refresh": "15s", + "timezone": "browser", + "templating": { + "list": [ + { + "name": "datasource", + "label": "Prometheus", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "refresh": 1 + }, + { + "name": "job", + "label": "instance (job)", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(redis_up, job)", + "refId": "job" + }, + "definition": "label_values(redis_up, job)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "instance", + "label": "pod", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(redis_up{job=~\"$job\"}, instance)", + "refId": "instance" + }, + "definition": "label_values(redis_up{job=~\"$job\"}, instance)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "Health", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 0 + }, + "panels": [], + "id": 1 + }, + { + "type": "stat", + "title": "Up", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "min(redis_up{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {}, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "gridPos": { + "w": 6, + "h": 8, + "x": 0, + "y": 1 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 2 + }, + { + "type": "stat", + "title": "Role", + "description": "master or slave, as INFO says it. A change is a failover.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_instance_info{job=~\"$job\", instance=~\"$instance\"}) by (job, instance, role)", + "legendFormat": "{{job}} {{instance}}: {{role}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 6, + "h": 8, + "x": 6, + "y": 1 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "none", + "graphMode": "none", + "textMode": "name" + }, + "id": 3 + }, + { + "type": "stat", + "title": "Uptime", + "description": "A reset is a restart: what wiped the sessions on 2026-09-07.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "min(redis_uptime_in_seconds{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 6, + "h": 8, + "x": 12, + "y": 1 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "id": 4 + }, + { + "type": "timeseries", + "title": "Connected clients", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(redis_connected_clients{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 6, + "h": 8, + "x": 18, + "y": 1 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 5 + }, + { + "type": "row", + "title": "Memory", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 9 + }, + "panels": [], + "id": 6 + }, + { + "type": "timeseries", + "title": "Memory used", + "description": "Against maxmemory (0 when there is none, then hidden). The yhub streams have no TTL and cannot be evicted under volatile-lru: this is the number that decides how many replicas and how much history valkey-yhub can hold.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_memory_used_bytes{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}} used", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_memory_max_bytes{job=~\"$job\", instance=~\"$instance\"} > 0) by (job, instance)", + "legendFormat": "{{job}} {{instance}} max", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 10 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7 + }, + { + "type": "timeseries", + "title": "Memory used, share of max", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_memory_used_bytes{job=~\"$job\", instance=~\"$instance\"} / (redis_memory_max_bytes{job=~\"$job\", instance=~\"$instance\"} > 0)) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": {}, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 10 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8 + }, + { + "type": "timeseries", + "title": "Evicted and expired keys", + "description": "Evictions on valkey-docs are sessions and cache entries thrown away under memory pressure.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(redis_evicted_keys_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance)", + "legendFormat": "{{job}} {{instance}} evicted", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(redis_expired_keys_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance)", + "legendFormat": "{{job}} {{instance}} expired", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 18 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 9 + }, + { + "type": "timeseries", + "title": "Fragmentation ratio", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_mem_fragmentation_ratio{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 18 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 10 + }, + { + "type": "row", + "title": "Traffic", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 26 + }, + "panels": [], + "id": 11 + }, + { + "type": "timeseries", + "title": "Commands", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(redis_commands_processed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 27 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 12 + }, + { + "type": "timeseries", + "title": "Commands by command, top 10", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(10, sum(rate(redis_commands_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, cmd))", + "legendFormat": "{{job}} {{cmd}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 27 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 13 + }, + { + "type": "timeseries", + "title": "Command latency p99", + "description": "Server-side, from LATENCY HISTOGRAM: the time valkey spends on a command, not what the client waits.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum(rate(redis_commands_latencies_usec_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (le, job, instance)) / 1e6", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 35 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 14 + }, + { + "type": "timeseries", + "title": "Time spent in commands, top 10", + "description": "Seconds of command time per second: which commands the CPU goes to.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(10, sum(rate(redis_commands_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, cmd))", + "legendFormat": "{{job}} {{cmd}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 35 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 15 + }, + { + "type": "timeseries", + "title": "Network", + "description": "On valkey-yhub the out side is the fan-out of every update to every replica of yhub.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(redis_net_input_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance)", + "legendFormat": "{{job}} {{instance}} in", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(redis_net_output_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance)", + "legendFormat": "{{job}} {{instance}} out", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 43 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 16 + }, + { + "type": "timeseries", + "title": "CPU", + "description": "One core is the ceiling: valkey runs its commands on one thread.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(redis_cpu_sys_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) + rate(redis_cpu_user_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 43 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 17 + }, + { + "type": "timeseries", + "title": "Blocked clients and rejected connections", + "description": "Blocked: yhub replicas waiting on XREAD, which is normal. Rejected: maxclients reached.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(redis_blocked_clients{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}} blocked", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(redis_rejected_connections_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance)", + "legendFormat": "{{job}} {{instance}} rejected/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 51 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 18 + }, + { + "type": "timeseries", + "title": "Slow log length", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_slowlog_length{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 51 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 19 + }, + { + "type": "row", + "title": "Keys", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 59 + }, + "panels": [], + "id": 20 + }, + { + "type": "timeseries", + "title": "Keys", + "description": "On valkey-docs, expiring keys are the sessions and the cache; on valkey-yhub the streams and locks.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(redis_db_keys{job=~\"$job\", instance=~\"$instance\"}) by (job, instance, db)", + "legendFormat": "{{job}} {{instance}} {{db}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(redis_db_keys_expiring{job=~\"$job\", instance=~\"$instance\"}) by (job, instance, db)", + "legendFormat": "{{job}} {{instance}} {{db}} expiring", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 60 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 21 + }, + { + "type": "timeseries", + "title": "Cache hit ratio", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(redis_keyspace_hits_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance) / (sum(rate(redis_keyspace_hits_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance) + sum(rate(redis_keyspace_misses_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance))", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": {}, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 60 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 22 + }, + { + "type": "row", + "title": "Streams (valkey-yhub, needs --check-streams on the exporter)", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 68 + }, + "panels": [], + "id": 23 + }, + { + "type": "timeseries", + "title": "Stream length", + "description": "yhub:worker is the compaction queue; the others are one per open document. Only exported when the exporter is started with --check-streams (REDIS_EXPORTER_CHECK_STREAMS=yhub:*).", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_stream_length{job=~\"$job\", instance=~\"$instance\"}) by (job, stream)", + "legendFormat": "{{job}} {{stream}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 69 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 24 + }, + { + "type": "timeseries", + "title": "Entries added", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(redis_stream_entries_added_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, stream)", + "legendFormat": "{{job}} {{stream}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 69 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 25 + }, + { + "type": "timeseries", + "title": "Consumer group pending and lag", + "description": "Pending: claimed by a worker and not acknowledged yet. Lag: not claimed by anyone yet.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_stream_group_messages_pending{job=~\"$job\", instance=~\"$instance\"}) by (job, stream, group)", + "legendFormat": "{{stream}} {{group}} pending", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_stream_group_lag{job=~\"$job\", instance=~\"$instance\"}) by (job, stream, group)", + "legendFormat": "{{stream}} {{group}} lag", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 77 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 26 + }, + { + "type": "row", + "title": "Replication and Sentinel", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 85 + }, + "panels": [], + "id": 27 + }, + { + "type": "timeseries", + "title": "Replicas per master", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_connected_slaves{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 86 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 28 + }, + { + "type": "timeseries", + "title": "Replication offset", + "description": "A replica falling behind its master is what turns a failover into data loss.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_master_repl_offset{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}} master", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_slave_repl_offset{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}} replica", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 86 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 29 + }, + { + "type": "timeseries", + "title": "Replica link", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "min(redis_master_link_up{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}} link up", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_master_last_io_seconds_ago{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}} last io (s)", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 94 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 30 + }, + { + "type": "timeseries", + "title": "Sentinel: quorum", + "description": "From the sentinel pods, when they are scraped too. Sentinels ok below the quorum, or quorum status 0, means no failover is possible.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "min(redis_sentinel_master_ok_sentinels) by (job, master_name)", + "legendFormat": "{{job}} {{master_name}} sentinels ok", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "min(redis_sentinel_master_ok_slaves) by (job, master_name)", + "legendFormat": "{{job}} {{master_name}} replicas ok", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "min(redis_sentinel_master_ckquorum_status) by (job, master_name)", + "legendFormat": "{{job}} {{master_name}} quorum", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 94 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 31 + }, + { + "type": "timeseries", + "title": "Sentinel: master status", + "description": "ok, or s_down / o_down when the sentinels have lost the master: the 2026-09-14 failover, seen from their side.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_sentinel_master_status) by (job, master_name, master_status)", + "legendFormat": "{{job}} {{master_name}} {{master_status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 102 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 32 + }, + { + "type": "row", + "title": "Persistence", + "collapsed": false, + "gridPos": { + "w": 24, + "h": 1, + "x": 0, + "y": 110 + }, + "panels": [], + "id": 33 + }, + { + "type": "timeseries", + "title": "Changes since last save", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_rdb_changes_since_last_save{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 0, + "y": 111 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 34 + }, + { + "type": "timeseries", + "title": "Last fork", + "description": "A fork for a snapshot or an AOF rewrite stalls valkey for this long.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(redis_latest_fork_seconds{job=~\"$job\", instance=~\"$instance\"}) by (job, instance)", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "w": 12, + "h": 8, + "x": 12, + "y": 111 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 35 + } + ] +}