diff --git a/CHANGELOG.md b/CHANGELOG.md
index 505b5d450..5c3083d6c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,10 @@ and this project adheres to
- ✨(loadtest) add k6 scenarios for the page-open sequence and the heavy
endpoints
- ✨(loadtest) add browser canaries measuring what a user feels under load
+- ✨(loadtest) add the grafana dashboards of the load-test campaign, valkey
+ included
+- 🔧(helm) run grafana with those dashboards in the dev cluster, in place of
+ the prometheus console
- ✨(backend) measure the calls to yhub and to the converters, the database
pool and the celery queue
- ✨(backend) add a `LoadTest` configuration and its `loadtest` application,
diff --git a/bin/Tiltfile b/bin/Tiltfile
index ffdbe5f2e..55cd7f255 100644
--- a/bin/Tiltfile
+++ b/bin/Tiltfile
@@ -84,17 +84,27 @@ k8s_resource('prometheus-operator', objects=[
])
k8s_resource('prometheus-prometheus', objects=[
'prometheus-prometheus:ingress',
- # the example console of django-prometheus and the rules feeding it
- 'django-console:configmap',
- 'django:prometheusrule',
], resource_deps=['prometheus-operator'], links=[
'https://docs-prometheus.127.0.0.1.nip.io/targets',
- 'https://docs-prometheus.127.0.0.1.nip.io/consoles/django.html',
+])
+# Grafana, with the dashboards of the load-test campaign
+# (src/loadtest/dashboards) as ConfigMaps its sidecar loads
+k8s_resource('prometheus-grafana', objects=[
+ 'prometheus-grafana:ingress',
+ 'dashboard-users:configmap',
+ 'dashboard-collaboration:configmap',
+ 'dashboard-backend:configmap',
+ 'dashboard-django:configmap',
+ 'dashboard-valkey:configmap',
+], resource_deps=['prometheus-prometheus'], links=[
+ 'https://docs-grafana.127.0.0.1.nip.io/dashboards',
])
k8s_resource(new_name='impress-docs-monitors', objects=[
'impress-docs-backend:servicemonitor',
'impress-docs-yhub:servicemonitor',
'impress-docs-yhub-worker:servicemonitor',
+ 'valkey-docs:servicemonitor',
+ 'valkey-yhub:servicemonitor',
], resource_deps=['prometheus-operator'])
# the valkey chart leaves its objects without a namespace, and the CRDs of the
diff --git a/documentation/metrics.md b/documentation/metrics.md
index cd630e9d1..d08770ffa 100644
--- a/documentation/metrics.md
+++ b/documentation/metrics.md
@@ -220,14 +220,12 @@ on with a token in the `docs-metrics` Secret, and `serviceMonitor.enabled`
builds the three monitors. The targets are at
https://docs-prometheus.127.0.0.1.nip.io/targets.
-That Prometheus also serves the
-[example console of django-prometheus](https://github.com/django-commons/django-prometheus/tree/master/examples/prometheus)
-at https://docs-prometheus.127.0.0.1.nip.io/consoles/django.html: requests per
-second, by view, median and tail latency, model writes and database queries,
-drawn from its recording rules. Both files are in
-`src/helm/env.d/dev/prometheus/`, the rules verbatim and the console with its
-job renamed to `backend`. Console templates draw with the classic UI, which
-Prometheus 3 removed, so the dev Prometheus is the last 2.x release.
+Next to it, a Grafana at https://docs-grafana.127.0.0.1.nip.io (admin /
+admin) with the dashboards of the load-test campaign, from
+`src/loadtest/dashboards/` (see its README): what the users feel, the
+collaboration server, the backend, and the generic
+[Django dashboard](https://grafana.com/grafana/dashboards/17658-django/)
+of the community for the per-view detail of django-prometheus.
### A Prometheus outside of the cluster
diff --git a/documentation/stress-test-plan.md b/documentation/stress-test-plan.md
index bcb880f62..711ac4db1 100644
--- a/documentation/stress-test-plan.md
+++ b/documentation/stress-test-plan.md
@@ -295,17 +295,32 @@ Rules for the load clients:
### 0.5 Dashboards
-One board per question, built before the first run:
+**Implemented** (2026-09-22): `src/loadtest/dashboards/` (see its README),
+loaded into the Grafana of the dev cluster
+(`src/helm/env.d/dev/values.prometheus.yaml.gotmpl`, Grafana 13, Prometheus
+3.14, in place of the django-prometheus console). Every panel query was
+checked against live data from the backend, yhub, the swarm, k6 and the canary.
-- users: `canary_page_open_seconds`, `canary_editor_ready_seconds`,
- `canary_propagation_seconds`, `canary_failures_total{step}` (from
- `src/loadtest/canary/`);
-- yhub: sockets and rooms per pod, event-loop lag, auth duration, backend
- call duration and inflight, pending tasks, task duration, seeds;
-- Django: latency and rate per view, yhub client latency, pool waiting,
- Celery queue length;
-- stores: Postgres connections and top queries, Valkey memory, commands and
- evictions.
+- users: `users.json` — the canary (page open, editor ready, keystroke to the
+ other screen, failures by step), plus the swarm's connect and propagation
+ and k6's latency and failures;
+- yhub: `collaboration.json` — sockets and rooms per replica, event-loop lag,
+ auth duration and results, backend calls (duration, in flight, failures),
+ compaction backlog and duration, seeds; then the swarm's view;
+- Django: `backend.json` — requests and latency by view, 5xx, calls to yhub
+ and the converters, the psycopg pool, queries, Celery queue length; then
+ k6's view. `django.json` is the community Django dashboard (17658) for the
+ per-view detail;
+- Valkey: `valkey.json` — the two instances through the operator's
+ `redis_exporter`: memory against `maxmemory`, evictions, commands and their
+ latency, network, CPU, the yhub streams, replication, Sentinel. The streams
+ row needs `--check-streams`, which the operator's exporter spec cannot pass
+ (see the README);
+- Postgres: not here, the exporter belongs to the team running it. The README
+ lists the queries the campaign needs from it.
+
+Pod CPU and memory come from the cluster's cAdvisor, not from these boards:
+the backend exports no process metrics in multiprocess mode.
## 1. Tooling
diff --git a/src/helm/env.d/dev/prometheus/django.html b/src/helm/env.d/dev/prometheus/django.html
deleted file mode 100644
index ac48b362f..000000000
--- a/src/helm/env.d/dev/prometheus/django.html
+++ /dev/null
@@ -1,201 +0,0 @@
-{{/*
- The example console of django-prometheus, from
- https://github.com/django-commons/django-prometheus/blob/master/examples/prometheus/consoles/django.html
- with two changes: the job is `backend` here (the ServiceMonitor names it after
- the component), and the connection errors graph read a metric that does not
- exist (`erros`). The graphs come from the recording rules of django.rules.
-*/}}
-{{template "head" .}}
-
-{{template "prom_right_table_head"}}
-
- | Django |
- {{ template "prom_query_drilldown" (args "sum(up{job='backend'})") }}
- / {{ template "prom_query_drilldown" (args "count(up{job='backend'})") }}
- |
-
-
- | avg CPU |
- {{ template "prom_query_drilldown" (args "avg by(job)(rate(process_cpu_seconds_total{job='backend'}[5m]))" "s/s" "humanizeNoSmallPrefix") }}
- |
-
-
- | avg Memory |
- {{ template "prom_query_drilldown" (args "avg by(job)(process_resident_memory_bytes{job='backend'})" "B" "humanize1024") }}
- |
-
-{{template "prom_right_table_tail"}}
-
-
-{{template "prom_content_head" .}}
-Django
-
-Requests
-Total
-
-
-
-By view
-
-
-
-Latency (median)
-
-
-
-Latency (99.9th percentile)
-
-
-
-Models
-Insertions/s
-
-
-
-Updates/s
-
-
-
-Deletions/s
-
-
-
-Database
-Connections/s
-
-
-
-Connections errors/s
-
-
-
-Queries/s
-
-
-
-Errors/s
-
-
-
-{{template "prom_content_tail" .}}
-
-{{template "tail"}}
diff --git a/src/helm/env.d/dev/prometheus/django.rules b/src/helm/env.d/dev/prometheus/django.rules
deleted file mode 100644
index 54209951e..000000000
--- a/src/helm/env.d/dev/prometheus/django.rules
+++ /dev/null
@@ -1,108 +0,0 @@
-# The example recording rules of django-prometheus, verbatim, from
-# https://github.com/django-commons/django-prometheus/blob/master/examples/prometheus/django.rules
-# They are what the graphs of django.html read.
-groups:
-- name: django.rules
- rules:
- - record: job:django_http_requests_before_middlewares_total:sum_rate30s
- expr: sum(rate(django_http_requests_before_middlewares_total[30s])) BY (job)
- - record: job:django_http_requests_unknown_latency_total:sum_rate30s
- expr: sum(rate(django_http_requests_unknown_latency_total[30s])) BY (job)
- - record: job:django_http_ajax_requests_total:sum_rate30s
- expr: sum(rate(django_http_ajax_requests_total[30s])) BY (job)
- - record: job:django_http_responses_before_middlewares_total:sum_rate30s
- expr: sum(rate(django_http_responses_before_middlewares_total[30s])) BY (job)
- - record: job:django_http_requests_unknown_latency_including_middlewares_total:sum_rate30s
- expr: sum(rate(django_http_requests_unknown_latency_including_middlewares_total[30s]))
- BY (job)
- - record: job:django_http_requests_body_total_bytes:sum_rate30s
- expr: sum(rate(django_http_requests_body_total_bytes[30s])) BY (job)
- - record: job:django_http_responses_streaming_total:sum_rate30s
- expr: sum(rate(django_http_responses_streaming_total[30s])) BY (job)
- - record: job:django_http_responses_body_total_bytes:sum_rate30s
- expr: sum(rate(django_http_responses_body_total_bytes[30s])) BY (job)
- - record: job:django_http_requests_total:sum_rate30s
- expr: sum(rate(django_http_requests_total_by_method[30s])) BY (job)
- - record: job:django_http_requests_total_by_method:sum_rate30s
- expr: sum(rate(django_http_requests_total_by_method[30s])) BY (job, method)
- - record: job:django_http_requests_total_by_transport:sum_rate30s
- expr: sum(rate(django_http_requests_total_by_transport[30s])) BY (job, transport)
- - record: job:django_http_requests_total_by_view:sum_rate30s
- expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) BY (job,
- view)
- - record: job:django_http_requests_total_by_view_transport_method:sum_rate30s
- expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) BY (job,
- view, transport, method)
- - record: job:django_http_responses_total_by_templatename:sum_rate30s
- expr: sum(rate(django_http_responses_total_by_templatename[30s])) BY (job, templatename)
- - record: job:django_http_responses_total_by_status:sum_rate30s
- expr: sum(rate(django_http_responses_total_by_status[30s])) BY (job, status)
- - record: job:django_http_responses_total_by_status_name_method:sum_rate30s
- expr: sum(rate(django_http_responses_total_by_status_name_method[30s])) BY (job,
- status, name, method)
- - record: job:django_http_responses_total_by_charset:sum_rate30s
- expr: sum(rate(django_http_responses_total_by_charset[30s])) BY (job, charset)
- - record: job:django_http_exceptions_total_by_type:sum_rate30s
- expr: sum(rate(django_http_exceptions_total_by_type[30s])) BY (job, type)
- - record: job:django_http_exceptions_total_by_view:sum_rate30s
- expr: sum(rate(django_http_exceptions_total_by_view[30s])) BY (job, view)
- - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
- expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
- BY (job, le))
- labels:
- quantile: "50"
- - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
- expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
- BY (job, le))
- labels:
- quantile: "95"
- - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
- expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
- BY (job, le))
- labels:
- quantile: "99"
- - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
- expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
- BY (job, le))
- labels:
- quantile: "99.9"
- - record: job:django_http_requests_latency_seconds:quantile_rate30s
- expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
- BY (job, le))
- labels:
- quantile: "50"
- - record: job:django_http_requests_latency_seconds:quantile_rate30s
- expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
- BY (job, le))
- labels:
- quantile: "95"
- - record: job:django_http_requests_latency_seconds:quantile_rate30s
- expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
- BY (job, le))
- labels:
- quantile: "99"
- - record: job:django_http_requests_latency_seconds:quantile_rate30s
- expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_seconds_bucket[30s]))
- BY (job, le))
- labels:
- quantile: "99.9"
- - record: job:django_model_inserts_total:sum_rate1m
- expr: sum(rate(django_model_inserts_total[1m])) BY (job, model)
- - record: job:django_model_updates_total:sum_rate1m
- expr: sum(rate(django_model_updates_total[1m])) BY (job, model)
- - record: job:django_model_deletes_total:sum_rate1m
- expr: sum(rate(django_model_deletes_total[1m])) BY (job, model)
- - record: job:django_db_new_connections_total:sum_rate30s
- expr: sum(rate(django_db_new_connections_total[30s])) BY (alias, vendor)
- - record: job:django_db_new_connection_errors_total:sum_rate30s
- expr: sum(rate(django_db_new_connection_errors_total[30s])) BY (alias, vendor)
- - record: job:django_db_execute_total:sum_rate30s
- expr: sum(rate(django_db_execute_total[30s])) BY (alias, vendor)
- - record: job:django_db_execute_many_total:sum_rate30s
- expr: sum(rate(django_db_execute_many_total[30s])) BY (alias, vendor)
- - record: job:django_db_errors_total:sum_rate30s
- expr: sum(rate(django_db_errors_total[30s])) BY (alias, vendor, type)
- - record: job:django_migrations_applied_total:max
- expr: max(django_migrations_applied_total) BY (job, connection)
- - record: job:django_migrations_unapplied_total:max
- expr: max(django_migrations_unapplied_total) BY (job, connection)
diff --git a/src/helm/env.d/dev/values.prometheus.yaml.gotmpl b/src/helm/env.d/dev/values.prometheus.yaml.gotmpl
index 448a9243e..90225cdb9 100644
--- a/src/helm/env.d/dev/values.prometheus.yaml.gotmpl
+++ b/src/helm/env.d/dev/values.prometheus.yaml.gotmpl
@@ -1,15 +1,11 @@
-# A Prometheus scraping the dev cluster, to see the metrics of the backend and
-# of yhub through the ServiceMonitors the impress chart builds. Trimmed down to
-# the Prometheus Operator, its CRDs and one Prometheus: no Grafana, no
-# Alertmanager, no exporter, and none of the monitors and rules of the
-# kubernetes control plane, which a kind cluster has nothing to show for.
-# UI: https://docs-prometheus.127.0.0.1.nip.io (Status > Targets lists the pods)
-#
-# It also serves the example console of django-prometheus, the graphs of
-# https://docs-prometheus.127.0.0.1.nip.io/consoles/django.html, fed by its
-# recording rules: both are in the prometheus/ directory next to this file.
-# Console templates draw with the classic UI, which Prometheus 3 removed along
-# with the console libraries, so this Prometheus is the last 2.x release.
+# A Prometheus and a Grafana for the dev cluster, to see the metrics of the
+# backend and of yhub through the ServiceMonitors the impress chart builds, and
+# the dashboards of the load-test campaign (src/loadtest/dashboards). Trimmed
+# down to the Prometheus Operator, its CRDs, one Prometheus and one Grafana: no
+# Alertmanager, no exporter, and none of the monitors, rules and dashboards of
+# the kubernetes control plane, which a kind cluster has nothing to show for.
+# Prometheus: https://docs-prometheus.127.0.0.1.nip.io (Status > Targets)
+# Grafana: https://docs-grafana.127.0.0.1.nip.io (admin / admin)
fullnameOverride: prometheus
defaultRules:
@@ -17,7 +13,25 @@ defaultRules:
alertmanager:
enabled: false
grafana:
- enabled: false
+ enabled: true
+ adminPassword: admin
+ # none of the kubernetes dashboards of the chart: the boards of the campaign
+ # only, loaded from the ConfigMaps below by the sidecar
+ defaultDashboardsEnabled: false
+ sidecar:
+ dashboards:
+ folderAnnotation: grafana_folder
+ provider:
+ foldersFromFilesStructure: true
+ ingress:
+ enabled: true
+ hosts:
+ - docs-grafana.127.0.0.1.nip.io
+ path: /
+ tls:
+ - secretName: docs-tls
+ hosts:
+ - docs-grafana.127.0.0.1.nip.io
nodeExporter:
enabled: false
kubeStateMetrics:
@@ -57,19 +71,6 @@ prometheus:
serviceMonitor:
selfMonitor: false
prometheusSpec:
- image:
- tag: v2.55.1
- # the console, in the directory the operator points --web.console.templates
- # at (/etc/prometheus/consoles is a symlink to it in the image). Only the
- # console libraries of the image are left next to it, which is all it needs
- volumes:
- - name: django-console
- configMap:
- name: django-console
- volumeMounts:
- - name: django-console
- mountPath: /usr/share/prometheus/consoles
- readOnly: true
# every ServiceMonitor and PodMonitor of the namespace, whatever its labels
serviceMonitorSelectorNilUsesHelmValues: false
podMonitorSelectorNilUsesHelmValues: false
@@ -107,23 +108,25 @@ extraManifests:
name: docs-metrics
stringData:
PROMETHEUS_API_KEY: ThisIsAnExampleMetricsTokenForDevPurposeOnly
- # The recording rules the console reads, selected like the monitors are
- - apiVersion: monitoring.coreos.com/v1
- kind: PrometheusRule
- metadata:
- name: django
- spec:
- {{- readFile "prometheus/django.rules" | nindent 6 }}
- # The console itself. The chart passes every extra manifest through helm's
- # tpl, which would render the actions of the console template: each opening
- # delimiter is escaped so that tpl writes it back as is (the closing one is
- # spelled in two halves so that it does not end this action)
+ # The dashboards of the load-test campaign, one ConfigMap each, which the
+ # sidecar of Grafana loads by their label into the folder of the annotation.
+ # The chart passes every extra manifest through helm's tpl, which would
+ # render the legend templates of a dashboard (a label name between double
+ # braces): each opening delimiter is escaped so that tpl writes it back as is
+ # (the closing one is spelled in two halves so that it does not end this
+ # action). This file is a template too, hence no brace in this comment.
{{- $open := "{{" }}
{{- $escaped := printf "%s %q %s" $open $open (print "}" "}") }}
+ {{- range $name := list "users" "collaboration" "backend" "django" "valkey" }}
- apiVersion: v1
kind: ConfigMap
metadata:
- name: django-console
+ name: dashboard-{{ $name }}
+ labels:
+ grafana_dashboard: "1"
+ annotations:
+ grafana_folder: Docs
data:
- django.html: |
- {{- readFile "prometheus/django.html" | replace $open $escaped | nindent 8 }}
+ {{ $name }}.json: |
+ {{- readFile (printf "../../../loadtest/dashboards/%s.json" $name) | replace $open $escaped | nindent 8 }}
+ {{- end }}
diff --git a/src/helm/env.d/dev/values.valkey.yaml.gotmpl b/src/helm/env.d/dev/values.valkey.yaml.gotmpl
index 5791edd50..91e04bf3e 100644
--- a/src/helm/env.d/dev/values.valkey.yaml.gotmpl
+++ b/src/helm/env.d/dev/values.valkey.yaml.gotmpl
@@ -10,3 +10,19 @@ auth:
# helm test pod that Tilt would run as a plain pod
aclConfig: |
user default on >pass ~* &* +@all
+
+# redis_exporter next to the pod, for the Prometheus of the prometheus release
+# (values.prometheus.yaml.gotmpl) and the valkey board of src/loadtest/dashboards.
+# The chart hands the exporter the password of the instance itself. The check
+# of the yhub streams (their length, the pending entries of the worker group)
+# is what the board's streams row reads; on valkey-docs it matches nothing and
+# costs nothing. The ServiceMonitor is only built with the operator's CRDs, i.e.
+# with the prometheus release (`monitoring` in the helmfile environment).
+metrics:
+ enabled: true
+ exporter:
+ extraEnvs:
+ REDIS_EXPORTER_CHECK_STREAMS: "yhub:*"
+ serviceMonitor:
+ enabled: {{ .Values | get "monitoring" false }}
+ interval: 15s
diff --git a/src/helm/helmfile.yaml.gotmpl b/src/helm/helmfile.yaml.gotmpl
index 5cce13c57..a3391a67f 100644
--- a/src/helm/helmfile.yaml.gotmpl
+++ b/src/helm/helmfile.yaml.gotmpl
@@ -34,6 +34,11 @@ releases:
namespace: {{ .Namespace }}
chart: valkey/valkey
version: 0.12.0
+ {{- if .Values | get "monitoring" false }}
+ # its ServiceMonitor needs the CRDs of the operator
+ needs:
+ - {{ .Namespace }}/prometheus
+ {{- end }}
values:
- env.d/{{ .Environment.Name }}/values.valkey.yaml.gotmpl
@@ -41,6 +46,11 @@ releases:
namespace: {{ .Namespace }}
chart: valkey/valkey
version: 0.12.0
+ {{- if .Values | get "monitoring" false }}
+ # its ServiceMonitor needs the CRDs of the operator
+ needs:
+ - {{ .Namespace }}/prometheus
+ {{- end }}
values:
- env.d/{{ .Environment.Name }}/values.valkey.yaml.gotmpl
diff --git a/src/loadtest/dashboards/README.md b/src/loadtest/dashboards/README.md
new file mode 100644
index 000000000..2c8b24189
--- /dev/null
+++ b/src/loadtest/dashboards/README.md
@@ -0,0 +1,88 @@
+# Dashboards of the load-test campaign
+
+Grafana dashboards over the metrics of the campaign: the backend's and yhub's
+`/metrics`, and what the load generators export (`../swarm`, `../k6`,
+`../canary`). One board per question of `documentation/stress-test-plan.md`
+(section 0.5): what the users feel, what the collaboration server does, what
+the backend does. Each is laid out so that a saturation reads left to right:
+the symptom the clients see, the server-side cause, the resource that ran out.
+
+| File | Board | For |
+| ---- | ----- | --- |
+| `users.json` | Docs load test — users | The canary: page open, editor ready, keystroke to the other screen, failures by step. Plus the swarm's and k6's client-side numbers. The board that says "the users noticed"; the others say why |
+| `collaboration.json` | Docs load test — collaboration server | yhub per replica: sockets, rooms, event-loop lag, auth duration, calls to the backend (duration, in flight, failures), compaction backlog and duration, seeds. Then the swarm: clients by state, connect and sync, propagation, reconnects, refused upgrades, close codes, traffic |
+| `backend.json` | Docs load test — backend | Django: requests and latency by view, 5xx, calls to yhub and the converters (duration, in flight, failures), the psycopg pool (waiting, queued, wait time), queries, Celery queue length. Then k6: rate and p95 by endpoint, failures, VUs, dropped iterations |
+| `valkey.json` | Docs load test — valkey | The two Valkey instances through `redis_exporter`, one target per pod: health and role, memory against `maxmemory`, evictions, commands and their latency, network, CPU, keys, the yhub streams, replication, Sentinel quorum and master status, persistence |
+| `django.json` | Django | The community [Django dashboard 17658](https://grafana.com/grafana/dashboards/17658-django/) (revision 2), for the per-view detail of django-prometheus |
+
+A `hostname` variable on the collaboration and backend boards narrows the
+panels to one replica: the label every sample carries (`documentation/metrics.md`).
+
+## Where they run
+
+The dev cluster (`monitoring: true`, `src/helm/env.d/dev/values.prometheus.yaml.gotmpl`)
+loads them into its Grafana, https://docs-grafana.127.0.0.1.nip.io, from
+ConfigMaps built from these files. Anywhere else, import the JSON files as they
+are: every panel goes through the `datasource` variable of the board, which
+defaults to the Grafana's default Prometheus and can be switched at the top of
+the board. No uid is hardcoded.
+
+The k6 panels expect k6's Prometheus remote write
+(`-o experimental-prometheus-rw`, see `../k6/README.md`): its trends arrive as
+gauges, `k6_http_req_duration_p99` by default; set
+`K6_PROMETHEUS_RW_TREND_STATS=p(95),p(99)` for a p95 too.
+
+## What stays empty, and why
+
+- On the valkey board, `maxmemory` panels when no limit is set, the streams
+ row without `--check-streams`, the Sentinel row without the sentinel pods
+ scraped (see Stores).
+- Pod memory and CPU: not on these boards. The backend exports no process
+ metrics (prometheus_client's multiprocess mode has none); take them from
+ the cluster's cAdvisor. yhub does export `process_resident_memory_bytes`.
+- Cache hit ratio, on the Django board: django-prometheus only counts cache
+ calls through its own cache backends, which the backend does not use.
+- The canary panels are empty until a canary runs; the k6 ones until k6 runs
+ with the remote write.
+
+## Changes to the community Django dashboard
+
+`django.json` is revision 2 of dashboard 17658 with two edits, so that the
+sidecar can load it as it is: the `${DS_PROMETHEUS}` import input is replaced
+by the board's own `${datasource}` variable, and its `app` label filter (which nothing
+here sets) by `job`, the label the chart's ServiceMonitor names the component
+with (`backend`). Take a new revision from grafana.com the same way.
+
+## Stores
+
+Valkey is `valkey.json`. The team running it (chideat/valkey-operator) enables
+the operator's exporter, which is `redis_exporter` as a sidecar of every valkey
+pod, so the board has one target per pod: the `job` variable picks the
+instance, `instance` the pod. Two things about that exporter:
+
+- the operator's `exporter` spec sets an image, resources and a security
+ context, and nothing else: no arguments, no environment. The streams row
+ (`redis_stream_*`) needs `--check-streams` (or
+ `REDIS_EXPORTER_CHECK_STREAMS=yhub:*`), which the operator cannot pass.
+ What it does let through is the image (`exporter.image`), and it starts it
+ as `/redis_exporter `: an image where `/redis_exporter` is a
+ two-line wrapper setting that variable and exec'ing the real binary gets the
+ streams row with the operator's own command line (checked against
+ redis_exporter v1.92.0). Otherwise, a second exporter run with the flag
+ against valkey-yhub. The dev cluster's chart can pass the variable, and does;
+- the Sentinel row needs the sentinel pods scraped as well.
+
+Postgres is not here: that exporter belongs to the team running it. What the
+campaign needs from it, per instance:
+
+- Postgres (`postgres_exporter` or pghero): connections by state and by
+ application name, transactions and tuples per second, the slowest queries
+ (`pg_stat_statements` by total and mean time), replication lag, and the
+ Patroni leader. On the `yhub` database as well as the backend's.
+
+## Editing
+
+The three `docs-loadtest-*` boards are generated by `generate.py`
+(`python3 generate.py .`): change the generator and regenerate, rather than the
+JSON. A legend template such as `{{hostname}}` is fine in the JSON;
+the dev values escape it for helm's `tpl` when building the ConfigMaps.
diff --git a/src/loadtest/dashboards/backend.json b/src/loadtest/dashboards/backend.json
new file mode 100644
index 000000000..adb7b203d
--- /dev/null
+++ b/src/loadtest/dashboards/backend.json
@@ -0,0 +1,1101 @@
+{
+ "uid": "docs-loadtest-backend",
+ "title": "Docs load test \u2014 backend",
+ "description": "The Django backend under load: what it spends on the other services, its database pool, its queue, and what k6 sees from the outside. The generic Django board (17658) has the per-view detail.",
+ "tags": [
+ "docs",
+ "loadtest"
+ ],
+ "schemaVersion": 39,
+ "version": 1,
+ "editable": true,
+ "graphTooltip": 1,
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "refresh": "15s",
+ "timezone": "browser",
+ "templating": {
+ "list": [
+ {
+ "name": "datasource",
+ "label": "Prometheus",
+ "type": "datasource",
+ "query": "prometheus",
+ "current": {},
+ "hide": 0,
+ "refresh": 1
+ },
+ {
+ "name": "hostname",
+ "label": "backend replica",
+ "type": "query",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "query": {
+ "query": "label_values(django_http_requests_total_by_view_transport_method_total, hostname)",
+ "refId": "hostname"
+ },
+ "definition": "label_values(django_http_requests_total_by_view_transport_method_total, hostname)",
+ "refresh": 2,
+ "includeAll": true,
+ "multi": true,
+ "allValue": ".*",
+ "current": {
+ "text": "All",
+ "value": "$__all"
+ },
+ "sort": 1
+ }
+ ]
+ },
+ "annotations": {
+ "list": []
+ },
+ "panels": [
+ {
+ "type": "row",
+ "title": "Requests",
+ "collapsed": false,
+ "gridPos": {
+ "w": 24,
+ "h": 1,
+ "x": 0,
+ "y": 0
+ },
+ "panels": [],
+ "id": 1
+ },
+ {
+ "type": "timeseries",
+ "title": "Requests by view",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(django_http_requests_total_by_view_transport_method_total{hostname=~\"$hostname\"}[$__rate_interval])) by (view)",
+ "legendFormat": "{{view}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "reqps",
+ "custom": {
+ "stacking": {
+ "mode": "normal"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 1
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 2
+ },
+ {
+ "type": "timeseries",
+ "title": "Latency p95 by view",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, view))",
+ "legendFormat": "{{view}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 1
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 3
+ },
+ {
+ "type": "timeseries",
+ "title": "Responses by status",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(django_http_responses_total_by_status_view_method_total{hostname=~\"$hostname\"}[$__rate_interval])) by (status)",
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "reqps",
+ "custom": {
+ "stacking": {
+ "mode": "normal"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 9
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 4
+ },
+ {
+ "type": "timeseries",
+ "title": "Errors (5xx) by view",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(django_http_responses_total_by_status_view_method_total{hostname=~\"$hostname\", status=~\"5..\"}[$__rate_interval])) by (view)",
+ "legendFormat": "{{view}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "reqps",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 9
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 5
+ },
+ {
+ "type": "row",
+ "title": "Calls to the other services",
+ "collapsed": false,
+ "gridPos": {
+ "w": 24,
+ "h": 1,
+ "x": 0,
+ "y": 17
+ },
+ "panels": [],
+ "id": 6
+ },
+ {
+ "type": "timeseries",
+ "title": "Outgoing calls p95",
+ "description": "yhub, and the converters. Made inside requests (duplicate, formatted-content, import) and from the Celery tasks.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(docs_outgoing_request_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, service, operation))",
+ "legendFormat": "{{service}} {{operation}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 18
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 7
+ },
+ {
+ "type": "timeseries",
+ "title": "Outgoing calls in flight",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(docs_outgoing_requests_inflight{hostname=~\"$hostname\"}) by (service, operation)",
+ "legendFormat": "{{service}} {{operation}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {
+ "stacking": {
+ "mode": "normal"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 18
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 8
+ },
+ {
+ "type": "timeseries",
+ "title": "Outgoing calls failed",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(docs_outgoing_request_duration_seconds_count{hostname=~\"$hostname\", status=~\"timeout|error|5..\"}[$__rate_interval])) by (service, operation, status)",
+ "legendFormat": "{{service}} {{operation}} {{status}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 26
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 9
+ },
+ {
+ "type": "timeseries",
+ "title": "Outgoing calls rate",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(docs_outgoing_request_duration_seconds_count{hostname=~\"$hostname\"}[$__rate_interval])) by (service, operation)",
+ "legendFormat": "{{service}} {{operation}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 26
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 10
+ },
+ {
+ "type": "row",
+ "title": "Database pool (DB_PSYCOPG_POOL_ENABLED)",
+ "collapsed": false,
+ "gridPos": {
+ "w": 24,
+ "h": 1,
+ "x": 0,
+ "y": 34
+ },
+ "panels": [],
+ "id": 11
+ },
+ {
+ "type": "timeseries",
+ "title": "Requests waiting for a connection",
+ "description": "The application waiting for connections, before Postgres shows anything. What was missing in the 2026-08-18 and 2026-09-07 outages.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(docs_db_pool_requests_waiting{hostname=~\"$hostname\"}) by (hostname)",
+ "legendFormat": "{{hostname}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {
+ "stacking": {
+ "mode": "normal"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 35
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 12
+ },
+ {
+ "type": "timeseries",
+ "title": "Queued requests and time spent waiting",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(docs_db_pool_requests_queued_total{hostname=~\"$hostname\"}[$__rate_interval]))",
+ "legendFormat": "queued/s",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(docs_db_pool_requests_wait_seconds_total{hostname=~\"$hostname\"}[$__rate_interval]))",
+ "legendFormat": "wait s/s",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 35
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 13
+ },
+ {
+ "type": "timeseries",
+ "title": "Pool size and idle",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(docs_db_pool_size{hostname=~\"$hostname\"})",
+ "legendFormat": "size",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(docs_db_pool_available{hostname=~\"$hostname\"})",
+ "legendFormat": "idle",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 43
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 14
+ },
+ {
+ "type": "timeseries",
+ "title": "Connections opened to Postgres",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(docs_db_pool_connections_total{hostname=~\"$hostname\"}[$__rate_interval]))",
+ "legendFormat": "opened/s",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(docs_db_pool_connections_errors_total{hostname=~\"$hostname\"}[$__rate_interval]))",
+ "legendFormat": "errors/s",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 43
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 15
+ },
+ {
+ "type": "row",
+ "title": "Queries and Celery",
+ "collapsed": false,
+ "gridPos": {
+ "w": 24,
+ "h": 1,
+ "x": 0,
+ "y": 51
+ },
+ "panels": [],
+ "id": 16
+ },
+ {
+ "type": "timeseries",
+ "title": "Query duration p95",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(django_db_query_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le))",
+ "legendFormat": "p95",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 52
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 17
+ },
+ {
+ "type": "timeseries",
+ "title": "Queries",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(django_db_execute_total{hostname=~\"$hostname\"}[$__rate_interval]))",
+ "legendFormat": "queries/s",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(django_db_errors_total{hostname=~\"$hostname\"}[$__rate_interval]))",
+ "legendFormat": "errors/s",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 52
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 18
+ },
+ {
+ "type": "timeseries",
+ "title": "Celery queue length",
+ "description": "One queue for the whole deployment, reported by every replica: max, not sum. The delete, restore and access cascades land on it.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(docs_celery_queue_length) by (queue)",
+ "legendFormat": "{{queue}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 60
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 19
+ },
+ {
+ "type": "timeseries",
+ "title": "Uvicorn workers alive",
+ "description": "Replicas answering scrapes. The memory and CPU of the pods come from the cluster (cAdvisor), not from the application: prometheus_client exports no process metrics in multiprocess mode.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "count(count(django_http_requests_total_by_view_transport_method_total{hostname=~\"$hostname\"}) by (hostname))",
+ "legendFormat": "replicas",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 60
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 20
+ },
+ {
+ "type": "row",
+ "title": "k6 \u2014 what the clients see",
+ "collapsed": false,
+ "gridPos": {
+ "w": 24,
+ "h": 1,
+ "x": 0,
+ "y": 68
+ },
+ "panels": [],
+ "id": 21
+ },
+ {
+ "type": "timeseries",
+ "title": "k6 requests by endpoint",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(k6_http_reqs_total[$__rate_interval])) by (name)",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "reqps",
+ "custom": {
+ "stacking": {
+ "mode": "normal"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 69
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 22
+ },
+ {
+ "type": "timeseries",
+ "title": "k6 latency p99 by endpoint",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(k6_http_req_duration_p99{name!=\"\"}) by (name)",
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ms",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 69
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 23
+ },
+ {
+ "type": "timeseries",
+ "title": "k6 failures",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(k6_http_req_failed_rate)",
+ "legendFormat": "requests failed",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "1 - max(k6_checks_rate)",
+ "legendFormat": "checks failed",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percentunit",
+ "custom": {},
+ "min": 0,
+ "max": 1
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 77
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 24
+ },
+ {
+ "type": "timeseries",
+ "title": "k6 virtual users and dropped iterations",
+ "description": "Dropped iterations: the arrival rate asked for could not be served with the VUs allocated, the API was too slow.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(k6_vus)",
+ "legendFormat": "VUs",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(k6_dropped_iterations_total[$__rate_interval]))",
+ "legendFormat": "dropped/s",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 77
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 25
+ }
+ ]
+}
diff --git a/src/loadtest/dashboards/collaboration.json b/src/loadtest/dashboards/collaboration.json
new file mode 100644
index 000000000..e676443fe
--- /dev/null
+++ b/src/loadtest/dashboards/collaboration.json
@@ -0,0 +1,1234 @@
+{
+ "uid": "docs-loadtest-collaboration",
+ "title": "Docs load test \u2014 collaboration server",
+ "description": "The collaboration server (yhub) under load, and what the swarm sees from the outside.",
+ "tags": [
+ "docs",
+ "loadtest"
+ ],
+ "schemaVersion": 39,
+ "version": 1,
+ "editable": true,
+ "graphTooltip": 1,
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "refresh": "15s",
+ "timezone": "browser",
+ "templating": {
+ "list": [
+ {
+ "name": "datasource",
+ "label": "Prometheus",
+ "type": "datasource",
+ "query": "prometheus",
+ "current": {},
+ "hide": 0,
+ "refresh": 1
+ },
+ {
+ "name": "hostname",
+ "label": "yhub replica",
+ "type": "query",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "query": {
+ "query": "label_values(yhub_ws_connections, hostname)",
+ "refId": "hostname"
+ },
+ "definition": "label_values(yhub_ws_connections, hostname)",
+ "refresh": 2,
+ "includeAll": true,
+ "multi": true,
+ "allValue": ".*",
+ "current": {
+ "text": "All",
+ "value": "$__all"
+ },
+ "sort": 1
+ }
+ ]
+ },
+ "annotations": {
+ "list": []
+ },
+ "panels": [
+ {
+ "type": "row",
+ "title": "Connections",
+ "collapsed": false,
+ "gridPos": {
+ "w": 24,
+ "h": 1,
+ "x": 0,
+ "y": 0
+ },
+ "panels": [],
+ "id": 1
+ },
+ {
+ "type": "timeseries",
+ "title": "Websocket connections per replica",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(yhub_ws_connections{hostname=~\"$hostname\"}) by (hostname)",
+ "legendFormat": "{{hostname}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {
+ "stacking": {
+ "mode": "normal"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 1
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 2
+ },
+ {
+ "type": "timeseries",
+ "title": "Open documents per replica",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(yhub_rooms{hostname=~\"$hostname\"}) by (hostname)",
+ "legendFormat": "{{hostname}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 1
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 3
+ },
+ {
+ "type": "timeseries",
+ "title": "Event loop lag p99 per replica",
+ "description": "One thread serves every socket of a replica: this is the first thing to move when it saturates.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(nodejs_eventloop_lag_p99_seconds{hostname=~\"$hostname\", job=~\"yhub.*\"}) by (hostname)",
+ "legendFormat": "{{hostname}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 9
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 4
+ },
+ {
+ "type": "timeseries",
+ "title": "Memory per replica",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(process_resident_memory_bytes{hostname=~\"$hostname\", job=~\"yhub.*\"}) by (hostname)",
+ "legendFormat": "{{hostname}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "bytes",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 9
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 5
+ },
+ {
+ "type": "row",
+ "title": "Admitting callers",
+ "collapsed": false,
+ "gridPos": {
+ "w": 24,
+ "h": 1,
+ "x": 0,
+ "y": 17
+ },
+ "panels": [],
+ "id": 6
+ },
+ {
+ "type": "timeseries",
+ "title": "Auth duration p95",
+ "description": "What a websocket upgrade, a recheck, a REST call or a fallback poll costs: the backend calls it makes, and the legacy seed under soft migration.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(yhub_auth_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, phase, endpoint))",
+ "legendFormat": "{{phase}} {{endpoint}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 18
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 7
+ },
+ {
+ "type": "timeseries",
+ "title": "Auth results",
+ "description": "unavailable: a 503 sent to the client, the backend did not answer.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(yhub_auth_duration_seconds_count{hostname=~\"$hostname\"}[$__rate_interval])) by (phase, result)",
+ "legendFormat": "{{phase}} {{result}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {
+ "stacking": {
+ "mode": "normal"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 18
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 8
+ },
+ {
+ "type": "timeseries",
+ "title": "Backend calls p95 by route",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(yhub_backend_request_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, route))",
+ "legendFormat": "{{route}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 26
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 9
+ },
+ {
+ "type": "timeseries",
+ "title": "Backend calls in flight",
+ "description": "What piles up when the backend slows down: backendFetch has no timeout.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(yhub_backend_requests_inflight{hostname=~\"$hostname\"}) by (route)",
+ "legendFormat": "{{route}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {
+ "stacking": {
+ "mode": "normal"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 26
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 10
+ },
+ {
+ "type": "timeseries",
+ "title": "Backend calls not answered 2xx",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(yhub_backend_request_duration_seconds_count{hostname=~\"$hostname\", status!~\"2..\"}[$__rate_interval])) by (route, status)",
+ "legendFormat": "{{route}} {{status}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 34
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 11
+ },
+ {
+ "type": "row",
+ "title": "Persistence (worker)",
+ "collapsed": false,
+ "gridPos": {
+ "w": 24,
+ "h": 1,
+ "x": 0,
+ "y": 42
+ },
+ "panels": [],
+ "id": 12
+ },
+ {
+ "type": "timeseries",
+ "title": "Compaction backlog",
+ "description": "One queue for the whole deployment, reported by every worker: max, not sum.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(yhub_worker_pending_tasks)",
+ "legendFormat": "pending",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 43
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 13
+ },
+ {
+ "type": "timeseries",
+ "title": "Compactions",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(yhub_worker_task_duration_seconds_count{hostname=~\"$hostname\"}[$__rate_interval])) by (result)",
+ "legendFormat": "{{result}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(yhub_doc_updates_total{hostname=~\"$hostname\"}[$__rate_interval]))",
+ "legendFormat": "with new content",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 43
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 14
+ },
+ {
+ "type": "timeseries",
+ "title": "Compaction duration p95",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(yhub_worker_task_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le))",
+ "legendFormat": "p95",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 51
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 15
+ },
+ {
+ "type": "timeseries",
+ "title": "Compactions in flight",
+ "description": "Against YHUB_TASK_CONCURRENCY per worker.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(yhub_worker_tasks_inflight{hostname=~\"$hostname\"}) by (hostname)",
+ "legendFormat": "{{hostname}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {
+ "stacking": {
+ "mode": "normal"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 51
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 16
+ },
+ {
+ "type": "row",
+ "title": "Soft migration",
+ "collapsed": false,
+ "gridPos": {
+ "w": 24,
+ "h": 1,
+ "x": 0,
+ "y": 59
+ },
+ "panels": [],
+ "id": 17
+ },
+ {
+ "type": "timeseries",
+ "title": "Seeds",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(yhub_seed_duration_seconds_count{hostname=~\"$hostname\"}[$__rate_interval])) by (result)",
+ "legendFormat": "{{result}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(yhub_seed_rejected_total{hostname=~\"$hostname\"}[$__rate_interval]))",
+ "legendFormat": "rejected",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 60
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 18
+ },
+ {
+ "type": "timeseries",
+ "title": "Seed duration p95",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(yhub_seed_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, result))",
+ "legendFormat": "{{result}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 60
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 19
+ },
+ {
+ "type": "timeseries",
+ "title": "Seeds in flight",
+ "description": "Refused with a 503 past 20 per replica.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(yhub_seeds_inflight{hostname=~\"$hostname\"}) by (hostname)",
+ "legendFormat": "{{hostname}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 68
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 20
+ },
+ {
+ "type": "row",
+ "title": "Swarm \u2014 what the clients see",
+ "collapsed": false,
+ "gridPos": {
+ "w": 24,
+ "h": 1,
+ "x": 0,
+ "y": 76
+ },
+ "panels": [],
+ "id": 21
+ },
+ {
+ "type": "timeseries",
+ "title": "Swarm clients",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(swarm_clients) by (state)",
+ "legendFormat": "{{state}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "custom": {
+ "stacking": {
+ "mode": "normal"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 77
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 22
+ },
+ {
+ "type": "timeseries",
+ "title": "Connect and sync p95",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(swarm_connect_duration_seconds_bucket{}[$__rate_interval])) by (le))",
+ "legendFormat": "connect",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(swarm_sync_duration_seconds_bucket{}[$__rate_interval])) by (le))",
+ "legendFormat": "sync",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 77
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 23
+ },
+ {
+ "type": "timeseries",
+ "title": "Propagation",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.5, sum(rate(swarm_propagation_latency_seconds_bucket{}[$__rate_interval])) by (le))",
+ "legendFormat": "p50",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.95, sum(rate(swarm_propagation_latency_seconds_bucket{}[$__rate_interval])) by (le))",
+ "legendFormat": "p95",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "histogram_quantile(0.99, sum(rate(swarm_propagation_latency_seconds_bucket{}[$__rate_interval])) by (le))",
+ "legendFormat": "p99",
+ "refId": "C"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 85
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 24
+ },
+ {
+ "type": "timeseries",
+ "title": "Reconnects and refused upgrades",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(swarm_reconnects_total[$__rate_interval]))",
+ "legendFormat": "reconnects",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(swarm_upgrade_failures_total[$__rate_interval])) by (status)",
+ "legendFormat": "refused {{status}}",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 85
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 25
+ },
+ {
+ "type": "timeseries",
+ "title": "Socket closes by code",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(swarm_ws_closes_total[$__rate_interval])) by (code)",
+ "legendFormat": "{{code}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 93
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 26
+ },
+ {
+ "type": "timeseries",
+ "title": "Websocket traffic",
+ "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum(rate(swarm_ws_bytes_total[$__rate_interval])) by (direction)",
+ "legendFormat": "{{direction}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "Bps",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 12,
+ "y": 93
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 27
+ },
+ {
+ "type": "timeseries",
+ "title": "Swarm event loop lag p99",
+ "description": "Past a point the load generator saturates before the server: its numbers stop meaning anything.",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(nodejs_eventloop_lag_p99_seconds{job=~\"swarm.*\"})",
+ "legendFormat": "lag",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "w": 12,
+ "h": 8,
+ "x": 0,
+ "y": 101
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "id": 28
+ }
+ ]
+}
diff --git a/src/loadtest/dashboards/django.json b/src/loadtest/dashboards/django.json
new file mode 100644
index 000000000..65360c5f7
--- /dev/null
+++ b/src/loadtest/dashboards/django.json
@@ -0,0 +1,1588 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Django metrics dashboard using django-prometheus metrics exporter",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "gnetId": 17658,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "liveNow": true,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 0,
+ "y": 0
+ },
+ "id": 13,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {
+ "valueSize": 40
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "sum(irate(django_http_requests_total_by_transport_total{job=~\"^$application$\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Requests",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 2,
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 3,
+ "x": 6,
+ "y": 0
+ },
+ "id": 15,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {
+ "valueSize": 20
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "sum(irate(django_http_responses_total_by_status_total{status=~\"2.+\",job=~\"^$application$\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "2XX Responses",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 2,
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 3,
+ "x": 9,
+ "y": 0
+ },
+ "id": 16,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {
+ "valueSize": 20
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "sum(irate(django_http_responses_total_by_status_total{status=~\"4.+\",job=~\"^$application$\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "4XX Responses",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "min": 0,
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 10
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 12,
+ "y": 0
+ },
+ "id": 37,
+ "options": {
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(max_over_time(django_db_errors_total{job=~\"^$application$\"}[$__range]))",
+ "format": "time_series",
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "DB Query Errors",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "min": 0,
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 5
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 15,
+ "y": 0
+ },
+ "id": 38,
+ "options": {
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "sum(max_over_time(django_db_new_connection_errors_total{job=~\"^$application$\"}[$__range]))",
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "DB Connection Errors",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 0,
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 0.4
+ },
+ {
+ "color": "green",
+ "value": 0.7
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 18,
+ "y": 0
+ },
+ "id": 31,
+ "options": {
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "sum(django_cache_get_hits_total{job=~\"^$application$\"}) by (backend) / sum(django_cache_get_total{job=~\"^$application$\"}) by (backend)",
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Cache Hit Ratio",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 2,
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 3,
+ "x": 6,
+ "y": 2
+ },
+ "id": 23,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {
+ "valueSize": 20
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "sum(irate(django_http_responses_total_by_status_total{status=~\"3.+\",job=~\"^$application$\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "3XX Responses",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 2,
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 3,
+ "x": 9,
+ "y": 2
+ },
+ "id": 17,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {
+ "valueSize": 20
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "sum(irate(django_http_responses_total_by_status_total{status=~\"5.+\",job=~\"^$application$\"}[$__rate_interval]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "5XX Responses",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 4
+ },
+ "id": 20,
+ "options": {
+ "calculate": false,
+ "cellGap": 1,
+ "cellValues": {
+ "unit": "reqps"
+ },
+ "color": {
+ "exponent": 0.5,
+ "fill": "dark-orange",
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Spectral",
+ "steps": 64
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-09
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto",
+ "value": "Responses"
+ },
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le)",
+ "format": "heatmap",
+ "interval": "",
+ "legendFormat": "{{le}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Number of Requests by Processing Time",
+ "type": "heatmap"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 4
+ },
+ "id": 22,
+ "options": {
+ "calculate": false,
+ "cellGap": 1,
+ "cellValues": {
+ "unit": "reqps"
+ },
+ "color": {
+ "exponent": 0.5,
+ "fill": "dark-orange",
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Spectral",
+ "steps": 64
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-09
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto",
+ "value": "Responses"
+ },
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "bytes"
+ }
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(rate(django_http_responses_body_total_bytes_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le)",
+ "format": "heatmap",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "{{le}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Number of Responses by Size",
+ "type": "heatmap"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "linearThreshold": 1,
+ "log": 2,
+ "type": "log"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "dashed"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 12
+ },
+ "id": 4,
+ "interval": "30s",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.50, sum(irate(django_http_requests_latency_seconds_by_view_method_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "50 quantile",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum(irate(django_http_requests_latency_seconds_by_view_method_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "95 quantile",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum(irate(django_http_requests_latency_seconds_by_view_method_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "99 quantile",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Request Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 2,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 12
+ },
+ "id": 11,
+ "interval": "30s",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "sum(irate(django_http_responses_total_by_status_total{job=~\"^$application$\"}[$__rate_interval])) by(status)",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{status}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Response Status",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 0,
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 0.3
+ },
+ {
+ "color": "red",
+ "value": 0.6
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 20
+ },
+ "id": 25,
+ "options": {
+ "displayMode": "gradient",
+ "minVizHeight": 10,
+ "minVizWidth": 0,
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "sum"
+ ],
+ "fields": "",
+ "limit": 20,
+ "values": true
+ },
+ "showUnfilled": true,
+ "text": {}
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "topk(20, (1 - (sum(max_over_time(django_http_requests_latency_seconds_by_view_method_bucket{job=~\"^$application$\",le=\"$threshold\"}[$__range]) / ignoring(le) max_over_time(django_http_requests_latency_seconds_by_view_method_count{job=~\"^$application$\"}[$__range])) by (method, view) / count(present_over_time(django_http_requests_latency_seconds_by_view_method_count{job=~\"^$application$\"}[$__range])) by (method, view))) > 0.0099)",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "legendFormat": "{{method}} {{view}}",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "Top 20 Views by Response Time (> $threshold)",
+ "type": "bargauge"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 2,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 20
+ },
+ "id": 2,
+ "interval": "30s",
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "topk(10, sum(irate(django_http_requests_total_by_view_transport_method_total{job=~\"^$application$\",view!~\"prometheus-django-metrics|healthcheck|.*shop_product_import_1c_status\"}[$__rate_interval])) by(method, view) > 0)",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{method}} /{{view}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Top Requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 28
+ },
+ "id": 35,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "sum(irate(django_db_execute_total{job=~\"^$application$\"}[$__rate_interval])) by (vendor)",
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Database Total Queries",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 28
+ },
+ "id": 33,
+ "options": {
+ "calculate": false,
+ "cellGap": 1,
+ "cellValues": {
+ "unit": "ops"
+ },
+ "color": {
+ "exponent": 0.5,
+ "fill": "dark-orange",
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Spectral",
+ "steps": 64
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-09
+ },
+ "legend": {
+ "show": true
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(django_db_query_duration_seconds_bucket{job=~\"^$application$\"}[$__rate_interval])) by (le)",
+ "format": "heatmap",
+ "legendFormat": "{{le}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Database Query Duration",
+ "type": "heatmap"
+ }
+ ],
+ "refresh": "30s",
+ "schemaVersion": 37,
+ "style": "dark",
+ "tags": [
+ "Web",
+ "Django",
+ "Database",
+ "Cache"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {},
+ "hide": 0,
+ "includeAll": false,
+ "multi": false,
+ "name": "datasource",
+ "options": [],
+ "query": "prometheus",
+ "queryValue": "",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "definition": "label_values(python_info,app)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "application",
+ "multi": false,
+ "name": "application",
+ "options": [],
+ "query": {
+ "query": "label_values(python_info,app)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": true,
+ "text": "1s",
+ "value": "1.0"
+ },
+ "description": "Request duration threshold",
+ "hide": 0,
+ "includeAll": false,
+ "label": "threshold",
+ "multi": false,
+ "name": "threshold",
+ "options": [
+ {
+ "selected": false,
+ "text": "100ms",
+ "value": "0.1"
+ },
+ {
+ "selected": false,
+ "text": "250ms",
+ "value": "0.25"
+ },
+ {
+ "selected": false,
+ "text": "500ms",
+ "value": "0.5"
+ },
+ {
+ "selected": false,
+ "text": "750ms",
+ "value": "0.75"
+ },
+ {
+ "selected": true,
+ "text": "1s",
+ "value": "1.0"
+ },
+ {
+ "selected": false,
+ "text": "2.5s",
+ "value": "2.5"
+ },
+ {
+ "selected": false,
+ "text": "5s",
+ "value": "5.0"
+ },
+ {
+ "selected": false,
+ "text": "7.5s",
+ "value": "7.5"
+ },
+ {
+ "selected": false,
+ "text": "10s",
+ "value": "10.0"
+ },
+ {
+ "selected": false,
+ "text": "25s",
+ "value": "25.0"
+ }
+ ],
+ "query": "100ms : 0.1, 250ms : 0.25, 500ms : 0.5, 750ms : 0.75, 1s : 1.0, 2.5s : 2.5, 5s : 5.0, 7.5s : 7.5, 10s : 10.0, 25s : 25.0",
+ "queryValue": "",
+ "skipUrlSync": false,
+ "type": "custom"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Django",
+ "uid": "O6v4rMpizda",
+ "version": 139,
+ "weekStart": ""
+}
diff --git a/src/loadtest/dashboards/generate.py b/src/loadtest/dashboards/generate.py
new file mode 100644
index 000000000..28c824a9c
--- /dev/null
+++ b/src/loadtest/dashboards/generate.py
@@ -0,0 +1,234 @@
+"""Generate the docs-loadtest-* dashboards: python3 generate.py