{
  "uid": "docs-loadtest-backend",
  "title": "Docs load test \u2014 backend",
  "description": "The Django backend under load: what it spends on the other services, its database pool, its queue, and what k6 sees from the outside. The generic Django board (17658) has the per-view detail.",
  "tags": [
    "docs",
    "loadtest"
  ],
  "schemaVersion": 39,
  "version": 1,
  "editable": true,
  "graphTooltip": 1,
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "refresh": "15s",
  "timezone": "browser",
  "templating": {
    "list": [
      {
        "name": "datasource",
        "label": "Prometheus",
        "type": "datasource",
        "query": "prometheus",
        "current": {},
        "hide": 0,
        "refresh": 1
      },
      {
        "name": "hostname",
        "label": "backend replica",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "query": {
          "query": "label_values(django_http_requests_total_by_view_transport_method_total, hostname)",
          "refId": "hostname"
        },
        "definition": "label_values(django_http_requests_total_by_view_transport_method_total, hostname)",
        "refresh": 2,
        "includeAll": true,
        "multi": true,
        "allValue": ".*",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "sort": 1
      }
    ]
  },
  "annotations": {
    "list": []
  },
  "panels": [
    {
      "type": "row",
      "title": "Requests",
      "collapsed": false,
      "gridPos": {
        "w": 24,
        "h": 1,
        "x": 0,
        "y": 0
      },
      "panels": [],
      "id": 1
    },
    {
      "type": "timeseries",
      "title": "Requests by view",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(django_http_requests_total_by_view_transport_method_total{hostname=~\"$hostname\"}[$__rate_interval])) by (view)",
          "legendFormat": "{{view}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": {
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 0,
        "y": 1
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 2
    },
    {
      "type": "timeseries",
      "title": "Latency p95 by view",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.95, sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, view))",
          "legendFormat": "{{view}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 12,
        "y": 1
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 3
    },
    {
      "type": "timeseries",
      "title": "Responses by status",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(django_http_responses_total_by_status_view_method_total{hostname=~\"$hostname\"}[$__rate_interval])) by (status)",
          "legendFormat": "{{status}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": {
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 0,
        "y": 9
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 4
    },
    {
      "type": "timeseries",
      "title": "Errors (5xx) by view",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(django_http_responses_total_by_status_view_method_total{hostname=~\"$hostname\", status=~\"5..\"}[$__rate_interval])) by (view)",
          "legendFormat": "{{view}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 12,
        "y": 9
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 5
    },
    {
      "type": "row",
      "title": "Calls to the other services",
      "collapsed": false,
      "gridPos": {
        "w": 24,
        "h": 1,
        "x": 0,
        "y": 17
      },
      "panels": [],
      "id": 6
    },
    {
      "type": "timeseries",
      "title": "Outgoing calls p95",
      "description": "yhub, and the converters. Made inside requests (duplicate, formatted-content, import) and from the Celery tasks.",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.95, sum(rate(docs_outgoing_request_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le, service, operation))",
          "legendFormat": "{{service}} {{operation}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 0,
        "y": 18
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 7
    },
    {
      "type": "timeseries",
      "title": "Outgoing calls in flight",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(docs_outgoing_requests_inflight{hostname=~\"$hostname\"}) by (service, operation)",
          "legendFormat": "{{service}} {{operation}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 12,
        "y": 18
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 8
    },
    {
      "type": "timeseries",
      "title": "Outgoing calls failed",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(docs_outgoing_request_duration_seconds_count{hostname=~\"$hostname\", status=~\"timeout|error|5..\"}[$__rate_interval])) by (service, operation, status)",
          "legendFormat": "{{service}} {{operation}} {{status}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ops",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 0,
        "y": 26
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 9
    },
    {
      "type": "timeseries",
      "title": "Outgoing calls rate",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(docs_outgoing_request_duration_seconds_count{hostname=~\"$hostname\"}[$__rate_interval])) by (service, operation)",
          "legendFormat": "{{service}} {{operation}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ops",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 12,
        "y": 26
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 10
    },
    {
      "type": "row",
      "title": "Database pool (DB_PSYCOPG_POOL_ENABLED)",
      "collapsed": false,
      "gridPos": {
        "w": 24,
        "h": 1,
        "x": 0,
        "y": 34
      },
      "panels": [],
      "id": 11
    },
    {
      "type": "timeseries",
      "title": "Requests waiting for a connection",
      "description": "The application waiting for connections, before Postgres shows anything. What was missing in the 2026-08-18 and 2026-09-07 outages.",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(docs_db_pool_requests_waiting{hostname=~\"$hostname\"}) by (hostname)",
          "legendFormat": "{{hostname}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 0,
        "y": 35
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 12
    },
    {
      "type": "timeseries",
      "title": "Queued requests and time spent waiting",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(docs_db_pool_requests_queued_total{hostname=~\"$hostname\"}[$__rate_interval]))",
          "legendFormat": "queued/s",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(docs_db_pool_requests_wait_seconds_total{hostname=~\"$hostname\"}[$__rate_interval]))",
          "legendFormat": "wait s/s",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 12,
        "y": 35
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 13
    },
    {
      "type": "timeseries",
      "title": "Pool size and idle",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(docs_db_pool_size{hostname=~\"$hostname\"})",
          "legendFormat": "size",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(docs_db_pool_available{hostname=~\"$hostname\"})",
          "legendFormat": "idle",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 0,
        "y": 43
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 14
    },
    {
      "type": "timeseries",
      "title": "Connections opened to Postgres",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(docs_db_pool_connections_total{hostname=~\"$hostname\"}[$__rate_interval]))",
          "legendFormat": "opened/s",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(docs_db_pool_connections_errors_total{hostname=~\"$hostname\"}[$__rate_interval]))",
          "legendFormat": "errors/s",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ops",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 12,
        "y": 43
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 15
    },
    {
      "type": "row",
      "title": "Queries and Celery",
      "collapsed": false,
      "gridPos": {
        "w": 24,
        "h": 1,
        "x": 0,
        "y": 51
      },
      "panels": [],
      "id": 16
    },
    {
      "type": "timeseries",
      "title": "Query duration p95",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.95, sum(rate(django_db_query_duration_seconds_bucket{hostname=~\"$hostname\"}[$__rate_interval])) by (le))",
          "legendFormat": "p95",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 0,
        "y": 52
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 17
    },
    {
      "type": "timeseries",
      "title": "Queries",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(django_db_execute_total{hostname=~\"$hostname\"}[$__rate_interval]))",
          "legendFormat": "queries/s",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(django_db_errors_total{hostname=~\"$hostname\"}[$__rate_interval]))",
          "legendFormat": "errors/s",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ops",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 12,
        "y": 52
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 18
    },
    {
      "type": "timeseries",
      "title": "Celery queue length",
      "description": "One queue for the whole deployment, reported by every replica: max, not sum. The delete, restore and access cascades land on it.",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "max(docs_celery_queue_length) by (queue)",
          "legendFormat": "{{queue}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 0,
        "y": 60
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 19
    },
    {
      "type": "timeseries",
      "title": "Uvicorn workers alive",
      "description": "Replicas answering scrapes. The memory and CPU of the pods come from the cluster (cAdvisor), not from the application: prometheus_client exports no process metrics in multiprocess mode.",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "count(count(django_http_requests_total_by_view_transport_method_total{hostname=~\"$hostname\"}) by (hostname))",
          "legendFormat": "replicas",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 12,
        "y": 60
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 20
    },
    {
      "type": "row",
      "title": "k6 \u2014 what the clients see",
      "collapsed": false,
      "gridPos": {
        "w": 24,
        "h": 1,
        "x": 0,
        "y": 68
      },
      "panels": [],
      "id": 21
    },
    {
      "type": "timeseries",
      "title": "k6 requests by endpoint",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(k6_http_reqs_total[$__rate_interval])) by (name)",
          "legendFormat": "{{name}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": {
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 0,
        "y": 69
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 22
    },
    {
      "type": "timeseries",
      "title": "k6 latency p99 by endpoint",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "max(k6_http_req_duration_p99{name!=\"\"}) by (name)",
          "legendFormat": "{{name}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ms",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 12,
        "y": 69
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 23
    },
    {
      "type": "timeseries",
      "title": "k6 failures",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "max(k6_http_req_failed_rate)",
          "legendFormat": "requests failed",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "1 - max(k6_checks_rate)",
          "legendFormat": "checks failed",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "custom": {},
          "min": 0,
          "max": 1
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 0,
        "y": 77
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 24
    },
    {
      "type": "timeseries",
      "title": "k6 virtual users and dropped iterations",
      "description": "Dropped iterations: the arrival rate asked for could not be served with the VUs allocated, the API was too slow.",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "max(k6_vus)",
          "legendFormat": "VUs",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(k6_dropped_iterations_total[$__rate_interval]))",
          "legendFormat": "dropped/s",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "w": 12,
        "h": 8,
        "x": 12,
        "y": 77
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "id": 25
    }
  ]
}