mirror of
https://github.com/suitenumerique/docs.git
synced 2026-09-24 02:25:08 +02:00
📝(documentation) finalize and polish the documentation
Also the AGENTS.md is now track to have all the same one and track any modification made on it.
This commit is contained in:
Manuel Raynaud
+1
-1
@@ -86,7 +86,7 @@ db.sqlite3
|
||||
|
||||
# AI
|
||||
CLAUDE.md
|
||||
AGENTS.md
|
||||
!AGENTS.md
|
||||
.claude/
|
||||
.cursor/
|
||||
.cursorrules
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
# Agent Guidelines for Docs
|
||||
|
||||
This file contains guidelines for AI agents coding in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
**La Suite Docs** is a collaborative text editor built by DINUM (French government) and ZenDiS (German government). It features real-time editing, offline support, AI actions, and multi-format export (PDF/DOCX/ODT).
|
||||
|
||||
Repository: https://github.com/suitenumerique/docs
|
||||
|
||||
## Monorepo Structure
|
||||
|
||||
- `src/backend/` - Django REST API (Python 3.14+)
|
||||
- `src/frontend/apps/impress/` - Main Next.js 15 application (TypeScript)
|
||||
- `src/frontend/apps/e2e/` - Playwright E2E tests
|
||||
- `src/frontend/packages/i18n/` - Shared i18n utilities
|
||||
- `src/frontend/packages/eslint-plugin-docs/` - Custom ESLint plugin
|
||||
- `src/frontend/servers/y-provider/` - Conversion service only (Express, `POST /api/convert/`). It no longer serves any websocket
|
||||
- `src/yhub-server/` - Collaboration server: a thin TypeScript wrapper around `@y/hub` (websockets, REST routes, worker). It holds the content of the documents
|
||||
- `src/mail/` - Email templates (MJML)
|
||||
- `src/helm/` - Kubernetes/Helm deployment
|
||||
|
||||
## Development Commands
|
||||
|
||||
### Setup and Services
|
||||
|
||||
```bash
|
||||
make bootstrap # Full dev setup (build + migrate + demo + run)
|
||||
make build # Build all Docker containers
|
||||
make run # Start all services
|
||||
make stop # Stop services
|
||||
make build-yhub # Build the collaboration server (yhub) image
|
||||
make migrate-yhub # Create/upgrade the yhub database schema (`yarn init-db`), safe to re-run
|
||||
make status # Check running services
|
||||
```
|
||||
|
||||
### Backend (Python/Django)
|
||||
|
||||
Tests run inside Docker containers. You must first build the backend image and ensure the `lasuite-network` Docker network exists before running tests.
|
||||
|
||||
```bash
|
||||
make build-backend # Build the backend Docker image (required before first test run)
|
||||
docker network create lasuite-network # Create the external network (required once)
|
||||
bin/pytest -n auto # Run all backend tests in parallel
|
||||
bin/pytest -n auto path/to/test # Run specific test file/directory
|
||||
bin/pytest -n auto path/to/test.py::TestClass::test_method # Run single test via docker compose
|
||||
make lint # ruff format + ruff check + pylint
|
||||
make lint-ruff-format # Format only
|
||||
make lint-ruff-check # Lint only
|
||||
make migrate # Run database migrations
|
||||
make makemigrations # Create new migrations
|
||||
make resetdb # Flush DB + create superuser (admin/admin)
|
||||
```
|
||||
|
||||
### Frontend (TypeScript/Next.js)
|
||||
|
||||
```bash
|
||||
# From src/frontend/apps/impress/:
|
||||
yarn dev # Development server (port 3000)
|
||||
yarn build # Production build (includes prettier + stylelint checks)
|
||||
yarn lint # TypeScript check + ESLint
|
||||
yarn test # Run Vitest tests
|
||||
yarn prettier # Format code
|
||||
yarn stylelint # Lint CSS
|
||||
|
||||
# From project root:
|
||||
make frontend-lint # Lint all frontend workspaces
|
||||
make frontend-test # Run frontend tests
|
||||
```
|
||||
|
||||
### Collaboration server (yhub, TypeScript)
|
||||
|
||||
```bash
|
||||
# From src/yhub-server/ (standalone package: its own package.json and yarn.lock,
|
||||
# not a workspace of src/frontend):
|
||||
yarn test # Vitest unit tests (@y/hub is mocked, no store needed)
|
||||
yarn typecheck # tsc --noEmit
|
||||
yarn lint # ESLint
|
||||
yarn build # Compile to dist/
|
||||
yarn init-db # Create/upgrade the yhub postgres schema (what `make migrate-yhub` runs)
|
||||
```
|
||||
|
||||
The container starts with `node --import ./dist/sentry.js dist/server.js`: keep the
|
||||
`--import` when overriding the command. `src/yhub-server/README.md` is the reference
|
||||
for everything this server does (permissions, roles, migration, storage, metrics).
|
||||
|
||||
### Mails
|
||||
|
||||
```bash
|
||||
make mails-install # Install dependencies
|
||||
make mails-build # Convert MJML to HTML + plaintext
|
||||
```
|
||||
|
||||
### Helm/Kubernetes Deployment
|
||||
|
||||
```bash
|
||||
make build-k8s-cluster # Create local Kind cluster
|
||||
make start-tilt # Start Tilt for hot-reload
|
||||
# From src/helm/: helmfile -n impress -e dev apply/destroy/diff
|
||||
```
|
||||
|
||||
### Dev Service URLs
|
||||
|
||||
- Frontend: http://localhost:3000
|
||||
- Backend API/Admin: http://localhost:8071
|
||||
- Keycloak (auth): http://localhost:8083
|
||||
- MinIO (S3): http://localhost:9000
|
||||
- Collaboration server (yhub): http://localhost:3002 (websocket on `/collaboration/ws/v1/docs/{docid}`)
|
||||
- yhub PostgreSQL: localhost:5433 (its own database, apart from the backend's on 15432)
|
||||
- Mailcatcher: http://localhost:1081
|
||||
|
||||
## Architecture
|
||||
|
||||
### Backend
|
||||
|
||||
- **Framework**: Django + DRF, configured via `django-configurations` (`src/backend/impress/settings.py`)
|
||||
- **Main app**: `src/backend/core/` (models, API viewsets, services, authentication)
|
||||
- **API**: REST on `/api/v1.0/` with nested routes (e.g., `/documents/{id}/accesses/`)
|
||||
- **Auth**: OIDC via `mozilla-django-oidc` (Keycloak in dev)
|
||||
- **Background tasks**: Celery + Redis
|
||||
- **Database**: PostgreSQL 16 with `django-treebeard` for document hierarchy
|
||||
- **Storage**: S3-compatible (MinIO in dev) for attachments and media. **The content of a document is not stored by the backend anymore**: it lives in the collaboration server (yhub). `Document.content`, `save_content` and the S3 object versions are gone; `file_key` only survives as a pointer for the migration of legacy documents
|
||||
- **Collaboration server client**: `core/services/yhub_services.py` (`YHubService`: `get_ydoc`, `create_ydoc`, `delete_ydoc`, `restore_ydoc`, `reset_ydoc`, `migrate`, `reset_connections`). It is called synchronously by document creation from a file, `duplicate`, `formatted-content`, and from Celery tasks for delete/restore/access changes. Mock it in tests (`CELERY_TASK_ALWAYS_EAGER=True` in the `Test` configuration, so tasks run inline)
|
||||
- **Service-to-service auth**: short-lived RS256 JWTs with an `aud` claim, never a shared bearer token. The backend signs with `JWT_PRIVATE_KEY` (`core/services/jwt_services.py`) and publishes its keys on `/api/v1.0/jwks`; yhub signs with `YHUB_JWT_PRIVATE_KEY` and publishes `/collaboration/jwks/v1`. Dev keys are generated in `data/jwt/` by `make bootstrap`
|
||||
- **Removed endpoints**: `documents/{id}/content/` (GET and PATCH), `documents/{id}/versions/`, `documents/{id}/can-edit/`. New ones: `documents/{id}/content-updated/` (called by yhub) and `documents/{id}/accesses/me/`. `documents/{id}/formatted-content/` now reads the content from yhub
|
||||
- **Legacy documents**: `python manage.py migrate_documents` hands the S3 content of existing documents to yhub (resumable, idempotent); `SOFT_MIGRATION=true` on yhub seeds a document on first open. See `UPGRADE.md`
|
||||
- **Search**: Full-text indexing via `manage.py index`
|
||||
- **Load-test tooling**: `src/backend/loadtest/` is a separate Django application that mints sessions for existing users (`create_load_test_sessions`, `revoke_load_test_sessions`). It is only installed by the `LoadTest` configuration (`DJANGO_CONFIGURATION=LoadTest`, a subclass of `Production`); `LOAD_TEST_TOOLS_ENABLED` is `False` everywhere else, pinned in `Production`, and never read from the environment. Anything added for load tests goes there, not in `core`, and must stay off in `Production`
|
||||
|
||||
### Frontend
|
||||
|
||||
- **Editor**: BlockNote.js 0.46.x (Tiptap-based block editor)
|
||||
- **Real-time**: Yjs with the plain `y-websocket` `WebsocketProvider` (`features/docs/doc-management/stores/useProviderStore.tsx`), and an HTTP polling fallback (`@y/yhub-http-fallback`) used while the websocket cannot be opened. Hocuspocus is gone
|
||||
- **Offline**: documents are kept locally (`y-indexeddb`) and a service worker (`features/service-worker/`) caches pages and API answers and replays queued mutations when back online
|
||||
- **Version history**: built from the activity API of yhub, not from the backend
|
||||
- **State**: Zustand (broadcast store for cross-tab sync), TanStack React Query for server state
|
||||
- **UI**: Mantine 8 + Cunningham design system
|
||||
- **Export**: BlockNote XL packages (PDF, DOCX, ODT)
|
||||
- **i18n**: i18next with Crowdin for community translations
|
||||
|
||||
### Collaboration Flow
|
||||
|
||||
Clients connect via WebSocket to the collaboration server (`src/yhub-server`, built on `@y/hub`), authenticated by their Django session cookie.
|
||||
|
||||
- On every connection (and permission recheck, REST call, fallback poll) yhub asks the backend who the caller is and what they may do: `GET /users/me/`, `GET /documents/{id}/`, and `GET /documents/{id}/accesses/me/` for the history. The backend stays the source of truth for access rights
|
||||
- Updates travel between replicas through Redis/Valkey streams, so yhub replicas are stateless and need no sticky sessions. A worker (`YHUB_ROLE=all|server|worker`) compacts the stream into yhub's own PostgreSQL database (optionally S3 for the blobs)
|
||||
- After a compaction that found new content, yhub calls `POST /documents/{id}/content-updated/` so that the backend updates `updated_at` and the search index
|
||||
- The backend reads and writes content through the REST routes of yhub with an admin JWT (see `YHubService`)
|
||||
|
||||
## Observability
|
||||
|
||||
- **Metrics** (opt-in, `PROMETHEUS_METRICS_ENABLED` + a required `PROMETHEUS_API_KEY` bearer token, both services refuse to start without the key):
|
||||
- backend: django-prometheus on `/metrics`, deliberately outside `/api/`, protected by `core.middleware.PrometheusAuthMiddleware`. Multiprocess mode is set up automatically for the uvicorn workers. See `documentation/metrics.md`
|
||||
- yhub: `src/yhub-server/src/metrics.ts`, served on its own port (`9464`) in every role, the worker included. See the "Metrics" section of `src/yhub-server/README.md`
|
||||
- chart: `ingressMetrics` publishes `/metrics`, `/metrics/yhub` and `/metrics/yhub-worker` on a dedicated host
|
||||
- never put a document id, a user id or a path in a metric label
|
||||
- **Sentry**: backend (`SENTRY_DSN`), y-provider, and yhub (`src/yhub-server/src/sentry.ts`, preloaded, `SENTRY_*` settings; `error`/`fatal` log lines of the shared pino logger are reported)
|
||||
- **Profiling**: django-silk, opt-in with `SILK_ENABLED` (`documentation/profiling.md`)
|
||||
- `documentation/load-testing.md` is the load-testing guide: tooling, setup of an instance, scenarios. Never against production
|
||||
- **Load generators**: `src/loadtest/k6/` (plain k6 scripts: the page-open HTTP sequence and the heavy endpoints; every request tagged with a `name` free of identifiers, CSRF by double-submit with the frontend's `Origin`) and `src/loadtest/swarm/` (standalone TypeScript package, like `src/yhub-server`) opens many Yjs clients on yhub with the frontend's client stack, reading the manifest of `create_load_test_sessions`. Tests run against an in-process y-websocket server (`__tests__/_server.ts`); `src/loadtest/canary/` (same shape, Playwright library) drives a few real Chromium pairs through the frontend to measure page open, editor ready and keystroke propagation, with the manifest's cookies (session + `csrftoken`) set on the context
|
||||
|
||||
## Code Style Guidelines
|
||||
|
||||
### Python (Backend)
|
||||
- **Formatter**: Ruff (line-length: 88)
|
||||
- **Linter**: Ruff + Pylint (pylint runs only on files changed from `origin/main`)
|
||||
- Imports (Ruff enforced): future, stdlib, django, third-party, impress (`core`), first-party, local
|
||||
- Naming: PascalCase classes, snake_case functions/variables, UPPER_SNAKE_CASE constants
|
||||
- **Print statements are forbidden** (T20 rule) - CI will reject them in backend code
|
||||
- Use `@transaction.atomic`, `select_related`/`prefetch_related`, type hints `list[str]`
|
||||
- Run migrations after model changes
|
||||
- In `settings.py`, `post_setup` runs after django-configurations has handed the settings to Django: change lists and dicts **in place** there (`append`, `insert`, item assignment), a reassignment is silently ignored
|
||||
- Use specific exceptions, no unused imports, double quotes, f-strings
|
||||
- **Tests**: Prefer the `settings` fixture from `pytest-django` over `django.test.override_settings`
|
||||
|
||||
### TypeScript/React (Frontend)
|
||||
- **Formatter**: Prettier (single quotes, trailing commas, semicolons, 80 char width)
|
||||
- **Linter**: ESLint 9 with custom `eslint-plugin-docs`
|
||||
- **Style**: Stylelint for CSS
|
||||
- Imports: React & Next.js, libraries, internal, types, styles
|
||||
- Naming: PascalCase components/types, camelCase functions, `use*` hooks, UPPER_SNAKE_CASE constants
|
||||
- Strict mode, interfaces for shapes, Zustand for state, React Query for data
|
||||
|
||||
### TypeScript (yhub-server)
|
||||
- Every environment variable is read, validated and refused at import time in `src/config.ts` (secrets through `secret()` in `src/env.ts`, which supports `NAME_FILE`): add new settings there, with a test in `__tests__/config.spec.ts`
|
||||
- Prettier style with single quotes; tests are Vitest and mock `@y/hub`
|
||||
- `@y/y` must stay the very copy `@y/hub` resolves (`__tests__/dependencies.spec.ts`)
|
||||
|
||||
## Helm/Kubernetes Deployment
|
||||
|
||||
### Structure
|
||||
- `helmfile.yaml.gotmpl` - Helmfile defining releases and environments
|
||||
- `impress/` - Helm chart with templates and default values
|
||||
- `env.d/{env}/` - Environment-specific values (dev=local, feature=CI)
|
||||
|
||||
### Services
|
||||
- Backend (Django + Celery), Frontend (Next.js), yhub (collaboration server, plus an optional separate worker with `yhub.worker.enabled`), yProvider (conversion only)
|
||||
- Two standalone Valkey instances from the official `valkey/valkey` chart: `valkey-docs` (backend cache, sessions, Celery) and `valkey-yhub`
|
||||
- `dev` only (`monitoring: true` in the helmfile environment): a trimmed `kube-prometheus-stack` release (`env.d/dev/values.prometheus.yaml.gotmpl`, operator + CRDs + one Prometheus at https://docs-prometheus.127.0.0.1.nip.io) scraping the backend and yhub through the `serviceMonitor` of the chart, with the token in the `docs-metrics` Secret it creates, and a Grafana at https://docs-grafana.127.0.0.1.nip.io (admin/admin) whose sidecar loads the dashboards of `src/loadtest/dashboards/` from ConfigMaps (legend `{{` escaped for helm's tpl, as the values file explains)
|
||||
- Websocket ingress on `/collaboration/ws/` with no `upstream-hash-by`: any yhub replica serves any document
|
||||
- DocSpec (conversion), Posthog (analytics) - optional
|
||||
|
||||
### Deployment Model
|
||||
- Kind cluster with mkcert HTTPS, local registry (localhost:5001)
|
||||
- CoreDNS for nip.io domains, dev-backend deps (Keycloak, PostgreSQL, Minio)
|
||||
|
||||
### Helm Guidelines
|
||||
- Never commit secrets; use Got templating `{{ now | unixEpoch }}`
|
||||
- Follow semantic versioning; values use `@param` annotations
|
||||
|
||||
## Git Conventions
|
||||
|
||||
### Commit Format
|
||||
|
||||
```
|
||||
<gitmoji>(type) lowercase title
|
||||
|
||||
Mandatory description explaining why.
|
||||
```
|
||||
|
||||
- **No space** between emoji and `(type)`, **one space** after closing parenthesis
|
||||
- Types: `backend`, `frontend`, `ci`, `docker`, `dependencies`, `e2e`, `export`, `auth`, etc.
|
||||
- Gitmoji examples: `✨` feature, `🐛` fix, `♻️` refactor, `⬆️` dependency upgrade, `🔒️` security
|
||||
- Commits must be signed (`-S`) and signed-off (`--signoff`)
|
||||
|
||||
### Changelog
|
||||
|
||||
Update `CHANGELOG.md` under `[Unreleased]` with format: `- <gitmoji>(type) description` (max 80 chars per line).
|
||||
|
||||
### PR Checklist
|
||||
|
||||
Run before submitting: `make lint && make frontend-lint && make test`
|
||||
|
||||
## General Practices
|
||||
|
||||
- Commit messages should be clear and descriptive
|
||||
- Run `make lint` and tests before committing
|
||||
- Use DRF for REST APIs with proper serializers
|
||||
- Implement proper error handling with try/except blocks
|
||||
- Use Django settings for configuration
|
||||
- Internationalize user-facing strings with `_()`
|
||||
- Cache expensive operations, use database indexes
|
||||
- Write docstrings for classes and complex functions
|
||||
- Use type checking: TypeScript (frontend), mypy optional (Python)
|
||||
|
||||
## Environment Configuration
|
||||
|
||||
Config files in `env.d/development/`: `common`, `postgresql`, `kc_postgresql`, `crowdin`, `yhub`, `yhub-postgres`. The collaboration server reads its own two files and none of the backend's. Create `.local` overrides (gitignored) for custom settings.
|
||||
|
||||
## i18n
|
||||
|
||||
```bash
|
||||
make i18n-generate # Extract translation strings (backend + frontend)
|
||||
make i18n-compile # Compile translations for all apps
|
||||
```
|
||||
|
||||
Internationalize user-facing strings with `_()` (backend) or i18next (frontend).
|
||||
|
||||
## Key Directories
|
||||
|
||||
- `src/backend/` - Django REST API backend
|
||||
- `src/frontend/` - Next.js (apps/impress, packages/i18n, servers/y-provider)
|
||||
- `src/yhub-server/` - Collaboration server (yhub)
|
||||
- `src/helm/` - Helmfile and Helm chart for Kubernetes deployment
|
||||
- `src/mail/` - Email template generator (MJML)
|
||||
- `documentation/` - Documentation
|
||||
- `.github/` - GitHub workflows and templates
|
||||
@@ -20,6 +20,7 @@ and this project adheres to
|
||||
- ✨(loadtest) add browser canaries measuring what a user feels under load
|
||||
- ✨(loadtest) add the grafana dashboards of the load-test campaign, valkey
|
||||
included
|
||||
- 📝(documentation) add the load-testing guide
|
||||
- 🔧(helm) run grafana with those dashboards in the dev cluster, in place of
|
||||
the prometheus console
|
||||
- ✨(backend) measure the calls to yhub and to the converters, the database
|
||||
|
||||
+21
-10
@@ -35,6 +35,10 @@ upgrade, in the order they are done, and end with the API changes.
|
||||
POSTGRES: postgres://{user}:{password}@{postgres-host}:5432/yhub
|
||||
```
|
||||
|
||||
The PostgreSQL connection is not encrypted unless the url asks for it: append
|
||||
`?sslmode=require` (or `verify-full`) to `POSTGRES` for a server that only
|
||||
accepts TLS, see "Database schema" in `src/yhub-server/README.md`.
|
||||
|
||||
It also needs `COLLABORATION_BACKEND_BASE_URL`, the backend it asks about
|
||||
users and document access rights, and `COLLABORATION_SERVER_ORIGIN`, the
|
||||
origins allowed to open a websocket — the two the `y-provider` already had.
|
||||
@@ -123,14 +127,16 @@ upgrade, in the order they are done, and end with the API changes.
|
||||
Both `/collaboration/` ingresses now point at the collaboration server. In
|
||||
the chart, `ingressCollaborationApi.path` (a single path) becomes
|
||||
`ingressCollaborationApi.paths` (a list, one ingress rule each), defaulting
|
||||
to `/collaboration/ydoc/` and `/collaboration/jwks/`. What is not listed
|
||||
stays in-cluster, which is how `create-ydoc`, `reset-connections`, `migrate`,
|
||||
`restore-ydoc` and `reset-ydoc` are kept unreachable: they are
|
||||
backend-internal, and publishing them would put document deletion and the
|
||||
legacy migration one request away from the internet. If you route
|
||||
`/collaboration/` by hand, publish the websocket, the browser-facing document
|
||||
routes (`ydoc`, `rollback`, `prune`, `changeset`, `activity`) and `jwks`, and
|
||||
keep those five in-cluster.
|
||||
to `/collaboration/ydoc/`, `/collaboration/activity/`,
|
||||
`/collaboration/changeset/`, `/collaboration/rollback/` and
|
||||
`/collaboration/jwks/`. What is not listed stays in-cluster, which is how
|
||||
`prune`, `create-ydoc`, `reset-connections`, `migrate`, `restore-ydoc` and
|
||||
`reset-ydoc` are kept unreachable: they are backend-internal or granted to
|
||||
nobody, and publishing them would put document deletion and the legacy
|
||||
migration one request away from the internet. If you route `/collaboration/`
|
||||
by hand, publish the websocket, the browser-facing document routes (`ydoc`,
|
||||
`activity`, `changeset`, `rollback`) and `jwks`, and keep those six
|
||||
in-cluster.
|
||||
|
||||
The `nginx.ingress.kubernetes.io/upstream-hash-by: $arg_room` annotation is
|
||||
dropped from the websocket ingress, and should be dropped from yours: the
|
||||
@@ -175,8 +181,7 @@ upgrade, in the order they are done, and end with the API changes.
|
||||
|
||||
The `y-provider` verifies the same way and only needs to reach the backend
|
||||
for it: `COLLABORATION_BACKEND_BASE_URL`, from which it derives
|
||||
`{base}/api/v1.0/jwks`, or `JWKS_URL` when that url is not the right one from
|
||||
where it runs. In a development environment, `make generate-secret-keys`
|
||||
`{base}/api/v1.0/jwks`. In a development environment, `make generate-secret-keys`
|
||||
creates the key in `data/jwt/`; on a cluster, `jwtKeys.enabled` makes the
|
||||
chart generate both keys in a secret the services mount read-only.
|
||||
|
||||
@@ -243,6 +248,12 @@ upgrade, in the order they are done, and end with the API changes.
|
||||
and it is a third bucket, not the backend's `AWS_S3_*` nor the legacy one the
|
||||
migration reads.
|
||||
|
||||
- The collaboration server reports its errors to Sentry when `SENTRY_DSN` is
|
||||
set, like the backend and the `y-provider` do (`SENTRY_ENVIRONMENT`,
|
||||
`SENTRY_RELEASE`, `SENTRY_TRACES_SAMPLE_RATE` and
|
||||
`SENTRY_PROFILES_SAMPLE_RATE` alongside, all optional). Unset, nothing is
|
||||
reported and the SDK is not loaded.
|
||||
|
||||
- The endpoint `/api/v1.0/documents/{document_id}/content/`, added in 5.0.0, is
|
||||
removed, both its `GET` and its `PATCH`. The content of a document is now
|
||||
saved and served by the collaboration server, the editor exchanging it over
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
# Load testing Docs
|
||||
|
||||
How to find out whether a deployment of Docs holds the load it is meant for,
|
||||
and where it stops: the tooling this repository ships for it, what it measures,
|
||||
and how to run a campaign with it.
|
||||
|
||||
> ⚠️ **Never run any of this against a production instance.** The load
|
||||
> generators log in as existing users with sessions minted outside of the
|
||||
> identity provider, write into documents, create and delete documents, and
|
||||
> put the whole deployment under stress on purpose. Run it on a copy: an
|
||||
> instance of its own, loaded with **anonymized** data, sized like the one
|
||||
> whose capacity you want to know. The `LoadTest` configuration that mints
|
||||
> the sessions cannot be enabled on the `Production` one, on purpose.
|
||||
|
||||
## What is being tested
|
||||
|
||||
Docs is a Django backend (HTTP API, Celery tasks), a collaboration server
|
||||
(`src/yhub-server`, built on `@y/hub`) holding the content of the documents,
|
||||
two Valkey instances, PostgreSQL (one database each for the backend and the
|
||||
collaboration server), an object storage, and a Next.js frontend. The parts
|
||||
that decide how it scales, and that the scenarios below target:
|
||||
|
||||
- Every websocket connection, reconnection and http-fallback poll makes the
|
||||
collaboration server ask the backend two or three questions (`users/me/`,
|
||||
`documents/{id}/`, `documents/{id}/accesses/me/`). A reconnection storm is
|
||||
therefore a backend storm.
|
||||
- The collaboration server's replicas share nothing: updates travel through
|
||||
Valkey streams, and a worker compacts them into PostgreSQL. Its throughput is
|
||||
`YHUB_TASK_CONCURRENCY` times the number of workers.
|
||||
- Each compaction that found new content calls the backend
|
||||
(`content-updated`), which updates the document and reindexes it, which
|
||||
reads the document back from the collaboration server: a loop that wide
|
||||
editing feeds.
|
||||
- Some backend endpoints call the collaboration server synchronously, inside
|
||||
the request: document creation from a file, `duplicate` (two round-trips per
|
||||
node, in one transaction), `formatted-content`. Delete, restore and access
|
||||
changes walk the subtree from Celery tasks, one call per node, on a single
|
||||
queue.
|
||||
- The backend opens a connection per call to the collaboration server, and
|
||||
the collaboration server's calls to the backend have no timeout: a slow
|
||||
backend piles them up.
|
||||
- The API throttles per user (80 requests a minute on the document
|
||||
endpoints), and the media route makes an access check for every attachment
|
||||
a page loads.
|
||||
- Under soft migration (`SOFT_MIGRATION=true`), a document is seeded from the
|
||||
legacy store the first time it is opened, inside the websocket upgrade,
|
||||
at most 20 at a time per replica; past that the client gets a 503 and
|
||||
retries.
|
||||
|
||||
## What the repository ships
|
||||
|
||||
| Piece | Where | What it does |
|
||||
| ----- | ----- | ------------ |
|
||||
| Metrics | `metrics.md`, `src/yhub-server/README.md` | Prometheus metrics of the backend (requests by view, SQL, calls to the other services, the database pool, the Celery queue) and of the collaboration server (sockets, authorizations, backend calls, compactions, seeds), behind a bearer token |
|
||||
| Session minting | `src/backend/loadtest/` | `create_load_test_sessions` logs existing users in without the identity provider and writes a manifest the generators read; `revoke_load_test_sessions` undoes it. Only with `DJANGO_CONFIGURATION=LoadTest` |
|
||||
| Swarm | `src/loadtest/swarm/` | Thousands of websocket clients on the collaboration server, with the frontend's own client stack: connect time, sync time, edit propagation, reconnections, convergence |
|
||||
| k6 scenarios | `src/loadtest/k6/` | The HTTP side: the sequence a browser runs when a document is opened, and the heavy endpoints |
|
||||
| Canary | `src/loadtest/canary/` | A few real browsers opening and editing documents in a loop: what a user feels while the rest applies load |
|
||||
| Dashboards | `src/loadtest/dashboards/` | Grafana boards over all of the above, plus Valkey and the community Django board |
|
||||
|
||||
Each directory has a README with its options. The dev cluster of this
|
||||
repository (`make start-tilt`) runs the whole stack with a Prometheus and a
|
||||
Grafana holding the boards, which is where to try the tooling before a
|
||||
campaign.
|
||||
|
||||
## Setting an instance up for a campaign
|
||||
|
||||
The instance under test is a deployment of Docs like any other, with:
|
||||
|
||||
| What | How |
|
||||
| ---- | --- |
|
||||
| Session minting | `DJANGO_CONFIGURATION=LoadTest` on the backend. It is `Production` plus the `loadtest` application. Anybody who can run a management command on that instance can then act as any of its users: never an instance with real users |
|
||||
| Metrics | `PROMETHEUS_METRICS_ENABLED` and a `PROMETHEUS_API_KEY` on the backend and on the collaboration server; in the Helm chart, `backend.metrics.enabled`, `yhub.metrics.enabled`, then `serviceMonitor.enabled` or `podMonitor.enabled` for a Prometheus inside the cluster (one target per pod, the better option), or `ingressMetrics` for one outside. See `metrics.md` |
|
||||
| Sizing | The same replicas, resource limits, database pool and pooler, `YHUB_TASK_CONCURRENCY` and worker split as the deployment whose capacity is in question. Otherwise the numbers are only relative |
|
||||
| Data | An anonymized copy of that deployment's database, with `migrate_documents` run to the end, and the object storage it points to |
|
||||
| Noise | Emails, analytics, AI and webhooks off; `django-silk` off or sampling 1 % (`profiling.md`) |
|
||||
| Error reporting | Sentry on both services, with a low trace sampling rate |
|
||||
| Stores | The Valkey and PostgreSQL exporters in the same Prometheus. The Valkey board's streams row needs `redis_exporter` started with `--check-streams` on the collaboration server's instance (`src/loadtest/dashboards/README.md`) |
|
||||
|
||||
Import the boards of `src/loadtest/dashboards/` into the Grafana; they take
|
||||
its default Prometheus datasource, switchable at the top of each board.
|
||||
|
||||
A workload model, from the deployment being sized rather than guessed: peak
|
||||
concurrent users, connections per document (mostly one, a few with dozens),
|
||||
edits per minute, share of readers. Run at 1×, 2× and 5× of it, then ramp
|
||||
until something bends.
|
||||
|
||||
## Logging the virtual users in
|
||||
|
||||
From a pod running the backend image with the `LoadTest` configuration (the
|
||||
chart's `backend.job.command` runs a management command as a Job):
|
||||
|
||||
```bash
|
||||
python manage.py create_load_test_sessions 5000 --heaviest 50 \
|
||||
--documents-per-user 20 --public-documents 100 \
|
||||
--storage-name campaign-1.json --ttl-hours 12
|
||||
```
|
||||
|
||||
The manifest goes to a private object of the default storage
|
||||
(`loadtest/campaign-1.json`), or to a `0600` file with `--output`. It lists
|
||||
the cookie name and, per user, a session key and the documents that user may
|
||||
edit or read. **It holds live sessions and is a secret**: only the generators
|
||||
read it, and it is revoked at the end:
|
||||
|
||||
```bash
|
||||
python manage.py revoke_load_test_sessions --storage-name campaign-1.json
|
||||
```
|
||||
|
||||
Users are picked among active, non-staff users holding an access to a live
|
||||
document; `--heaviest N` takes the N holding the most. One distinct user per
|
||||
virtual client, or the per-user throttle distorts every measurement: mint at
|
||||
least as many sessions as the largest run needs, and give each generator pod
|
||||
its own slice of the manifest's `sessions`.
|
||||
|
||||
Check once, before any run, that the collaboration server sees the users:
|
||||
open one socket with a cookie of the manifest and read its logs for the
|
||||
`userid`. A manifest minted for another configuration or another session
|
||||
store makes the clients anonymous, which looks like success on public
|
||||
documents. (In this repository's compose stack, the `Development`
|
||||
configuration keeps its sessions in another Redis database than `LoadTest`:
|
||||
mint with `REDIS_URL=redis://redis:6379/2` there.)
|
||||
|
||||
## Where the load comes from
|
||||
|
||||
From inside the cluster of the instance under test, as Jobs on nodes that do
|
||||
not host the application, going through the ingress like a browser would.
|
||||
Not from a laptop, and not from the application nodes: at ten thousand
|
||||
sockets the swarm burns a core of its own, and the ingress, TLS termination
|
||||
and timeouts are part of what is measured.
|
||||
|
||||
| Generator | Image | Runs as |
|
||||
| --------- | ----- | ------- |
|
||||
| swarm | `docker build -f src/loadtest/swarm/Dockerfile .` | one Job per slice of the manifest, a few thousand sockets each |
|
||||
| k6 | `grafana/k6`, with `src/loadtest/k6` mounted | one Job per scenario |
|
||||
| canary | `docker build -f src/loadtest/canary/Dockerfile .` | one Job, always on during the campaign |
|
||||
|
||||
The swarm and canary images are not built by CI: build them from the
|
||||
repository root and push them to the cluster's registry. Every generator
|
||||
serves `/metrics` (`--metrics-port`, `--metrics-token`): scrape the generator
|
||||
pods too, so that their numbers sit next to the servers' on the boards. k6
|
||||
pushes its own with `-o experimental-prometheus-rw` and
|
||||
`K6_PROMETHEUS_RW_SERVER_URL`.
|
||||
|
||||
The options every run shares, with `docs.example.com` as the instance:
|
||||
|
||||
```bash
|
||||
# swarm
|
||||
node dist/index.js --manifest /manifest.json --url wss://docs.example.com \
|
||||
--metrics-port 9465 --metrics-token "$TOKEN" --report /out/swarm.json ...
|
||||
# k6
|
||||
k6 run --env MANIFEST=/manifest.json --env BASE_URL=https://docs.example.com \
|
||||
--env MEDIA_BASE_URL=https://docs.example.com \
|
||||
-o experimental-prometheus-rw scenarios/page-open.js
|
||||
# canary
|
||||
node dist/index.js --manifest /manifest.json --url https://docs.example.com \
|
||||
--pairs 3 --duration 28800 --metrics-port 9466 --metrics-token "$TOKEN"
|
||||
```
|
||||
|
||||
k6's `ORIGIN` defaults to `BASE_URL`, right when `/api` is served on the
|
||||
application host; it has to be one of `DJANGO_CSRF_TRUSTED_ORIGINS`. The
|
||||
swarm's `--origin` defaults to `https://` plus the host of `--url`, which has
|
||||
to be in `COLLABORATION_SERVER_ORIGIN`.
|
||||
|
||||
## The runs
|
||||
|
||||
Targets to set before starting, adjusted to the workload model: a document
|
||||
opens in under 2 s at the 95th percentile, an edit reaches the other clients
|
||||
in under 500 ms, fewer than 0.1 % of requests fail, no document diverges.
|
||||
|
||||
One scenario at a time; stepped ramps with plateaus; the canary up and the
|
||||
boards open throughout. At each knee, note which panel bent first: that is
|
||||
the capacity limit of the scenario, and the number that goes into the table
|
||||
the campaign produces (sockets per collaboration replica, connections per
|
||||
second per backend replica, edits per second per worker, documents migrated
|
||||
per hour).
|
||||
|
||||
| # | Scenario | Command | What to watch |
|
||||
| - | -------- | ------- | ------------- |
|
||||
| 1 | Migration | `migrate_documents --concurrency N` on the full volumetry; then, with `SOFT_MIGRATION=true`, swarm `--mode wide` on unmigrated documents | documents per hour, growth of the collaboration database; `yhub_seed_*`, refused seeds |
|
||||
| 2 | HTTP baseline | k6 `page-open.js --env RATE=<1×> --env DURATION=10m`, on the previous release and on this one, same data | backend board: latency by view, database pool, `media-auth` |
|
||||
| 3 | Connect ramp | swarm `--mode idle --ramp 20`, raising `--ramp` per run until upgrades fail | `yhub_auth_duration_seconds`, backend calls in flight, `swarm_upgrade_failures_total`, `swarm_connect_duration_seconds` |
|
||||
| 4 | Idle steady state | swarm `--mode idle --clients 10000 --duration 1800`, several Jobs | memory per collaboration replica, event-loop lag, Valkey memory and network |
|
||||
| 5 | Hot document | swarm `--mode hot --doc <id> --clients 200 --writers 0.5 --edit-interval 500` | propagation p95, Valkey output, event-loop lag |
|
||||
| 6 | Wide editing | swarm `--mode wide --clients 3000 --writers 0.3` | `yhub_worker_pending_tasks`, compaction duration, `docs_outgoing_*` (the reindexing loop), Celery queue |
|
||||
| 7 | Reconnection storms | any of 4 to 6 with `--storm-at 300`; then, during a hold, a rolling restart of the collaboration server, a Valkey failover, a PostgreSQL failover | `swarm_reconnects_total`, closes by code, authorizations `unavailable`, backend calls in flight; sessions and cache on the Valkey board |
|
||||
| 8 | HTTP fallback | not automated: a browser on a network that refuses websocket upgrades, with the canary | backend load per such client (about 0.3 requests/s each) |
|
||||
| 9 | Heavy endpoints | k6 `heavy.js --env VUS=5` | `docs_outgoing_*`, pool waiting, Celery queue. See the known issue on `duplicate` below |
|
||||
| 10 | Large documents | canary `--doc <one of the largest>` in turn | `canary_editor_ready_seconds` |
|
||||
| 11 | Soak | 4 to 8 h at 1×: swarm wide, k6 page-open and the canary together | memory growth, Valkey memory, bloat of the collaboration database, `scrape_duration_seconds` of the backend |
|
||||
|
||||
Keep, per run: the generators' JSON reports and the k6 summary, a Grafana
|
||||
snapshot or the time range, the values the instance ran with, and the line of
|
||||
the capacity table it produced.
|
||||
|
||||
## Known issues and limits
|
||||
|
||||
- **Concurrent `duplicate` calls collide on tree paths.** Several users
|
||||
duplicating a document at the same time get a 500 (`IntegrityError` on the
|
||||
document path) or a 400 ("Document with this Path already exists") on a
|
||||
fraction of the calls: root path allocation races between requests. Fix it
|
||||
before scenario 9, or the scenario measures the bug.
|
||||
- Under soft migration, a seed refused at the per-replica limit is logged as
|
||||
an error and reported to Sentry, one event per refused open.
|
||||
- The collaboration server's calls to the backend have no timeout. Decide
|
||||
before the storm scenario whether to measure that as is or to add one.
|
||||
- The backend exports no process metrics (CPU, memory) in multiprocess mode;
|
||||
take them from the cluster. Celery task duration is not exported.
|
||||
- The swarm does not drive the http fallback, and does not check the
|
||||
persisted document against the collaboration server's REST API (it would
|
||||
need an admin token).
|
||||
- The swarm and the canary write into the documents they open, and the heavy
|
||||
scenario leaves copies and imports in the trash until
|
||||
`TRASHBIN_CUTOFF_DAYS`. Anonymized data only.
|
||||
|
||||
## After the campaign
|
||||
|
||||
- `revoke_load_test_sessions`, and delete the manifest object.
|
||||
- Put `DJANGO_CONFIGURATION` back to its normal value: the `loadtest`
|
||||
application must not stay installed.
|
||||
- `purge_silk_profiles` if silk was on.
|
||||
@@ -1,478 +0,0 @@
|
||||
# Stress test plan — yhub architecture (next major release)
|
||||
|
||||
Status: plan, written 2026-09-18 from a code read of branch `yhub`. Nothing in
|
||||
here has been run yet. File and line references are valid for that branch at
|
||||
that date; re-check them before relying on them.
|
||||
|
||||
## Goal
|
||||
|
||||
Determine whether the new architecture (Django backend + `yhub-server` built on
|
||||
`@y/hub`, Valkey streams, Postgres persistence) scales, and find where it stops
|
||||
scaling, before the release reaches production.
|
||||
|
||||
Environment: preprod, loaded with an anonymized copy of the production database
|
||||
on which the full migration has been run, so that the volumetry is realistic.
|
||||
|
||||
Deliverable: a capacity table (sockets per yhub pod, connects per second per
|
||||
backend pod, edits per second per yhub worker, migrated documents per hour)
|
||||
that tells how to size production and where autoscaling makes sense.
|
||||
|
||||
## Architecture facts that shape the tests
|
||||
|
||||
Collaboration path:
|
||||
|
||||
- The browser uses plain `y-websocket` `WebsocketProvider`
|
||||
(`src/frontend/apps/impress/src/features/docs/doc-management/stores/useProviderStore.tsx:154-160`)
|
||||
with `disableBc: true`, `maxBackoffTime: 30000`, `resyncInterval: 20000`.
|
||||
Wire protocol is y-protocols sync + awareness.
|
||||
- WS URL: `wss://{host}/collaboration/ws/v1/docs/{docid}`. The doc id must be a
|
||||
lowercase uuid4, the org must be `docs`.
|
||||
- HTTP fallback on the same rooms (`@y/yhub-http-fallback`), about one poll
|
||||
every 10 s, only while the socket is down.
|
||||
- Authentication of the WS upgrade is the Django session cookie, forwarded by
|
||||
yhub to the backend. There is no `collaboration-auth` ingress subrequest
|
||||
anymore.
|
||||
- Every WS connect, reconnect and fallback request triggers 2 to 3 Django
|
||||
calls from yhub: `GET /users/me/` (`src/yhub-server/src/server.ts:190`),
|
||||
`GET /documents/{id}/` (`:252`), and `GET /documents/{id}/accesses/me/` when
|
||||
the user can list versions (`:279`).
|
||||
- `backendFetch` in yhub has no timeout and no retry
|
||||
(`src/yhub-server/src/backend.ts:116-139`). A failure gives a 503, which
|
||||
`y-websocket` retries forever.
|
||||
- yhub pods are stateless, cross-pod fan-out goes through Valkey, there are no
|
||||
sticky sessions (`upstream-hash-by` was removed on purpose).
|
||||
- A worker drains the Valkey stream into Postgres. Throughput is
|
||||
`YHUB_TASK_CONCURRENCY` (default 5) times worker replicas, debounce
|
||||
`YHUB_TASK_DEBOUNCE_MS` 10000.
|
||||
- Each compaction fires `POST /documents/{id}/content-updated/` on Django
|
||||
(5 s timeout, not awaited). Django updates the row and triggers the search
|
||||
indexer, which calls `get_ydoc` back on yhub. This is a feedback loop.
|
||||
|
||||
Backend:
|
||||
|
||||
- The Django to yhub client opens a new connection per call, with no
|
||||
`requests.Session`, no retry, 30 s timeout
|
||||
(`src/backend/core/services/yhub_services.py:196-229`).
|
||||
- Synchronous yhub calls inside HTTP requests: document creation from a file,
|
||||
`duplicate` (2 round-trips per node, inside `transaction.atomic`),
|
||||
`formatted-content` (yhub + converter), `create-for-owner`, first login with
|
||||
the onboarding sandbox document.
|
||||
- Asynchronous (Celery): delete, restore and access changes walk the whole
|
||||
subtree with one HTTP call per node. Single default queue, 1 worker replica.
|
||||
- No `CONN_MAX_AGE`. The psycopg pool exists but is off by default
|
||||
(`DB_PSYCOPG_POOL_ENABLED`).
|
||||
- uvicorn, `WEB_CONCURRENCY=4`, `--limit-max-requests=20000`.
|
||||
- Sessions are cache-backed (`SESSION_ENGINE = cache`).
|
||||
- DRF throttles: 80/min per user on documents, 50/min on document accesses.
|
||||
yhub is exempt through `X-Y-Provider-Key`.
|
||||
|
||||
Migration:
|
||||
|
||||
- One cheap Django migration (`0034_documentmigration`).
|
||||
- `migrate_documents`: thread pool, `--concurrency` default 2, `--rate`,
|
||||
`--limit`, resumable and idempotent.
|
||||
- `SOFT_MIGRATION=true`: unmigrated documents are seeded on first open, inside
|
||||
the WS upgrade handler. Each replica runs at most `MAX_CONCURRENT_SEEDS=20`
|
||||
seeds at once and fails fast beyond that: the caller gets a 503 and relies
|
||||
on the client's retry backoff, it is not queued. A 30 s Redis lock
|
||||
serialises seeds of the same document across replicas
|
||||
(`src/yhub-server/src/migration.ts:63-64`, `:419-429`).
|
||||
- `generate_volumetry` writes no document content. Only `create_demo` and
|
||||
`migrate_documents` put content in yhub.
|
||||
|
||||
Deployment (`src/helm`):
|
||||
|
||||
- No HPA anywhere, `resources: {}` on every impress container, static replicas
|
||||
(backend 3, yhub 3, yhub worker disabled by default, Celery 1).
|
||||
- When this plan was written there was no metrics endpoint in Django or yhub
|
||||
(done since, see sections 0.1 and 0.2).
|
||||
- Valkey is not deployed from this repository on real clusters: another team
|
||||
runs it, through the valkey operator (Sentinel mode), one instance for the
|
||||
backend (`valkey-docs`: cache, sessions, Celery) and one for yhub
|
||||
(`valkey-yhub`). In this repository, dev and feature only run two standalone
|
||||
instances from the `valkey/valkey` chart
|
||||
(`src/helm/env.d/{env}/values.valkey.yaml.gotmpl`), with
|
||||
`maxmemory-policy volatile-lru`. Stream entries without a TTL cannot be
|
||||
evicted under that policy, so the memory limit of `valkey-yhub` matters.
|
||||
- The only nginx auth subrequest left is `media-auth`.
|
||||
|
||||
Frontend:
|
||||
|
||||
- On connection loss, each client waits up to 3 s of jitter, then refetches
|
||||
`GET /documents/{id}/`.
|
||||
- The service worker `SyncManager` replays queued mutations with no backoff
|
||||
and no jitter
|
||||
(`src/frontend/apps/impress/src/features/service-worker/SyncManager.ts:17-45`).
|
||||
|
||||
## 0. Prerequisites
|
||||
|
||||
Without metrics a test only shows that something broke, not where. Real
|
||||
clusters are synced by Argo CD and there is no kubectl access, so dashboards
|
||||
are the only way to observe a run. Everything below ships through git and
|
||||
helm values.
|
||||
|
||||
First question to settle with the ops team: what scrapes metrics in preprod
|
||||
(Prometheus operator with ServiceMonitor and PodMonitor, or annotation-based
|
||||
scraping) and where dashboards live. The chart work depends on the answer.
|
||||
|
||||
### 0.1 yhub-server
|
||||
|
||||
**Implemented** (2026-09-18): metrics and Sentry. See the "Metrics" and "Error
|
||||
reporting" sections of `src/yhub-server/README.md`.
|
||||
|
||||
- `PROMETHEUS_METRICS_ENABLED=true` and `PROMETHEUS_API_KEY` (required) start
|
||||
a listener on its own port (`9464`) in every role, the worker included. It
|
||||
answers one path to a bearer token, like the backend's `/metrics`.
|
||||
- Chart: `yhub.metrics.enabled`. `serviceMonitor.enabled` or
|
||||
`podMonitor.enabled` then scrapes every pod from inside the cluster (one
|
||||
monitor per component, `backend.metrics.enabled` for the backend); the
|
||||
dedicated `ingressMetrics` publishes `/metrics/yhub` and
|
||||
`/metrics/yhub-worker` next to the backend's `/metrics` for a Prometheus
|
||||
outside of it.
|
||||
- `@y/hub` 0.9.0 owns the uWebSockets server and has no hook on socket open,
|
||||
close or message. What is measured is what our code and its public events
|
||||
allow:
|
||||
|
||||
| Metric | What to watch during a run |
|
||||
|---|---|
|
||||
| `yhub_ws_connections`, `yhub_rooms` | sockets and open documents per replica (read off `stream.subs`, an internal of yhub) |
|
||||
| `yhub_auth_duration_seconds{phase,endpoint,result}` | cost of admitting a caller: upgrades, rechecks, fallback polls. `result="unavailable"` is a 503 sent to the client |
|
||||
| `yhub_backend_request_duration_seconds{route,status}`, `yhub_backend_requests_inflight{route}` | the 2 to 3 Django calls per connect. Inflight is what piles up when Django slows down, since `backendFetch` has no timeout |
|
||||
| `yhub_worker_pending_tasks` | compaction backlog for the whole deployment: use `max`, not `sum` |
|
||||
| `yhub_worker_task_duration_seconds`, `yhub_worker_tasks_inflight` | worker saturation against `YHUB_TASK_CONCURRENCY` |
|
||||
| `yhub_doc_updates_total` | rate of `content-updated` notifications sent to Django |
|
||||
| `yhub_seed_duration_seconds`, `yhub_seeds_inflight`, `yhub_seed_rejected_total` | soft migration under load, and the opens refused at 20 concurrent seeds |
|
||||
| `nodejs_eventloop_lag_seconds` | the first signal of a saturated replica: one thread serves all its sockets |
|
||||
|
||||
- Not visible from inside yhub: messages and bytes per socket, close codes,
|
||||
Postgres pool state. Read them from the Valkey exporter, ingress-nginx and
|
||||
`pg_stat_activity` on the `yhub` database.
|
||||
- Logs: `@y/hub` logs every socket connect and close at info level. Check the
|
||||
log pipeline can take it at 10k sockets.
|
||||
- Sentry: `error` and `fatal` log lines are reported. The refusal of a seed at
|
||||
20 concurrent ones is logged at error level (`seed.failed`), so the soft
|
||||
migration scenario will send one Sentry event per refused open. Lower that
|
||||
log line to `warn`, or expect the volume.
|
||||
|
||||
Decision still to take before the first storm scenario: `backendFetch` has no
|
||||
timeout. Either leave it as is for the first run, to measure the real
|
||||
behaviour, or add `AbortSignal.timeout` first.
|
||||
|
||||
### 0.2 Django backend
|
||||
|
||||
1. Request and SQL metrics: **implemented** (2026-09-18), see
|
||||
`documentation/metrics.md`. `django-prometheus` is opt-in
|
||||
(`PROMETHEUS_METRICS_ENABLED`), labels are view name, method and status.
|
||||
- Served on `/metrics` (not under `/api/`), behind a bearer token
|
||||
(`PROMETHEUS_API_KEY`, required) checked by the first middleware. The
|
||||
chart has a dedicated `ingressMetrics`, to be filtered by address.
|
||||
- The uvicorn workers share their numbers through a local directory
|
||||
(multiprocess mode, required with `WEB_CONCURRENCY=4`), defaulted so that
|
||||
nothing has to be configured.
|
||||
- Scraping through the ingress reaches a different pod each time. Every
|
||||
sample has a `hostname` label so the replicas stay apart, but each series
|
||||
is only sampled one scrape out of N. For the load test, with many backend
|
||||
pods, prefer scraping each pod from inside the cluster if preprod allows
|
||||
it; otherwise use a short scrape interval and 5 min rate windows.
|
||||
- Known limit: workers recycled by `--limit-max-requests=20000` leave
|
||||
their files behind, so the directory grows until the pod is replaced.
|
||||
Watch `scrape_duration_seconds` during the soak scenario.
|
||||
2. Custom metrics on the known pressure points: **implemented**
|
||||
(2026-09-21), see `documentation/metrics.md`.
|
||||
- `docs_outgoing_request_duration_seconds{service,operation,method,status}`
|
||||
and `docs_outgoing_requests_inflight`: the calls to yhub
|
||||
(`YHubService.request`) and to the two converters, timeouts told apart
|
||||
from errors;
|
||||
- `docs_db_pool_*`: state and exact counters of the psycopg pool when
|
||||
`DB_PSYCOPG_POOL_ENABLED` is on. `requests_queued_total` and
|
||||
`requests_wait_seconds_total` are the signal that was missing in the
|
||||
2026-08-18 and 2026-09-07 outages;
|
||||
- `docs_celery_queue_length`: backlog of the single default queue the
|
||||
subtree cascades land on (use `max`, not `sum`);
|
||||
- not done: Celery task duration by task name. The workers serve no
|
||||
endpoint, so it needs either a pushgateway or the multiprocess directory
|
||||
shared with a small exporter.
|
||||
3. Query-level detail: silk is already wired. Enable it in preprod with
|
||||
`SILKY_INTERCEPT_PERCENT` at 1 to 2 % and run `purge_silk_profiles`
|
||||
between runs. Set `SENTRY_TRACES_SAMPLE_RATE` (default 0.0,
|
||||
`settings.py:532`) to about 0.01 if Sentry is available in preprod, for
|
||||
traces across Django, Celery and outgoing HTTP calls.
|
||||
4. Request logs: keep `LOGGING_LEVEL_REQUEST_SUMMARY` on, the dockerflow
|
||||
summary carries the path, status and duration of every request and is the
|
||||
fallback when a metric is missing.
|
||||
5. Configuration parity with production: `DB_PSYCOPG_POOL_*`, pooler,
|
||||
`WEB_CONCURRENCY`, cache and session Valkey settings, Celery concurrency,
|
||||
throttle rates. Disable outgoing emails, PostHog, AI and webhooks.
|
||||
|
||||
### 0.3 Synthetic sessions
|
||||
|
||||
**Implemented** (2026-09-21) as a dedicated Django application,
|
||||
`src/backend/loadtest/`, that only the `LoadTest` configuration installs.
|
||||
|
||||
Why it is needed: authentication is session-only
|
||||
(`DEFAULT_AUTHENTICATION_CLASSES` holds only `SessionAuthentication`) and the
|
||||
Keycloak realm has 4 static users, so load clients cannot log in through OIDC at
|
||||
scale. Sessions are cache-backed, `MIDDLEWARE` contains no OIDC token-refresh
|
||||
middleware, and yhub forwards the same cookie to `/users/me/`, so a session
|
||||
written straight to the store covers HTTP, WS and the HTTP fallback.
|
||||
|
||||
How it is kept out of production:
|
||||
|
||||
- `LOAD_TEST_TOOLS_ENABLED` is `False` in `Base`, pinned to `False` again in
|
||||
`Production` (which `Feature`, `Staging`, `PreProduction` and `Demo` inherit),
|
||||
and is not read from the environment: no variable can turn it on.
|
||||
- `class LoadTest(Production)` is the only configuration that sets it to `True`
|
||||
and adds `loadtest` to `INSTALLED_APPS`. Everywhere else the commands do not
|
||||
exist (`Unknown command`).
|
||||
- The application refuses to load when the setting is off, so adding it to
|
||||
another configuration stops the process from starting.
|
||||
- The commands check the setting again, and refuse the `Production`
|
||||
configuration by name.
|
||||
|
||||
Usage, in preprod: deploy the backend with `DJANGO_CONFIGURATION=LoadTest`,
|
||||
then run the commands through the chart's generic backend job (Argo CD
|
||||
`PostSync` hook, `backend.job.command`):
|
||||
|
||||
```bash
|
||||
python manage.py create_load_test_sessions 5000 --heaviest 50 \
|
||||
--storage-name campaign-1.json --ttl-hours 8
|
||||
python manage.py revoke_load_test_sessions --storage-name campaign-1.json
|
||||
```
|
||||
|
||||
- Users: active, not staff, not superuser, with at least one access to a live
|
||||
document. `--heaviest N` takes the N users holding the most accesses, the
|
||||
rest is a random draw. Staff are excluded because a minted session of theirs
|
||||
would open the admin.
|
||||
- Manifest (JSON): `cookie_name`, `expires_at`, `public_documents`, and per
|
||||
session `user_id`, `session_key`, `editable_documents`,
|
||||
`readonly_documents` (most recently updated first, `--documents-per-user`
|
||||
each). This is what the swarm and k6 read to open documents they are allowed
|
||||
to open.
|
||||
- The manifest holds live cookies and is a secret. It goes to a `0600` file
|
||||
(`--output`) or to a private object under `loadtest/` in the default storage
|
||||
(`--storage-name`), never to stdout or logs. The media route only serves
|
||||
`{document id}/attachments/…` keys, so that prefix cannot be fetched through
|
||||
`/media/`. The load generator needs read access to that object.
|
||||
- Revocation reads an index kept next to the sessions, so it works without the
|
||||
manifest and across several runs. Sessions live 12 h by default, 7 days at
|
||||
most.
|
||||
|
||||
Rules for the load clients:
|
||||
|
||||
- One distinct user per virtual client, otherwise the per-user DRF throttles
|
||||
(80/min on documents) distort results. Keep the throttles on, they are part
|
||||
of production behaviour.
|
||||
- Keep the cookie jar. `ForceSessionMiddleware` creates a 12 h session in
|
||||
Valkey for every request that carries none.
|
||||
- Team-based accesses are not listed in the manifest, only direct ones.
|
||||
- `documents/search/` refreshes the OIDC token when
|
||||
`OIDC_STORE_REFRESH_TOKEN` is on: minted sessions hold no token, so leave
|
||||
that endpoint out of the scenarios or keep the setting off in preprod.
|
||||
|
||||
### 0.4 Chart and infrastructure
|
||||
|
||||
- Done in the chart: the metrics of the backend, of yhub and of its worker
|
||||
are served behind a bearer token, and `ingressMetrics` publishes them on a
|
||||
dedicated host (`/metrics`, `/metrics/yhub`, `/metrics/yhub-worker`), to be
|
||||
filtered by address. Through an ingress each scrape reaches one replica at
|
||||
random: if the preprod Prometheus runs inside the cluster, scraping each pod
|
||||
directly (ServiceMonitor and PodMonitor carrying the token) gives better data
|
||||
with many replicas, and would be a chart addition.
|
||||
- Valkey, to ask the team that runs it (it is not managed from this
|
||||
repository): the exporter enabled on `valkey-docs` and `valkey-yhub`, their
|
||||
metrics in the same Prometheus, and the memory limit and `maxmemory-policy` of
|
||||
each instance in preprod and in production.
|
||||
- Postgres: pghero or `pg_stat_statements`, connection counts per database
|
||||
and per application name, replication and failover state.
|
||||
- ingress-nginx metrics: request rate, latency, active connections, per
|
||||
ingress.
|
||||
- Pod CPU, memory, restarts and throttling for every component, load
|
||||
generators included.
|
||||
- Preprod matches production on replicas, resource requests and limits,
|
||||
Postgres pooler, Valkey memory and eviction policy for `valkey-docs` and
|
||||
`valkey-yhub`, yhub worker replicas and `YHUB_TASK_CONCURRENCY`. If preprod
|
||||
differs from production, results are only relative.
|
||||
|
||||
### 0.5 Dashboards
|
||||
|
||||
**Implemented** (2026-09-22): `src/loadtest/dashboards/` (see its README),
|
||||
loaded into the Grafana of the dev cluster
|
||||
(`src/helm/env.d/dev/values.prometheus.yaml.gotmpl`, Grafana 13, Prometheus
|
||||
3.14, in place of the django-prometheus console). Every panel query was
|
||||
checked against live data from the backend, yhub, the swarm, k6 and the canary.
|
||||
|
||||
- users: `users.json` — the canary (page open, editor ready, keystroke to the
|
||||
other screen, failures by step), plus the swarm's connect and propagation
|
||||
and k6's latency and failures;
|
||||
- yhub: `collaboration.json` — sockets and rooms per replica, event-loop lag,
|
||||
auth duration and results, backend calls (duration, in flight, failures),
|
||||
compaction backlog and duration, seeds; then the swarm's view;
|
||||
- Django: `backend.json` — requests and latency by view, 5xx, calls to yhub
|
||||
and the converters, the psycopg pool, queries, Celery queue length; then
|
||||
k6's view. `django.json` is the community Django dashboard (17658) for the
|
||||
per-view detail;
|
||||
- Valkey: `valkey.json` — the two instances through the operator's
|
||||
`redis_exporter`: memory against `maxmemory`, evictions, commands and their
|
||||
latency, network, CPU, the yhub streams, replication, Sentinel. The streams
|
||||
row needs `--check-streams`, which the operator's exporter spec cannot pass
|
||||
(see the README);
|
||||
- Postgres: not here, the exporter belongs to the team running it. The README
|
||||
lists the queries the campaign needs from it.
|
||||
|
||||
Pod CPU and memory come from the cluster's cAdvisor, not from these boards:
|
||||
the backend exports no process metrics in multiprocess mode.
|
||||
|
||||
## 1. Tooling
|
||||
|
||||
- HTTP: k6 (scenarios, thresholds, Prometheus output). Replay the real
|
||||
page-open sequence: `config`, `users/me`, `documents/{id}`, tree, list,
|
||||
plus `media-auth`.
|
||||
- Collaboration: a Node swarm using the real client stack (`yjs`,
|
||||
`y-websocket`, `ws` with the cookie header, same options as the frontend).
|
||||
One process holds a few thousand sockets. Run it as a Job that goes through
|
||||
the ingress, on nodes that do not host the application.
|
||||
- Latency: writers stamp a timestamp into the doc, readers measure the
|
||||
propagation delay.
|
||||
- Convergence: compare state vectors across clients at the end.
|
||||
- Durability: compare against the admin `ydoc` endpoint.
|
||||
- k6 cannot reasonably speak the Yjs protocol, which is why this is a
|
||||
separate tool.
|
||||
- Playwright canaries: 3 to 5 browsers reusing the `auth.setup.ts` storage
|
||||
state and the helpers in `src/frontend/apps/e2e` (`createDoc`,
|
||||
`writeInEditor`). They measure the user-perceived time to open a document
|
||||
and start typing while the swarm applies load.
|
||||
- Workload model from production, not guesses:
|
||||
- `profile_volumetry` for the data shape;
|
||||
- access logs or PostHog for peak concurrent users, connections per
|
||||
document (long tail: mostly 1, a few at 50+), edit rate, reader/writer
|
||||
ratio.
|
||||
- Run at 1x, 2x and 5x, then ramp until the knee.
|
||||
|
||||
Location of the tooling: `src/loadtest/` (`swarm/` exists, see its README).
|
||||
|
||||
Note from the dev-stack check of the swarm: `create_load_test_sessions` writes
|
||||
the sessions where the `LoadTest` configuration keeps them. In preprod the
|
||||
server runs that same configuration; a server running another one reads its
|
||||
sessions elsewhere (the dev stack's `Development` uses Redis db 2, `LoadTest`
|
||||
db 0) and the cookies are worth nothing to it. yhub then admits the clients
|
||||
anonymously on public documents, which looks like success: check the `userid`
|
||||
in yhub's logs once before a campaign.
|
||||
|
||||
## 2. Scenarios, in order of value
|
||||
|
||||
1. **Migration.** Time the `migrate_documents` backfill at full volumetry:
|
||||
documents per second, growth of the yhub Postgres database, total
|
||||
duration, best `--concurrency`. Then test the `SOFT_MIGRATION` path under
|
||||
load: many users opening unmigrated documents at once. Each replica
|
||||
refuses seeds beyond 20 at once with a 503, so measure how many opens are
|
||||
refused and how long clients take to get in.
|
||||
2. **HTTP baseline against the current release.** Same dataset, same k6
|
||||
script. Catches regressions unrelated to yhub.
|
||||
3. **Connect ramp.** Find the connects-per-second ceiling per backend pod.
|
||||
`documents/{id}/` is not cheap: ancestor annotation, LinkTrace `exists()`
|
||||
and a possible INSERT.
|
||||
4. **Idle steady state.** Ramp to 10k or more open sockets. Measure memory
|
||||
per connection, the 20 s resync floor, Valkey pub/sub traffic.
|
||||
5. **Hot document.** 50 to 200 editors plus readers on one document.
|
||||
Awareness fan-out grows with the square of the number of clients.
|
||||
6. **Wide editing.** Thousands of documents with 1 to 3 typists each.
|
||||
Stresses stream to worker to Postgres, and the `content-updated` to
|
||||
indexer to `get_ydoc` feedback loop. Watch stream length and worker lag.
|
||||
7. **Reconnect storms.** Restart yhub (Argo-driven rollout), call
|
||||
`reset-connections`, trigger a Valkey failover and a Patroni failover (the
|
||||
Valkey one has to be run by the team that operates it).
|
||||
Clients return within 30 s of backoff plus 3 s of jitter, each costing 2
|
||||
to 3 Django calls and a document refetch. With no timeout in
|
||||
`backendFetch`, a slow Django means piled-up upgrades. This is the failure
|
||||
shape of the 2026-08-18, 2026-09-07 and 2026-09-14 outages.
|
||||
8. **WS blocked, forcing the HTTP fallback.** Every poll re-runs the Django
|
||||
permission calls: N clients generate roughly 0.3 N requests per second on
|
||||
the backend. Probably the worst amplification in the system.
|
||||
9. **Heavy synchronous endpoints.** Duplicate with descendants (holds a
|
||||
Postgres connection in a transaction during N yhub round-trips),
|
||||
`formatted-content`, import, and delete, restore and access changes on deep
|
||||
trees (one HTTP call per node on a single Celery queue with 1 worker).
|
||||
10. **Large and long-history documents.** Open the 100 largest documents
|
||||
after migration and measure time to first sync.
|
||||
11. **Soak.** 4 to 8 h at 1x. Watch memory leaks, Valkey memory growth,
|
||||
Postgres bloat in the yhub database, uvicorn worker recycling.
|
||||
12. **Offline replay burst (browser only).** A fleet coming back online
|
||||
replays queued mutations at once. Lower priority.
|
||||
|
||||
## 3. Method
|
||||
|
||||
- Define SLOs first. Example targets: document open p95 under 2 s, edit
|
||||
propagation p95 under 500 ms, errors under 0.1 %, zero divergence, zero
|
||||
lost updates.
|
||||
- One scenario at a time, stepped ramps with plateaus.
|
||||
- At each knee, record which resource saturated (CPU, Postgres connections,
|
||||
Valkey memory, Node event loop), fix it, rerun.
|
||||
- Keep each run's parameters, dashboards snapshot and results together so
|
||||
runs are comparable.
|
||||
|
||||
## 4. Hypotheses to verify
|
||||
|
||||
Suspected weak points from the code read, none confirmed by measurement:
|
||||
|
||||
- 2 to 3 uncached Django calls per WS connect and per fallback poll.
|
||||
- No timeout or retry in yhub `backendFetch`.
|
||||
- Django to yhub client without connection reuse, 30 s timeout, called inside
|
||||
`transaction.atomic` in `duplicate`.
|
||||
- `content-updated` to indexer to `get_ydoc` feedback loop under wide editing.
|
||||
- Single Celery queue with 1 replica for subtree cascades.
|
||||
- Limit of 20 concurrent seeds per replica during soft migration, refused
|
||||
with a 503 beyond it, inside the upgrade handler.
|
||||
- `migrate_documents` default concurrency of 2 making the backfill very long.
|
||||
- Valkey memory: non-evictable stream entries against a small limit.
|
||||
- No database connection persistence or pooling by default.
|
||||
- No HPA and no resource requests, so scheduling and scaling are untested.
|
||||
- `SyncManager` replay without backoff or jitter.
|
||||
- JWKS fetch inside request authentication on cache miss (10 s timeout).
|
||||
|
||||
## 4b. Findings so far
|
||||
|
||||
Found while checking the tooling against the dev stack, before any campaign:
|
||||
|
||||
- **Concurrent `duplicate` calls collide on treebeard paths** (2026-09-22).
|
||||
Three users duplicating their own document at once, in `heavy.js`, get a
|
||||
500 (`IntegrityError: duplicate key value violates unique constraint
|
||||
"impress_document_path_key"`) or a 400 (`Document with this Path already
|
||||
exists`) on about a request out of seven. Root path allocation races between
|
||||
requests. To fix before the campaign, or scenario 9 measures the bug rather
|
||||
than the endpoint.
|
||||
|
||||
## 5. Work to do in this repository
|
||||
|
||||
1. Done: metrics in the backend (`/metrics`) and in `src/yhub-server`, Sentry
|
||||
in yhub, and the dedicated `ingressMetrics` in the chart (sections 0.1 and
|
||||
0.2).
|
||||
2. Done: the `loadtest` application and the `LoadTest` configuration, minting
|
||||
sessions for synthetic clients (section 0.3).
|
||||
3. Load generators reading the manifest of section 0.3, under `src/loadtest/`
|
||||
(not `src/backend/loadtest/`, which is the Django application):
|
||||
- Done (2026-09-22): the Node Yjs swarm, `src/loadtest/swarm/` (see its
|
||||
README). Modes `idle`, `hot`, `wide`, a ramp, a reconnect storm,
|
||||
propagation latency, convergence check, `/metrics`, a JSON report, a
|
||||
Dockerfile. Checked against the dev stack with sessions minted by
|
||||
`create_load_test_sessions`.
|
||||
- Done (2026-09-22): the k6 scripts, `src/loadtest/k6/` (see its README):
|
||||
`page-open.js`, the HTTP baseline (scenario 2) as a rate-based ramp of the
|
||||
frontend's document-open sequence plus `media-auth`; `heavy.js`, the
|
||||
synchronous endpoints (scenario 9): duplicate with descendants,
|
||||
`formatted-content`, delete/restore cascades, creation from a file. Both
|
||||
checked against the dev stack.
|
||||
- Done (2026-09-22): the canaries, `src/loadtest/canary/` (see its README).
|
||||
Real Chromium browsers in writer/reader pairs, through the real frontend
|
||||
and editor: page open, editor ready (provider synced), keystroke to the
|
||||
other screen, failures by step, on `/metrics` and in a JSON report.
|
||||
Checked against the dev stack: open p50 0.5 s, editor ready 1.3 s,
|
||||
propagation 70 ms with nothing else running.
|
||||
- Not done: the durability check against the admin `ydoc` endpoint (it
|
||||
needs an admin JWT the swarm should not hold), and the HTTP fallback
|
||||
scenario (`@y/yhub-http-fallback` is not driven by the swarm).
|
||||
4. To do: preprod helm values — `DJANGO_CONFIGURATION=LoadTest`, metrics
|
||||
enabled on both services, resources and replicas matching production.
|
||||
Outside of this repository: the Valkey exporter and sizing, with the team
|
||||
that runs Valkey.
|
||||
5. Done, from section 0.2: the custom backend metrics (outgoing calls,
|
||||
psycopg pool, Celery queue length). Left: Celery task duration.
|
||||
@@ -12,8 +12,7 @@ editor and the real collaboration path, and times what a user would notice:
|
||||
| `canary_iterations_total{result}`, `canary_failures_total{step}` | Iterations, and the step the failed ones died at (`page-open`, `editor-ready`, `focus`, `propagation`) |
|
||||
| `canary_console_errors_total` | Errors the pages logged |
|
||||
|
||||
It is the "users" board of the plan (`documentation/stress-test-plan.md`,
|
||||
section 0.5): the swarm and k6 tell what the servers do, the canary tells what
|
||||
It is the "users" board of `documentation/load-testing.md`: the swarm and k6 tell what the servers do, the canary tells what
|
||||
it feels like. It is not a load generator — a few pairs of browsers, nothing
|
||||
more — and it is the collaboration half's counterpart of the e2e tests, built
|
||||
with the Playwright library rather than its test runner so that it can run for
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
|
||||
Grafana dashboards over the metrics of the campaign: the backend's and yhub's
|
||||
`/metrics`, and what the load generators export (`../swarm`, `../k6`,
|
||||
`../canary`). One board per question of `documentation/stress-test-plan.md`
|
||||
(section 0.5): what the users feel, what the collaboration server does, what
|
||||
`../canary`). One board per question of `documentation/load-testing.md`: what the users feel, what the collaboration server does, what
|
||||
the backend does. Each is laid out so that a saturation reads left to right:
|
||||
the symptom the clients see, the server-side cause, the resource that ran out.
|
||||
|
||||
@@ -55,9 +54,9 @@ with (`backend`). Take a new revision from grafana.com the same way.
|
||||
|
||||
## Stores
|
||||
|
||||
Valkey is `valkey.json`. The team running it (chideat/valkey-operator) enables
|
||||
the operator's exporter, which is `redis_exporter` as a sidecar of every valkey
|
||||
pod, so the board has one target per pod: the `job` variable picks the
|
||||
Valkey is `valkey.json`, built for `redis_exporter` as a sidecar of every
|
||||
valkey pod (what chideat/valkey-operator runs, and what the dev cluster's chart
|
||||
runs), so the board has one target per pod: the `job` variable picks the
|
||||
instance, `instance` the pod. Two things about that exporter:
|
||||
|
||||
- the operator's `exporter` spec sets an image, resources and a security
|
||||
@@ -72,13 +71,13 @@ instance, `instance` the pod. Two things about that exporter:
|
||||
against valkey-yhub. The dev cluster's chart can pass the variable, and does;
|
||||
- the Sentinel row needs the sentinel pods scraped as well.
|
||||
|
||||
Postgres is not here: that exporter belongs to the team running it. What the
|
||||
Postgres is not here: use the board of your `postgres_exporter`. What a
|
||||
campaign needs from it, per instance:
|
||||
|
||||
- Postgres (`postgres_exporter` or pghero): connections by state and by
|
||||
application name, transactions and tuples per second, the slowest queries
|
||||
(`pg_stat_statements` by total and mean time), replication lag, and the
|
||||
Patroni leader. On the `yhub` database as well as the backend's.
|
||||
leader of the cluster. On the `yhub` database as well as the backend's.
|
||||
|
||||
## Editing
|
||||
|
||||
|
||||
@@ -470,7 +470,7 @@
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Requests waiting for a connection",
|
||||
"description": "The application waiting for connections, before Postgres shows anything. What was missing in the 2026-08-18 and 2026-09-07 outages.",
|
||||
"description": "The application waiting for connections, before Postgres shows anything: a connection storm seen from the side that suffers it.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
|
||||
@@ -147,7 +147,7 @@ backend = dashboard("docs-loadtest-backend", "Docs load test — backend",
|
||||
panel("Outgoing calls rate", [target(f"sum(rate(docs_outgoing_request_duration_seconds_count{{{H}}}[$__rate_interval])) by (service, operation)", "{{service}} {{operation}}")], "ops"),
|
||||
row("Database pool (DB_PSYCOPG_POOL_ENABLED)"),
|
||||
panel("Requests waiting for a connection", [target(f"sum(docs_db_pool_requests_waiting{{{H}}}) by (hostname)", "{{hostname}}")], "short", stack=True,
|
||||
desc="The application waiting for connections, before Postgres shows anything. What was missing in the 2026-08-18 and 2026-09-07 outages."),
|
||||
desc="The application waiting for connections, before Postgres shows anything: a connection storm seen from the side that suffers it."),
|
||||
panel("Queued requests and time spent waiting", [target(f"sum(rate(docs_db_pool_requests_queued_total{{{H}}}[$__rate_interval]))", "queued/s"), target(f"sum(rate(docs_db_pool_requests_wait_seconds_total{{{H}}}[$__rate_interval]))", "wait s/s")], "short"),
|
||||
panel("Pool size and idle", [target(f"sum(docs_db_pool_size{{{H}}})", "size"), target(f"sum(docs_db_pool_available{{{H}}})", "idle")], "short"),
|
||||
panel("Connections opened to Postgres", [target(f"sum(rate(docs_db_pool_connections_total{{{H}}}[$__rate_interval]))", "opened/s"), target(f"sum(rate(docs_db_pool_connections_errors_total{{{H}}}[$__rate_interval]))", "errors/s")], "ops"),
|
||||
@@ -179,7 +179,7 @@ valkey = dashboard("docs-loadtest-valkey", "Docs load test — valkey",
|
||||
opts={"textMode": "name", "colorMode": "none", "graphMode": "none"},
|
||||
desc="master or slave, as INFO says it. A change is a failover."),
|
||||
panel("Uptime", [target(f"min(redis_uptime_in_seconds{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "s", kind="stat", w=6,
|
||||
desc="A reset is a restart: what wiped the sessions on 2026-09-07."),
|
||||
desc="A reset is a restart, and on valkey-docs a restart without persistence wipes the sessions."),
|
||||
panel("Connected clients", [target(f"sum(redis_connected_clients{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "short", w=6),
|
||||
row("Memory"),
|
||||
panel("Memory used", [target(f"max(redis_memory_used_bytes{{{J}}}) by (job, instance)", "{{job}} {{instance}} used"), target(f"max(redis_memory_max_bytes{{{J}}} > 0) by (job, instance)", "{{job}} {{instance}} max")], "bytes",
|
||||
@@ -220,7 +220,7 @@ valkey = dashboard("docs-loadtest-valkey", "Docs load test — valkey",
|
||||
panel("Sentinel: quorum", [target('min(redis_sentinel_master_ok_sentinels) by (job, master_name)', "{{job}} {{master_name}} sentinels ok"), target('min(redis_sentinel_master_ok_slaves) by (job, master_name)', "{{job}} {{master_name}} replicas ok"), target('min(redis_sentinel_master_ckquorum_status) by (job, master_name)', "{{job}} {{master_name}} quorum")], "short",
|
||||
desc="From the sentinel pods, when they are scraped too. Sentinels ok below the quorum, or quorum status 0, means no failover is possible."),
|
||||
panel("Sentinel: master status", [target('max(redis_sentinel_master_status) by (job, master_name, master_status)', "{{job}} {{master_name}} {{master_status}}")], "short",
|
||||
desc="ok, or s_down / o_down when the sentinels have lost the master: the 2026-09-14 failover, seen from their side."),
|
||||
desc="ok, or s_down / o_down when the sentinels have lost the master: a failover, seen from their side."),
|
||||
row("Persistence"),
|
||||
panel("Changes since last save", [target(f"max(redis_rdb_changes_since_last_save{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "short"),
|
||||
panel("Last fork", [target(f"max(redis_latest_fork_seconds{{{J}}}) by (job, instance)", "{{job}} {{instance}}")], "s",
|
||||
|
||||
@@ -184,7 +184,7 @@
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Uptime",
|
||||
"description": "A reset is a restart: what wiped the sessions on 2026-09-07.",
|
||||
"description": "A reset is a restart, and on valkey-docs a restart without persistence wipes the sessions.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
@@ -1384,7 +1384,7 @@
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Sentinel: master status",
|
||||
"description": "ok, or s_down / o_down when the sentinels have lost the master: the 2026-09-14 failover, seen from their side.",
|
||||
"description": "ok, or s_down / o_down when the sentinels have lost the master: a failover, seen from their side.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# k6 — HTTP load scenarios for the backend
|
||||
|
||||
The HTTP half of `documentation/stress-test-plan.md`: what a browser asks the
|
||||
The HTTP half of `documentation/load-testing.md`: what a browser asks the
|
||||
Django backend when a user opens a document, and the endpoints that call the
|
||||
collaboration server or walk a subtree inside one request. The websocket half is
|
||||
`../swarm`.
|
||||
@@ -56,7 +56,7 @@ with the same options: what differs is what the new architecture costs on the
|
||||
HTTP side.
|
||||
|
||||
The `media-auth` call names a file the bucket does not hold: the access check
|
||||
— the costly part, and what brought production down on 2026-08-18 — runs, and
|
||||
— the costly part, an access check over the document tree — runs, and
|
||||
the answer is a 403 once it has passed. That status is expected and not counted
|
||||
as a failure.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
* frontend does it — `config`, `users/me`, the document, its tree, the list of
|
||||
* the user's documents — plus the `media-auth` subrequest nginx makes when the
|
||||
* page loads an attachment. It is the HTTP baseline of the plan
|
||||
* (`documentation/stress-test-plan.md`, scenario 2): run it against the
|
||||
* (`documentation/load-testing.md`, scenario 2): run it against the
|
||||
* current release and against this branch, on the same data, with the same
|
||||
* options.
|
||||
*
|
||||
@@ -69,9 +69,9 @@ export default function () {
|
||||
expect(get(session, `documents/${doc}/tree/`, 'documents/{id}/tree/'), 'documents/{id}/tree/', 200);
|
||||
expect(get(session, 'documents/?page=1&ordering=-updated_at', 'documents/'), 'documents/', 200);
|
||||
// the auth subrequest nginx makes for `/media/{doc}/attachments/{file}`. The
|
||||
// file need not exist for the access check — the costly part, and what
|
||||
// brought production down on 2026-08-18 — to run: a file the bucket does
|
||||
// not hold answers 403 once that check has passed
|
||||
// file need not exist for the access check — the costly part, a query
|
||||
// over the document tree — to run: a file the bucket does not hold
|
||||
// answers 403 once that check has passed
|
||||
const original = `${__ENV.MEDIA_BASE_URL || 'https://docs.example.com'}/media/${doc}/attachments/00000000-0000-4000-8000-000000000000.png`;
|
||||
expect(
|
||||
get(session, 'documents/media-auth/', 'documents/media-auth/', {
|
||||
|
||||
@@ -5,7 +5,7 @@ the frontend uses — `yjs`, `y-websocket`'s `WebsocketProvider` with the same
|
||||
options, a `ws` socket carrying the session cookie and the origin a browser
|
||||
would send — and measures what a user would feel: time to connect, time to the
|
||||
first sync, and how long an edit takes to reach the other clients of the same
|
||||
document. It is the collaboration half of `documentation/stress-test-plan.md`;
|
||||
document. It is the collaboration half of `documentation/load-testing.md`;
|
||||
the HTTP half is k6.
|
||||
|
||||
It is a standalone package (its own `package.json` and `yarn.lock`, not a
|
||||
|
||||
@@ -49,7 +49,7 @@ const socketClass = (
|
||||
constructor(url: string, protocols?: string | string[]) {
|
||||
const options: SocketOptions = {
|
||||
headers: { cookie, origin },
|
||||
// a bad certificate on preprod is not what is being tested
|
||||
// a bad certificate on the instance under test is not what is being tested
|
||||
rejectUnauthorized: false,
|
||||
};
|
||||
super(url, protocols, options);
|
||||
|
||||
Reference in New Issue
Block a user