From f714c7fae3b8e958b82dc268aae345bb42894de1 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Thu, 20 Aug 2026 16:21:36 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(backend)=20profile=20api=20using=20dj?= =?UTF-8?q?ango-silk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have performance issue with the media_auth endpoint. We need to monitor and profile this endpoint to understand what is happening. We decided to install and configure django-silk. By default django-silk is not enabled and not accessible, once enabled migration must be run and the /silk/ endpoint will be accessible only using a super user. To have relecant profiles and monitor we must have a database comparable to a production one because we are facing a scaling issue. Unless a million documents probably nothing is happening, for this we added several management commands to help us. A first one to anonymize a database than can be reused without any risk to leak PII data, can be called using python manage.py anonymize_database with some mandatory options. An other command to generate a database profile, this command should be run on a production database, this profile can then be used by the generate_volumetry to populate a development database with same volumetry than describe in the profile. --- CHANGELOG.md | 1 + documentation/env.md | 5 + documentation/profiling.md | 144 ++++++ .../management/commands/anonymize_database.py | 289 +++++++++++ .../management/commands/generate_volumetry.py | 484 ++++++++++++++++++ .../management/commands/profile_volumetry.py | 212 ++++++++ .../commands/purge_silk_profiles.py | 128 +++++ src/backend/impress/settings.py | 71 +++ src/backend/impress/urls.py | 6 + src/backend/pyproject.toml | 1 + src/backend/uv.lock | 25 + 11 files changed, 1366 insertions(+) create mode 100644 documentation/profiling.md create mode 100644 src/backend/core/management/commands/anonymize_database.py create mode 100644 src/backend/core/management/commands/generate_volumetry.py create mode 100644 src/backend/core/management/commands/profile_volumetry.py create mode 100644 src/backend/core/management/commands/purge_silk_profiles.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 53dd6ff37..0b5978dfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to - ♿️(frontend) restore skip to content link after header redesign #2510 - 🌐(i18n) rename cn_CN to zh_CN, add eo_PL and zh_TW locales #2486 - ✨(backend) conditional email notification in server to server api #2554 +- ✨(backend) profile api using django-silk ### Changed diff --git a/documentation/env.md b/documentation/env.md index 6da1e30e8..b13829922 100644 --- a/documentation/env.md +++ b/documentation/env.md @@ -133,6 +133,11 @@ These are the environment variables you can set for the `impress-backend` contai | SENTRY_DSN | Sentry host | | | SESSION_COOKIE_AGE | duration of the cookie session | 60*60*12 | | SIGNUP_NEW_USER_TO_MARKETING_EMAIL | Register new user to the marketing onboarding. If True, see env LASUITE_MARKETING_* system | False | +| SILK_ENABLED | Enable the django-silk request/SQL/cProfile profiler and its /silk/ UI. OFF by default; never enable against production with real users. See documentation/profiling.md | False | +| SILK_PYTHON_PROFILER | When silk is enabled, run a cProfile on each intercepted request | True | +| SILK_PYTHON_PROFILER_BINARY | Also write each cProfile as a downloadable binary .prof (stored in S3 via the SILKY_STORAGE backend, not the pod filesystem) | True | +| SILK_INTERCEPT_PERCENT | Percentage of requests silk records (lower it under load) | 100 | +| SILK_MAX_RECORDED_REQUESTS | Ring-buffer size: oldest recorded requests are dropped past this | 10000 | | SPECTACULAR_SETTINGS_ENABLE_DJANGO_DEPLOY_CHECK | | false | | STORAGES_STATICFILES_BACKEND | | whitenoise.storage.CompressedManifestStaticFilesStorage | | THEME_CUSTOMIZATION_CACHE_TIMEOUT | Cache duration for the customization settings | 86400 | diff --git a/documentation/profiling.md b/documentation/profiling.md new file mode 100644 index 000000000..388120cef --- /dev/null +++ b/documentation/profiling.md @@ -0,0 +1,144 @@ +# Profiling the backend with django-silk + +The backend ships an **opt-in** profiler, [django-silk](https://github.com/jazzband/django-silk), +to answer questions like *"where does `media-auth` spend its time β€” Postgres, S3, +or Python?"* under a realistic data volume. It records, per HTTP request: + +- the **SQL** it ran, each query with its own duration and the Python stack that + issued it (so a slow `readable_per_se` `EXISTS` points straight at the line in + `viewsets.py`); +- a **cProfile** call graph of the request, downloadable as a binary `.prof`; +- silk's own overhead, so you can subtract it. + +It is **disabled by default and absent from every environment that does not opt +in, production included** β€” when `SILK_ENABLED` is unset, `silk` is not in +`INSTALLED_APPS`, its middleware is not installed, and `/silk/` is not routed. + +> ⚠️ **Never enable silk against production with real users.** It persists +> request metadata to the database. Enable it only on an isolated environment β€” +> local dev, or a throwaway staging database populated with synthetic volumetry +> (see [Reproducing production volumetry](#reproducing-production-volumetry)). +> Request and response **bodies are never stored** (`SILKY_MAX_*_BODY_SIZE = 0`), +> so document content, titles and emails cannot leak into the silk tables. + +## Enabling it + +Set `SILK_ENABLED=1` in the backend environment and run silk's migrations once: + +```bash +# in the backend container / environment +export SILK_ENABLED=1 +python manage.py migrate silk +``` + +Then restart the backend so the middleware is picked up. The profiling UI is +served at **`/silk/`**, gated behind an authenticated **staff** session +(`SILKY_AUTHENTICATION` / `SILKY_AUTHORISATION`). Create one if needed: + +```bash +python manage.py createsuperuser +``` + +All knobs are environment variables (defaults in `documentation/env.md`): + +| Variable | Purpose | Default | +| --- | --- | --- | +| `SILK_ENABLED` | Master switch | `False` | +| `SILK_PYTHON_PROFILER` | cProfile each request | `True` | +| `SILK_PYTHON_PROFILER_BINARY` | Also emit a downloadable `.prof` | `True` | +| `SILK_INTERCEPT_PERCENT` | Sample only N % of requests | `100` | +| `SILK_MAX_RECORDED_REQUESTS` | Ring-buffer cap on stored requests | `10000` | + +Under load (a thundering-herd reproduction), lower `SILK_INTERCEPT_PERCENT` +(e.g. `5`) so silk records a representative sample without becoming the +bottleneck itself, and keep `SILK_MAX_RECORDED_REQUESTS` bounded so a long run +cannot fill the disk. + +## Analysing a request in the UI + +1. Reproduce the traffic (open a document, or run a load test β€” see below). +2. Open `/silk/`, sort **Requests** by *Time* or *Overall time*. +3. Click the slow request: + - **SQL** tab β€” every query with its duration and the *join / N+1* summary. + Click a query to see its `EXPLAIN`-free timing and the **stack trace** back + into our code. + - **Profile** tab β€” the cProfile breakdown (cumulative/total time per + function), the fastest way to see the Python↔DB↔S3 split. + +## Analysing the binary `.prof` offline + +With `SILK_PYTHON_PROFILER_BINARY=True`, each request's cProfile is also written +as a binary `.prof`, **through the Django storage framework** rather than the +local disk. It is stored via the `SILKY_STORAGE` entry of the `STORAGES` setting, +which points at the S3 backend under the `silk/` prefix β€” so profiling works on +read-only / ephemeral Kubernetes pods where there is no writable filesystem. + +Download the `.prof` from the request's **Profile** tab in the UI (silk serves it +back out of object storage), then open it with any standard tool: + +```bash +# interactive flame-ish call graph in the browser +pip install snakeviz && snakeviz .prof + +# or the stdlib, sorted by cumulative time +python -m pstats .prof <<< "sort cumulative +stats 30" +``` + +To point the binaries at a different bucket/backend, override the `SILKY_STORAGE` +entry in `STORAGES` (any Django storage backend works). + +## Cleaning up stored profiles + +Silk's ring-buffer (`SILK_MAX_RECORDED_REQUESTS`) prunes the request rows in the +database, but it does **not** delete the corresponding `.prof` objects, so they +accumulate in storage. Reclaim the space with: + +```bash +python manage.py purge_silk_profiles --dry-run # preview orphans +python manage.py purge_silk_profiles # delete orphaned profiles +python manage.py purge_silk_profiles --all # wipe every profile +python manage.py purge_silk_profiles --older-than 7 # only >7 days old +``` + +By default it deletes only **orphans** (profiles no longer referenced by any +`silk.Request` row), which is always safe. `--all` is required β€” and is the only +mode available β€” when silk is disabled, since there are then no rows to compare +against. Everything goes through the `SILKY_STORAGE` backend, so it works on +filesystem-less pods. + +## Profiling a code block directly (no HTTP) + +To profile a specific path without going through nginx/DRF β€” handy for the +`media-auth` authorization logic, which normally runs as an nginx subrequest β€” +wrap it with silk's context manager or decorator; it appears under **Profiling** +in the UI: + +```python +from silk.profiling.profiler import silk_profile + +with silk_profile(name="media_auth authorization"): + ... # the code under test +``` + +## Reproducing production volumetry + +Silk shows *where* time goes; it only tells the truth if the database has a +production-shaped volume. The incident class we chase (`media-auth` under a large +readable set) does not reproduce on an empty staging DB. Rebuild the shape from a +value-free profile β€” **without copying any production data** β€” then profile +against it: + +```bash +# 1. On a read replica: capture the shape (counts + distributions only, no PII) +python manage.py profile_volumetry --output volumetry/prod.json + +# 2. On the throwaway profiling DB: synthesize a same-shaped dataset +python manage.py generate_volumetry --profile volumetry/prod.json --scale 1.0 + +# 3. Enable silk, drive traffic (or the media-auth load tool), read /silk/ +``` + +Alternatively, anonymize a real production dump in an isolated database with +`anonymize_database` and profile against that. See those commands' `--help` for +details. diff --git a/src/backend/core/management/commands/anonymize_database.py b/src/backend/core/management/commands/anonymize_database.py new file mode 100644 index 000000000..507dc3f81 --- /dev/null +++ b/src/backend/core/management/commands/anonymize_database.py @@ -0,0 +1,289 @@ +"""anonymize_database β€” irreversibly scrub PII from a RESTORED production copy. + +Intended workflow for turning a production ``pg_dump`` into a shareable, +same-shaped staging database WITHOUT ever handling production personal data in +staging: + + 1. Restore the dump into an ISOLATED throwaway database (separate cluster / + namespace, no egress, credentials that cannot reach production). + 2. Point this command at that restored database and run it. + 3. Re-dump the now-anonymized database; that dump is safe to load anywhere. + +It overwrites β€” in place, irreversibly β€” every column that holds personal data +(emails, names, the OIDC ``sub``), user-generated content (document titles and +excerpts, comment bodies, thread/comment metadata) and secrets (password +hashes), and TRUNCATEs the operational tables that embed identity (sessions, +admin log, thumbnail cache). Structural columns that make +the data realistic for load testing β€” the tree ``path``/``depth``/``numchild``, +``link_reach``, roles, foreign keys, row counts, timestamps β€” are deliberately +preserved, so the result pairs naturally with ``profile_volumetry`` / +``generate_volumetry``. + +Anonymization is length-preserving where it can be (titles/excerpts become runs +of ``x`` of the same length) and keyed on each row's primary key where a value +must stay unique (emails, ``sub``), so unique constraints keep holding and the +scrub is deterministic and idempotent. + +SAFETY. Because it destroys data, it refuses to run unless BOTH of these hold: + * the environment variable ``DOCS_ALLOW_ANONYMIZATION=1`` is set (production + must never set it), and + * ``--yes`` is passed on the command line. +Use ``--expect-db NAME`` to assert the connected database's name before writing, +and ``--dry-run`` to print exactly what would run and touch nothing. It never +targets object storage β€” the S3 documents/attachments the dump references are +not in the dump and are handled separately. +""" + +import os + +from django.core.management.base import BaseCommand, CommandError +from django.db import connection, transaction + +ALLOW_ENV = "DOCS_ALLOW_ANONYMIZATION" + +# Field-level scrubs: (table, {column: sql_expression}). Each expression is +# evaluated per row and may reference that row's own columns (notably `id`, a +# UUID, used to keep anonymized unique values unique). Identifiers here are +# trusted constants from this file, never user input. +SCRUBS = [ + ( + "impress_user", + { + # Nullable identity columns: preserve NULLs (they matter to the + # profile's shape); otherwise replace with a per-row value that stays + # unique against sub / admin_email unique constraints. + "sub": "CASE WHEN sub IS NULL THEN NULL ELSE 'anon-' || id END", + "email": ( + "CASE WHEN email IS NULL THEN NULL " + "ELSE 'user-' || id || '@anon.invalid' END" + ), + "admin_email": ( + "CASE WHEN admin_email IS NULL THEN NULL " + "ELSE 'admin-' || id || '@anon.invalid' END" + ), + "full_name": ( + "CASE WHEN full_name IS NULL THEN NULL " + "ELSE 'User ' || left(id::text, 8) END" + ), + "short_name": "CASE WHEN short_name IS NULL THEN NULL ELSE 'User' END", + # Reset every password hash to an unusable placeholder. + "password": "'!'", + }, + ), + ( + "impress_user_reconciliation", + { + "active_email": "'active-' || id || '@anon.invalid'", + "inactive_email": "'inactive-' || id || '@anon.invalid'", + "source_unique_id": ( + "CASE WHEN source_unique_id IS NULL THEN NULL ELSE 'src-' || id END" + ), + "logs": "''", + # Invalidate any email-confirmation token that leaked via the dump. + "active_email_confirmation_id": "gen_random_uuid()", + "inactive_email_confirmation_id": "gen_random_uuid()", + }, + ), + ( + "impress_user_reconciliation_csv_import", + # `file` is the storage path of an uploaded CSV of real users; `logs` may + # quote their emails. The CSV object itself lives in storage, not the dump. + {"file": "''", "logs": "''"}, + ), + ( + "impress_document", + { + # Length-preserving: strip content, keep the size so search/list and + # index volumetry stay representative. attachments (UUID-based S3 + # keys) carry no PII and are kept for media-auth volumetry. + "title": "CASE WHEN title IS NULL THEN NULL " + "ELSE left(repeat('x', length(title)), 255) END", + "excerpt": "CASE WHEN excerpt IS NULL THEN NULL " + "ELSE left(repeat('x', length(excerpt)), 300) END", + }, + ), + ( + "impress_document_access", + # Team identifiers may name a real org/team. Pseudonymize deterministically + # so equal teams stay equal (preserving the unique(team, document) shape). + {"team": "CASE WHEN team = '' THEN '' ELSE 'team-' || left(md5(team), 10) END"}, + ), + ( + "impress_invitation", + # email is NOT NULL and unique per document; key on id to keep it unique. + {"email": "'invitee-' || id || '@anon.invalid'"}, + ), + ( + "impress_thread", + {"metadata": "'{}'::jsonb"}, + ), + ( + "impress_comment", + # body/metadata are free-form user content. + {"body": "'{}'::jsonb", "metadata": "'{}'::jsonb"}, + ), +] + +# Whole tables wiped: operational/transient data that embeds identity and is not +# needed for a staging clone. TRUNCATE ... CASCADE so FK-dependent rows go too. +TRUNCATES = [ + "django_session", # session_data can carry the authenticated identity + "django_admin_log", # object_repr / change_message leak titles and emails + "easy_thumbnails_source", # thumbnail cache: media file paths + "easy_thumbnails_thumbnail", +] + + +class Command(BaseCommand): + """Irreversibly anonymize the connected (restored) database in place.""" + + help = __doc__ + + def add_arguments(self, parser): + """Define command arguments.""" + parser.add_argument( + "--yes", + action="store_true", + help="Confirm the irreversible in-place scrub (required to write).", + ) + parser.add_argument( + "--expect-db", + default=None, + help="Abort unless the connected database has this exact NAME.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print every statement and table check; change nothing.", + ) + parser.add_argument( + "--report-unhandled", + action="store_true", + help="Also list text/json columns not covered here, for human review.", + ) + + def handle(self, *args, **options): + """Run the guarded anonymization pass.""" + db = connection.settings_dict + target = ( + f"{db.get('NAME')} on {db.get('HOST')}:{db.get('PORT')} as {db.get('USER')}" + ) + + if os.environ.get(ALLOW_ENV) != "1": + raise CommandError( + f"Refusing to run: {ALLOW_ENV} is not set to '1'. This command " + "destroys data and must only run against an ISOLATED restored copy. " + f"Set {ALLOW_ENV}=1 in that environment (production never should) " + "and retry." + ) + if options["expect_db"] and db.get("NAME") != options["expect_db"]: + raise CommandError( + f"Connected database is '{db.get('NAME')}', not " + f"'{options['expect_db']}' as asserted by --expect-db. Aborting." + ) + if not options["yes"] and not options["dry_run"]: + raise CommandError( + "Refusing to write without --yes. Re-run with --dry-run to preview, " + "or --yes to perform the irreversible scrub." + ) + + self.stdout.write(self.style.WARNING(f"Target database: {target}")) + if options["dry_run"]: + self.stdout.write(self.style.WARNING("DRY RUN β€” nothing will be written.")) + + with connection.cursor() as cursor: + if options["dry_run"]: + self._run(cursor, dry_run=True) + else: + # All-or-nothing: a failure rolls the whole scrub back. + with transaction.atomic(): + self._run(cursor, dry_run=False) + if options["report_unhandled"]: + self._report_unhandled(cursor) + + if not options["dry_run"]: + self.stdout.write( + self.style.SUCCESS( + "\nAnonymization complete. Run VACUUM ANALYZE, then re-dump; " + "the new dump is safe to load into staging." + ) + ) + + def _run(self, cursor, dry_run): + """Execute (or print) every scrub and truncate against existing tables.""" + for table, columns in SCRUBS: + present = self._existing_columns(cursor, table) + if present is None: + self.stdout.write(f" skip {table}: table absent") + continue + assignments = {c: e for c, e in columns.items() if c in present} + missing = set(columns) - set(assignments) + if missing: + self.stdout.write( + self.style.WARNING( + f" {table}: columns absent, not scrubbed: {sorted(missing)}" + ) + ) + if not assignments: + continue + set_clause = ", ".join( + f"{col} = {expr}" for col, expr in assignments.items() + ) + sql = f"UPDATE {table} SET {set_clause}" # noqa: S608 (trusted constants) + if dry_run: + self.stdout.write(f" would scrub {table}: {sorted(assignments)}") + continue + cursor.execute(sql) + self.stdout.write( + f" scrubbed {table}.{sorted(assignments)} ({cursor.rowcount} rows)" + ) + + for table in TRUNCATES: + if self._existing_columns(cursor, table) is None: + self.stdout.write(f" skip {table}: table absent") + continue + if dry_run: + self.stdout.write(f" would TRUNCATE {table} CASCADE") + continue + cursor.execute(f"TRUNCATE {table} CASCADE") + self.stdout.write(f" truncated {table}") + + @staticmethod + def _existing_columns(cursor, table): + """Return the set of column names of ``table``, or None if it is absent.""" + cursor.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = %s", + [table], + ) + rows = cursor.fetchall() + return {r[0] for r in rows} if rows else None + + def _report_unhandled(self, cursor): + """List text/json columns in public not covered here, for manual review. + + A coverage aid, not a guarantee: it flags every free-text-ish column that + is neither scrubbed nor in a wiped table, so an operator can confirm none + of them can hold personal data before trusting the dump. + """ + handled = {(t, c) for t, cols in SCRUBS for c in cols} + wiped = set(TRUNCATES) + cursor.execute( + "SELECT table_name, column_name, data_type FROM information_schema.columns " + "WHERE table_schema = 'public' " + "AND data_type IN ('character varying', 'text', 'json', 'jsonb', 'citext') " + "ORDER BY table_name, column_name" + ) + unhandled = [ + (t, c, d) + for t, c, d in cursor.fetchall() + if t not in wiped and (t, c) not in handled + ] + self.stdout.write( + self.style.WARNING( + f"\nCoverage review: {len(unhandled)} text/json column(s) are NOT " + "scrubbed or wiped. Confirm none can hold personal data:" + ) + ) + for table, column, dtype in unhandled: + self.stdout.write(f" {table}.{column} ({dtype})") diff --git a/src/backend/core/management/commands/generate_volumetry.py b/src/backend/core/management/commands/generate_volumetry.py new file mode 100644 index 000000000..7fe5134f4 --- /dev/null +++ b/src/backend/core/management/commands/generate_volumetry.py @@ -0,0 +1,484 @@ +# ruff: noqa: S311, S106 +"""generate_volumetry β€” rebuild a Docs database at a profiled scale. + +Takes a JSON profile produced by ``profile_volumetry`` and synthesises a dataset +with the same *shape* β€” tree depth and breadth, link_reach ratio, accesses per +user and per document, favorites, link traces, attachment keys β€” but with no +real content, titles or emails. The point is to reproduce, in staging, the exact +volumetry that makes incidents like the 2026-08-18 media-auth thundering herd +appear, without ever copying production data. + + python manage.py generate_volumetry --profile prod-2026-08.json --scale 1.0 + +What it reproduces, and which endpoint each driver feeds: + * tree depth + breadth -> tree / children / all / list root reduction + * link_reach ratio -> readable_per_se selectivity (media_auth, list) + * accesses per user/document -> get_queryset id-lists + user_roles annotation + * a single WORST-CASE user -> the large readable set the herd melted on + * favorites / link traces -> favorite_list / annotate_is_favorite / list + +It writes rows with ``bulk_create`` (no ``save()``, no S3, no ``full_clean``), +computing treebeard ``path``/``depth``/``numchild`` directly, exactly like the +``create_demo`` command. Attachment keys are synthetic strings; the S3 objects +they name need not exist β€” the media_auth DB queries run before ``head_object``. + +Run it against a throwaway staging database. It refuses a non-empty document +table (path collisions, skewed counts) and, like the demo command, refuses to +run outside DEBUG without ``--force``. +""" + +import json +import math +import random +from collections import defaultdict +from uuid import uuid4 + +from django import db +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from core import models + +# Emails/subs are namespaced so a run is trivially identifiable and removable. +DOMAIN = "volumetry.local" +WORST_CASE_EMAIL = f"worst.case@{DOMAIN}" + + +class BulkQueue: + """Create model instances in bulk by pushing to a per-model queue. + + Mirrors demo.create_demo's helper: it batches ``bulk_create`` and resets the + query cache to avoid the DEBUG-mode memory leak. ``ignore_conflicts`` lets + the relation tables tolerate the odd duplicate ``(user, document)`` pair. + """ + + BATCH_SIZE = 20000 + + def __init__(self, stdout, ignore_conflicts=False): + self.queue = defaultdict(list) + self.stdout = stdout + self.ignore_conflicts = ignore_conflicts + + def _bulk_create(self, objects): + """Insert one model's queued instances and clear the query cache.""" + if not objects: + return + objects[0]._meta.model.objects.bulk_create( # noqa: SLF001 + objects, ignore_conflicts=self.ignore_conflicts + ) + db.reset_queries() # DEBUG keeps a query cache -> memory leak at scale + self.queue[objects[0]._meta.model.__name__] = [] # noqa: SLF001 + + def push(self, obj): + """Queue an instance, flushing its model's batch once it is full.""" + objects = self.queue[obj._meta.model.__name__] # noqa: SLF001 + objects.append(obj) + if len(objects) > self.BATCH_SIZE: + self._bulk_create(objects) + self.stdout.write(".", ending="") + self.stdout.flush() + + def flush(self): + """Create every remaining queued instance across all models.""" + for objects in list(self.queue.values()): + self._bulk_create(objects) + + +class Picker: + """Weighted categorical sampler over a {value: weight} histogram. + + Precomputes a cumulative table so each draw is one ``random.random`` plus a + linear scan β€” fast enough to call once per document for millions of rows. + """ + + def __init__(self, histogram, fallback): + items = [(v, float(w)) for v, w in histogram.items() if float(w) > 0] + if not items: + items = [(fallback, 1.0)] + total = sum(w for _, w in items) + acc, self.cum = 0.0, [] + for value, weight in items: + acc += weight / total + self.cum.append((acc, value)) + + def draw(self): + """Return one value, sampled in proportion to its weight.""" + r = random.random() + for threshold, value in self.cum: + if r <= threshold: + return value + return self.cum[-1][1] + + +def _step(cls, n): + """The n-th (0-based) treebeard path segment: ``_int2str(n)`` left-padded.""" + key = cls._int2str(n) # pylint: disable=protected-access + return cls.alphabet[0] * (cls.steplen - len(key)) + key + + +def _build_forest(depth_histogram, scale): + """Return (paths, depths, numchildren) reproducing the depth histogram. + + Level 1 becomes the roots; every deeper level's nodes are scattered at + random over the previous level, which yields a natural breadth spread (many + leaves, a few high-fan-out parents) whose *totals per depth* match the + profile exactly. Breadth average therefore matches; the numchild tail is + approximated, which is immaterial to the endpoints' cost. + """ + cls = models.Document + levels = { + int(d): max(0, int(round(int(n) * scale))) for d, n in depth_histogram.items() + } + if levels.get(1, 0) == 0: + # A tree must have roots; seed at least one if the profile scaled to zero. + levels[1] = max(1, levels.get(1, 0)) + max_depth = max(levels) + + paths, depths, numchildren = [], [], [] + prev_paths, prev_index = [], [] # prev_index -> position in numchildren + + # Roots + for i in range(levels.get(1, 0)): + prev_paths.append(_step(cls, i)) + prev_index.append(len(paths)) + paths.append(prev_paths[-1]) + depths.append(1) + numchildren.append(0) + + for depth in range(2, max_depth + 1): + count = levels.get(depth, 0) + if count == 0 or not prev_paths: + prev_paths, prev_index = [], [] + continue + per_parent = [0] * len(prev_paths) + cur_paths, cur_index = [], [] + for _ in range(count): + pi = random.randrange(len(prev_paths)) + child_path = prev_paths[pi] + _step(cls, per_parent[pi]) + per_parent[pi] += 1 + cur_index.append(len(paths)) + cur_paths.append(child_path) + paths.append(child_path) + depths.append(depth) + numchildren.append(0) + for local_i, made in enumerate(per_parent): + numchildren[prev_index[local_i]] = made + prev_paths, prev_index = cur_paths, cur_index + + return paths, depths, numchildren + + +def _sample_count(mean, cap): + """A non-negative integer with expectation ~mean, capped at ``cap``.""" + if mean <= 0 or cap <= 0: + return 0 + value = int(round(random.gauss(mean, math.sqrt(mean)))) + return max(0, min(cap, value)) + + +class Command(BaseCommand): # pylint: disable=too-many-instance-attributes + """Synthesise a same-shaped Docs dataset from a volumetry profile.""" + + help = __doc__ + + def __init__(self, *args, **kwargs): + """Declare the run-scoped state shared across the generation steps.""" + super().__init__(*args, **kwargs) + self.profile = {} + self.counts = {} + self.scale = 1.0 + self.worst_email = WORST_CASE_EMAIL + self.docs_queue = None + self.rel_queue = None + self.user_ids = [] + self.doc_ids = [] + self.worst_id = None + self.n_users = 0 + self.n_docs = 0 + + def add_arguments(self, parser): + """Define command arguments.""" + parser.add_argument( + "-p", + "--profile", + required=True, + help="Path to a profile_volumetry JSON file.", + ) + parser.add_argument( + "--scale", + type=float, + default=1.0, + help="Multiply every count by this factor (default: 1.0 = production size).", + ) + parser.add_argument( + "--seed", + type=int, + default=0, + help="RNG seed for reproducible runs (default: 0).", + ) + parser.add_argument( + "--worst-case-email", + default=WORST_CASE_EMAIL, + help=f"Email of the seeded worst-case user (default: {WORST_CASE_EMAIL}).", + ) + parser.add_argument( + "--no-attachments", + action="store_true", + help="Skip attachment-key generation (media_auth load testing needs them).", + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + help="Run despite DEBUG=False and/or a non-empty document table.", + ) + + def handle(self, *args, **options): + """Read the profile and generate the dataset.""" + if not settings.DEBUG and not options["force"]: + raise CommandError( + "Refusing to run outside DEBUG. This writes a large synthetic " + "dataset; only run it on a throwaway staging DB, with --force." + ) + if models.Document.objects.exists() and not options["force"]: + raise CommandError( + "The document table is not empty. This command expects a fresh " + "database (path collisions, skewed counts otherwise). Reset the " + "staging database first, or pass --force to append anyway." + ) + + with open(options["profile"], encoding="utf-8") as handle: + self.profile = json.load(handle) + + self.scale = options["scale"] + self.counts = self.profile["counts"] + self.worst_email = options["worst_case_email"] + random.seed(options["seed"]) + + # Two queues: strict for the entities we author from scratch (a path bug + # must surface), tolerant for the relation tables (the odd duplicate + # (user, document) pair is expected and harmless). + self.docs_queue = BulkQueue(self.stdout) + self.rel_queue = BulkQueue(self.stdout, ignore_conflicts=True) + + self._create_users() + self._create_documents() + self._create_accesses() + self._grant_worst_case() + self._seed_pairs( + models.DocumentFavorite, + mean=self._mean("favorites_per_user", "favorites"), + label="favorites", + ) + self._seed_pairs( + models.LinkTrace, + mean=(self.counts["link_traces"] / max(1, self.n_users)), + label="link traces", + ) + if not options["no_attachments"]: + self._seed_attachments() + self._report() + + def _scaled(self, n): + """Scale a production count by --scale, never below zero.""" + return max(0, int(round(int(n) * self.scale))) + + def _create_users(self): + """Create the user population plus one designated worst-case user.""" + self.n_users = max(1, self._scaled(self.counts["users"])) + self.stdout.write(f"Creating {self.n_users} users", ending="") + for i in range(self.n_users): + self.docs_queue.push( + models.User( + sub=f"vol-{i:d}", + email=f"user.vol{i:d}@{DOMAIN}", + admin_email=f"user.vol{i:d}@{DOMAIN}", + password="!", + is_active=True, + is_first_connection=False, + ) + ) + worst_user = models.User( + sub="vol-worst-case", + email=self.worst_email, + admin_email=self.worst_email, + password="!", + is_active=True, + is_first_connection=False, + ) + self.docs_queue.push(worst_user) + self.docs_queue.flush() + self.stdout.write(" done") + self.user_ids = list(models.User.objects.values_list("id", flat=True)) + self.worst_id = worst_user.id + + def _create_documents(self): + """Insert the whole document forest with computed treebeard paths.""" + self.stdout.write("Building tree shape", ending="") + paths, depths, numchildren = _build_forest( + self.profile["depth_histogram"], self.scale + ) + self.stdout.write(f" -> {len(paths)} documents; inserting", ending="") + reach = Picker( + self.profile.get("link_reach", {}), models.LinkReachChoices.RESTRICTED + ) + link_role = Picker( + self.profile.get("link_role", {}), models.LinkRoleChoices.READER + ) + for path, depth, numchild in zip(paths, depths, numchildren, strict=True): + self.docs_queue.push( + models.Document( + id=uuid4(), + path=path, + depth=depth, + numchild=numchild, + title=f"doc-{path}", + link_reach=reach.draw(), + link_role=link_role.draw(), + creator_id=random.choice(self.user_ids), + ) + ) + self.docs_queue.flush() + self.stdout.write(" done") + del paths, depths, numchildren # free the in-memory forest before the joins + self.doc_ids = list(models.Document.objects.values_list("id", flat=True)) + self.n_docs = len(self.doc_ids) + + def _create_accesses(self): + """Create accesses: per-document count centred on the profiled mean. + + Distinct users per document (so no duplicate (user, document)); a slice + made team-based to match the observed team share; roles drawn from the + profiled mix. Per-user counts fall out ~Poisson, as in reality. + """ + counts = self.counts + mean_per_doc = ( + counts["accesses"] / counts["documents"] if counts["documents"] else 0 + ) + team_share = ( + counts["accesses_team_based"] / counts["accesses"] + if counts.get("accesses") + else 0 + ) + role = Picker( + self.profile.get("role_distribution", {}), models.RoleChoices.READER + ) + self.stdout.write("Creating accesses", ending="") + for doc_id in self.doc_ids: + k = _sample_count(mean_per_doc, cap=min(len(self.user_ids), 200)) + if not k: + continue + for uid in random.sample(self.user_ids, k): + if random.random() < team_share: + self.rel_queue.push( + models.DocumentAccess( + document_id=doc_id, + user=None, + team=f"team-{random.randint(0, max(1, self.n_users // 5)):d}", + role=role.draw(), + ) + ) + else: + self.rel_queue.push( + models.DocumentAccess( + document_id=doc_id, user_id=uid, role=role.draw() + ) + ) + self.rel_queue.flush() + self.stdout.write(" done") + + def _grant_worst_case(self): + """Give the worst-case user a deliberately large readable set. + + readable_per_se already hands this user every non-restricted document; + on top of that we give them direct OWNER access to a wide sample so the + role annotation and id-list paths are stressed as in the incident. + """ + worst_target = min( + self.n_docs, + self._scaled( + self.profile["worst_case_readable_per_se"]["max_accesses_per_user"] + ), + ) + if not worst_target: + return + self.stdout.write( + f"Granting the worst-case user ({self.worst_email}) direct access to " + f"{worst_target} documents", + ending="", + ) + for doc_id in random.sample(self.doc_ids, worst_target): + self.rel_queue.push( + models.DocumentAccess( + document_id=doc_id, + user_id=self.worst_id, + role=models.RoleChoices.OWNER, + ) + ) + self.rel_queue.flush() + self.stdout.write(" done") + + def _mean(self, dist_key, count_key): + """Prefer the profiled per-user mean; fall back to total/users.""" + dist = self.profile.get(dist_key) + if dist and dist.get("mean"): + return float(dist["mean"]) + return self.counts.get(count_key, 0) / (self.counts["users"] or 1) + + def _seed_pairs(self, model, mean, label): + """Create ~mean rows per user of a (user, document) relation model.""" + if mean <= 0 or not self.doc_ids: + return + self.stdout.write(f"Creating {label}", ending="") + for uid in self.user_ids: + k = _sample_count(mean, cap=min(len(self.doc_ids), 500)) + if not k: + continue + for doc_id in random.sample(self.doc_ids, k): + self.rel_queue.push(model(user_id=uid, document_id=doc_id)) + self.rel_queue.flush() + self.stdout.write(" done") + + def _seed_attachments(self): + """Stamp synthetic attachment keys onto a sample of documents. + + Keys follow the real ``/attachments/.`` shape so + the ``attachments @> [key]`` GIN lookup behaves as in production. The S3 + objects are never created; media_auth's queries precede head_object. + """ + att = self.profile.get("attachments", {}) + target_docs = self._scaled(att.get("documents_with_attachments", 0)) + if not target_docs: + return + max_per = max(1, int(att.get("max_per_document", 1))) + ids = list( + models.Document.objects.filter(attachments=[]).values_list("id", flat=True) + ) + random.shuffle(ids) + chosen = ids[:target_docs] + self.stdout.write(f"Stamping attachments on {len(chosen)} documents", ending="") + for i, doc_id in enumerate(chosen): + n = random.randint(1, max_per) + keys = [f"{doc_id!s}/attachments/{uuid4()!s}.pdf" for _ in range(n)] + models.Document.objects.filter(id=doc_id).update(attachments=keys) + if i % 5000 == 0: + db.reset_queries() # DEBUG query cache would grow over the loop + self.stdout.write(" done") + + def _report(self): + """Print a summary and how to drive load against the worst-case user.""" + self.stdout.write(self.style.SUCCESS("\nGeneration complete.")) + self.stdout.write( + f" scale {self.scale} -> {self.n_docs} documents, " + f"{models.User.objects.count()} users, " + f"{models.DocumentAccess.objects.count()} accesses" + ) + self.stdout.write( + " Run VACUUM ANALYZE before load-testing so the planner sees the new " + "volumetry (index vs seq-scan crossover depends on it)." + ) + self.stdout.write( + f" Worst-case user: {self.worst_email} β€” mint a session for it (e.g. " + "via shell force_login) and point data/logs/production/media_stress.py " + "at a document it can reach to reproduce the herd." + ) diff --git a/src/backend/core/management/commands/profile_volumetry.py b/src/backend/core/management/commands/profile_volumetry.py new file mode 100644 index 000000000..b500b2881 --- /dev/null +++ b/src/backend/core/management/commands/profile_volumetry.py @@ -0,0 +1,212 @@ +"""profile_volumetry β€” capture the statistical shape of a Docs database. + +Read-only. Emits a JSON profile of the row counts and distributions that drive +endpoint cost β€” tree depth and breadth, the link_reach ratio, accesses per user +and per document, attachments cardinality, favorites β€” so that +``generate_volumetry`` can rebuild a same-shaped dataset in staging *without +copying any production content*. + +Every incident of the 2026-08-18 media-auth class is impossible to reproduce in +staging because staging never has production's volumetry: too few non-restricted +documents, too shallow a tree, too small a "worst" user. This command measures +exactly those drivers as aggregates only β€” no rows, no PII leaves the database β€” +so the shape can be reproduced elsewhere. + +Run it against a READ REPLICA, never the primary. It performs a handful of +full-table aggregates (a few sequential scans / group-bys); harmless on a +replica, avoidable load on the leader. + + # print to stdout + python manage.py profile_volumetry + + # write through the default storage backend (S3 here), overwriting the key + python manage.py profile_volumetry --output volumetry/prod-2026-08.json + +The profile is intentionally value-free: it contains counts and quantiles, never +titles, emails, keys or contents. +""" + +import json +import sys + +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage +from django.core.management.base import BaseCommand +from django.db import connection +from django.db import models as django_models + +from core import models + +# Quantiles reported for every "per-something" distribution. p99 (and max) are +# what matter for cost: the worst user, not the average one, is what melted down. +_QUANTILES = (0.5, 0.9, 0.99) + + +def _dist(cursor, table, group_col, count_col="*"): + """Return {max, mean, p50, p90, p99} for the per-group row counts. + + Computes ``count(count_col)`` grouped by ``group_col`` and summarises the + resulting counts with Postgres ``percentile_disc``. All identifiers come + from Django model metadata, never user input. + """ + counted = "count(*)" if count_col == "*" else f'count("{count_col}")' + pct = ", ".join( + f"percentile_disc({q}) within group (order by c)" for q in _QUANTILES + ) + cursor.execute( + f"select max(c), avg(c), {pct} from (" # noqa: S608 (identifiers are trusted metadata) + f' select "{group_col}" g, {counted} c from "{table}"' + f' where "{group_col}" is not null group by "{group_col}"' + f") s" + ) + row = cursor.fetchone() + if row is None or row[0] is None: + return {"max": 0, "mean": 0.0, "p50": 0, "p90": 0, "p99": 0} + max_, mean, p50, p90, p99 = row + return { + "max": int(max_), + "mean": round(float(mean), 2), + "p50": int(p50), + "p90": int(p90), + "p99": int(p99), + } + + +def _histogram(qs, field): + """Return {value: count} for a GROUP BY over one column, keys as strings.""" + rows = qs.values(field).annotate(n=django_models.Count("id")).order_by(field) + return {str(r[field]): r["n"] for r in rows} + + +def build_profile(): + """Collect every distribution the generator needs, as plain JSON data.""" + doc_qs = models.Document.objects.all() + doc_table = models.Document._meta.db_table # noqa: SLF001 + access_table = models.DocumentAccess._meta.db_table # noqa: SLF001 + fav_table = models.DocumentFavorite._meta.db_table # noqa: SLF001 + + non_restricted = doc_qs.exclude( + link_reach=models.LinkReachChoices.RESTRICTED + ).count() + + with connection.cursor() as cursor: + # Attachments: how many documents carry keys, and how many keys each. + cursor.execute( + f'select count(*) filter (where cardinality("attachments") > 0), ' # noqa: S608 + f' coalesce(sum(cardinality("attachments")), 0), ' + f' coalesce(max(cardinality("attachments")), 0) ' + f'from "{doc_table}"' + ) + att_docs, att_total, att_max = cursor.fetchone() + + accesses_per_user = _dist(cursor, access_table, "user_id") + accesses_per_document = _dist(cursor, access_table, "document_id") + favorites_per_user = _dist(cursor, fav_table, "user_id") + + team_accesses = models.DocumentAccess.objects.exclude( + django_models.Q(team__isnull=True) | django_models.Q(team="") + ).count() + total_accesses = models.DocumentAccess.objects.count() + + return { + # A note for whoever reads the file; the generator ignores it. + "_about": ( + "Aggregate shape of a Docs database (no PII). Feed to " + "`generate_volumetry --profile`. Quantiles are per-entity row counts." + ), + "counts": { + "users": models.User.objects.count(), + "documents": doc_qs.count(), + "documents_root": doc_qs.filter(depth=1).count(), + "accesses": total_accesses, + "accesses_team_based": team_accesses, + "favorites": models.DocumentFavorite.objects.count(), + "invitations": models.Invitation.objects.count(), + "link_traces": models.LinkTrace.objects.count(), + }, + "link_reach": _histogram(doc_qs, "link_reach"), + "link_role": _histogram(doc_qs, "link_role"), + # depth = len(path)/steplen, stored denormalised on the row; breadth is + # numchild. Together they fix the tree shape the generator rebuilds. + "depth_histogram": _histogram(doc_qs, "depth"), + "numchild_histogram": _histogram(doc_qs, "numchild"), + "role_distribution": _histogram(models.DocumentAccess.objects.all(), "role"), + "accesses_per_user": accesses_per_user, + "accesses_per_document": accesses_per_document, + "favorites_per_user": favorites_per_user, + "attachments": { + "documents_with_attachments": att_docs, + "keys_total": int(att_total), + "max_per_document": int(att_max), + }, + "deleted": { + "soft_deleted": doc_qs.filter(deleted_at__isnull=False).count(), + "ancestor_deleted": doc_qs.filter( + ancestors_deleted_at__isnull=False + ).count(), + }, + # The crux number for the media-auth class of incident: the size of the + # largest readable-per-se set a single user can hold is bounded below by + # the non-restricted document count (everyone sees those) plus that + # user's direct accesses. The generator seeds one worst-case user to + # this scale so the pathology actually reproduces. + "worst_case_readable_per_se": { + "non_restricted_documents": non_restricted, + "max_accesses_per_user": accesses_per_user["max"], + "estimated": non_restricted + accesses_per_user["max"], + }, + } + + +class Command(BaseCommand): + """Emit a JSON volumetry profile of the current database (read-only).""" + + help = __doc__ + + def add_arguments(self, parser): + """Define command arguments.""" + parser.add_argument( + "-o", + "--output", + type=str, + default=None, + help=( + "Storage key to write the profile to via Django's default storage " + "backend, e.g. volumetry/prod.json (default: print to stdout)." + ), + ) + parser.add_argument( + "--indent", + type=int, + default=2, + help="JSON indentation (default: 2).", + ) + + def handle(self, *args, **options): + """Build the profile and write it out.""" + profile = build_profile() + text = json.dumps(profile, indent=options["indent"], sort_keys=True) + "\n" + + output = options["output"] + if not output: + sys.stdout.write(text) + return + + # Persist through Django's storage framework (the configured default + # storage β€” S3 in this project) so the artifact lands in durable, + # config-driven storage rather than an ephemeral pod filesystem when the + # profiler runs against a production/staging replica. + if default_storage.exists(output): + # save() would otherwise write to a suffixed key; we want the path + # the caller asked for, overwriting any earlier profile there. + default_storage.delete(output) + name = default_storage.save(output, ContentFile(text.encode("utf-8"))) + + self.stderr.write( + self.style.SUCCESS( + f"Wrote profile to storage key '{name}': " + f"{profile['counts']['documents']} documents, " + f"{profile['counts']['users']} users, worst-case readable set " + f"~{profile['worst_case_readable_per_se']['estimated']}." + ) + ) diff --git a/src/backend/core/management/commands/purge_silk_profiles.py b/src/backend/core/management/commands/purge_silk_profiles.py new file mode 100644 index 000000000..cee8a605f --- /dev/null +++ b/src/backend/core/management/commands/purge_silk_profiles.py @@ -0,0 +1,128 @@ +"""purge_silk_profiles β€” delete django-silk ``.prof`` binaries from object storage. + +django-silk writes a binary cProfile per intercepted request through the +``SILKY_STORAGE`` backend (S3, under the ``silk/`` prefix β€” see settings). Its +ring-buffer (``SILK_MAX_RECORDED_REQUESTS``) prunes the *database* rows once the +cap is exceeded, but it does NOT delete the corresponding object-storage files, +so ``.prof`` binaries accumulate indefinitely. This command reclaims that space. + +By default it deletes only **orphans** β€” ``.prof`` objects no longer referenced +by any ``silk.Request`` row β€” which is always safe. ``--all`` blind-purges every +profile under the prefix (the only mode available when silk is disabled, since +there are then no DB rows to compare against). ``--older-than DAYS`` further +restricts deletion by object age. + +Everything goes through Django's storage API (``storages['SILKY_STORAGE']``), so +it works on read-only/ephemeral Kubernetes pods with no local filesystem. + + python manage.py purge_silk_profiles --dry-run # preview orphans + python manage.py purge_silk_profiles # delete orphans + python manage.py purge_silk_profiles --all # wipe every profile + python manage.py purge_silk_profiles --older-than 7 # only >7 days old +""" + +from django.apps import apps +from django.core.files.storage import storages +from django.core.files.storage.handler import InvalidStorageError +from django.core.management.base import BaseCommand, CommandError +from django.utils import timezone + +SILK_STORAGE_ALIAS = "SILKY_STORAGE" + + +class Command(BaseCommand): + """Delete orphaned (or all) django-silk profile binaries from storage.""" + + help = __doc__ + + def add_arguments(self, parser): + """Define command arguments.""" + parser.add_argument( + "--all", + action="store_true", + help="Delete every .prof under the prefix, not just orphaned ones.", + ) + parser.add_argument( + "--older-than", + type=int, + default=None, + metavar="DAYS", + help="Only delete profiles whose object is older than DAYS days.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="List what would be deleted; delete nothing.", + ) + + def handle(self, *args, **options): + """List the profile store and delete the targeted objects.""" + try: + storage = storages[SILK_STORAGE_ALIAS] + except InvalidStorageError as err: + raise CommandError( + f"No '{SILK_STORAGE_ALIAS}' entry in the STORAGES setting; " + "cannot locate the silk profile store." + ) from err + + purge_all = options["all"] + dry_run = options["dry_run"] + + referenced = set() if purge_all else self._referenced_prof_files() + + cutoff = None + if options["older_than"] is not None: + cutoff = timezone.now() - timezone.timedelta(days=options["older_than"]) + + try: + _dirs, files = storage.listdir("") + except FileNotFoundError: + files = [] + + profiles = [name for name in files if name.endswith(".prof")] + self.stdout.write(f"Found {len(profiles)} profile object(s) in storage.") + + deleted = kept = 0 + for name in profiles: + if not purge_all and name in referenced: + kept += 1 + continue + if cutoff is not None and storage.get_modified_time(name) >= cutoff: + kept += 1 + continue + if dry_run: + self.stdout.write(f" would delete {name}") + deleted += 1 + continue + storage.delete(name) + deleted += 1 + + verb = "would delete" if dry_run else "deleted" + self.stdout.write( + self.style.SUCCESS( + f"\n{verb} {deleted} profile(s); kept {kept} " + f"({'referenced/recent' if not purge_all else 'recent'})." + ) + ) + + def _referenced_prof_files(self): + """Return the set of prof_file names still referenced by silk.Request. + + Requires the silk app to be installed (SILK_ENABLED). Without it there + are no rows to compare against, so orphan detection is impossible and the + caller must use --all instead. + """ + if not apps.is_installed("silk"): + raise CommandError( + "The silk app is not installed (SILK_ENABLED is off), so orphaned " + "profiles cannot be identified. Re-run with --all to blind-purge " + "every profile under the prefix, or set SILK_ENABLED=1 to compute " + "orphans against the silk.Request table." + ) + + request_model = apps.get_model("silk", "Request") + return set( + request_model.objects.exclude(prof_file="").values_list( + "prof_file", flat=True + ) + ) diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index bce3ea815..dde7cd5ed 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -164,6 +164,15 @@ class Base(Configuration): environ_name="STORAGES_STATICFILES_BACKEND", ), }, + # django-silk looks up its binary cProfile (.prof) storage under this + # exact alias (see silk.models). Routing it through the S3 backend keeps + # profiling artifacts off the pod filesystem, which is read-only / + # ephemeral in Kubernetes; the `silk/` prefix isolates them in the + # bucket. Only used when SILK_ENABLED and the binary profiler are on. + "SILKY_STORAGE": { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": {"location": "silk"}, + }, } # Media @@ -1117,6 +1126,52 @@ class Base(Configuration): environ_prefix=None, ) + # -- Profiling (django-silk) --------------------------------------------- + # Opt-in request/SQL/cProfile profiler, OFF by default. Turn it on in a + # given environment with SILK_ENABLED=1 (typically a throwaway staging pod + # loaded via `generate_volumetry`, or local dev) to record, per request, + # the SQL it ran with timing + originating stack, and an optional cProfile + # you can download as a binary `.prof`. When enabled, `silk` is appended to + # INSTALLED_APPS, `SilkyMiddleware` is wired near the top of MIDDLEWARE, and + # the UI is served at /silk/ (see impress/urls.py and post_setup below). + # + # NEVER enable against production with real users: silk persists request + # metadata to the database. Request/response BODIES are deliberately never + # stored (the two MAX_*_BODY_SIZE = 0 below) so document content, titles and + # emails cannot leak into the silk tables β€” only method, path, headers-free + # metadata, SQL and timings are kept. + SILK_ENABLED = values.BooleanValue( + False, environ_name="SILK_ENABLED", environ_prefix=None + ) + # Per-request cProfile. Binary output lets you download a `.prof` and open + # it in snakeviz / pstats / tuna offline for a full call graph. The binary + # is written through the SILKY_STORAGE backend (S3, see STORAGES above), not + # the local filesystem, so it works on read-only/ephemeral pods. + SILKY_PYTHON_PROFILER = values.BooleanValue( + True, environ_name="SILK_PYTHON_PROFILER", environ_prefix=None + ) + SILKY_PYTHON_PROFILER_BINARY = values.BooleanValue( + True, environ_name="SILK_PYTHON_PROFILER_BINARY", environ_prefix=None + ) + # Under load, record only a sample of requests to bound silk's own overhead + # and storage (100 = every request; drop it for a thundering-herd repro). + SILKY_INTERCEPT_PERCENT = values.IntegerValue( + 100, environ_name="SILK_INTERCEPT_PERCENT", environ_prefix=None + ) + # Ring-buffer the stored requests so a long load run cannot fill the disk. + SILKY_MAX_RECORDED_REQUESTS = values.IntegerValue( + 10000, environ_name="SILK_MAX_RECORDED_REQUESTS", environ_prefix=None + ) + SILKY_MAX_RECORDED_REQUESTS_CHECK_PERCENT = 10 + # Record silk's own per-request overhead so you can subtract it. + SILKY_META = True + # Lock the /silk/ UI behind an authenticated staff session. + SILKY_AUTHENTICATION = True + SILKY_AUTHORISATION = True + # RGPD: never persist request/response bodies (0 bytes kept). + SILKY_MAX_REQUEST_BODY_SIZE = 0 + SILKY_MAX_RESPONSE_BODY_SIZE = 0 + # pylint: disable=invalid-name @property def ENVIRONMENT(self): @@ -1216,6 +1271,21 @@ class Base(Configuration): posthog.api_key = cls.POSTHOG_KEY posthog.host = cls.POSTHOG_HOST + if cls.SILK_ENABLED: + # Activate django-silk only when explicitly turned on for this + # environment. Appending here (rather than in INSTALLED_APPS) keeps + # silk absent from every environment that does not opt in, including + # production. Guards make re-entry (post_setup can run per subclass) + # idempotent. + if "silk" not in cls.INSTALLED_APPS: + cls.INSTALLED_APPS.append("silk") + # SilkyMiddleware must be high enough to time the whole request but + # after AuthenticationMiddleware so it can attribute request.user; + # process_response runs inner-to-outer, so index 1 (just after + # SecurityMiddleware) satisfies both. + if "silk.middleware.SilkyMiddleware" not in cls.MIDDLEWARE: + cls.MIDDLEWARE.insert(1, "silk.middleware.SilkyMiddleware") + class Build(Base): """Settings used when the application is built. @@ -1286,6 +1356,7 @@ class Development(Base): self.CONTENT_SECURITY_POLICY["EXCLUDE_URL_PREFIXES"] += [ f"/api/{self.API_VERSION}/swagger", f"/api/{self.API_VERSION}/redoc", + "/silk", ] diff --git a/src/backend/impress/urls.py b/src/backend/impress/urls.py index 2c5964d42..fd594dc61 100644 --- a/src/backend/impress/urls.py +++ b/src/backend/impress/urls.py @@ -17,6 +17,12 @@ urlpatterns = [ path("", include("core.urls")), ] +# Serve the django-silk profiling UI at /silk/ only when profiling is enabled +# for this environment (SILK_ENABLED=1). The view itself is further gated behind +# a staff session by SILKY_AUTHENTICATION / SILKY_AUTHORISATION. +if settings.SILK_ENABLED: + urlpatterns += [path("silk/", include("silk.urls", namespace="silk"))] + if settings.DEBUG: urlpatterns = ( urlpatterns diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index cfbf3d367..2a1e20d63 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "django-lasuite[all]==0.0.27", "django-parler==2.4", "django-redis==7.0.0", + "django-silk==5.5.2", "django-storages[s3]==1.14.6", "django-timezone-field>=5.1", "django<6.0.0", diff --git a/src/backend/uv.lock b/src/backend/uv.lock index 5db67ffad..f59a52a05 100644 --- a/src/backend/uv.lock +++ b/src/backend/uv.lock @@ -582,6 +582,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/9f/09cdb9a1eebe8533b02a7694ca787acfc1e4d93b5b6175ff99366d4e6d64/django_redis-7.0.0-py3-none-any.whl", hash = "sha256:4b23aa6e0cd0937bb1242e9a463809e6004de3ca2150f34e986306bb6220d688", size = 38932, upload-time = "2026-06-02T14:17:47.281Z" }, ] +[[package]] +name = "django-silk" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django" }, + { name = "gprof2dot" }, + { name = "sqlparse" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/f4/e5f50497db9a3e59121237e3f74d6b2a2e5fc09ae0d242a659f3e0ca33c6/django_silk-5.5.2.tar.gz", hash = "sha256:c488ee1eab763a6f3d0db09a969e7a91b637743f04159cd8d10141386157ec0d", size = 4498731, upload-time = "2026-08-13T04:25:18.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/da/48607c4a756df4a3d37752006824004fe6ecb5e0a2b34864c97a9903ea08/django_silk-5.5.2-py3-none-any.whl", hash = "sha256:6f11ae724699192bdda247513aa40160fd5e7e3de3083ec307b7151d801d1d51", size = 1945106, upload-time = "2026-08-13T04:25:29.316Z" }, +] + [[package]] name = "django-storages" version = "1.14.6" @@ -807,6 +821,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743, upload-time = "2026-04-02T21:22:49.108Z" }, ] +[[package]] +name = "gprof2dot" +version = "2025.4.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/fd/cad13fa1f7a463a607176432c4affa33ea162f02f58cc36de1d40d3e6b48/gprof2dot-2025.4.14.tar.gz", hash = "sha256:35743e2d2ca027bf48fa7cba37021aaf4a27beeae1ae8e05a50b55f1f921a6ce", size = 39536, upload-time = "2025-04-14T07:21:45.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/ed/89d760cb25279109b89eb52975a7b5479700d3114a2421ce735bfb2e7513/gprof2dot-2025.4.14-py3-none-any.whl", hash = "sha256:0742e4c0b4409a5e8777e739388a11e1ed3750be86895655312ea7c20bd0090e", size = 37555, upload-time = "2025-04-14T07:21:43.319Z" }, +] + [[package]] name = "griffelib" version = "2.0.2" @@ -942,6 +965,7 @@ dependencies = [ { name = "django-lasuite", extra = ["all"] }, { name = "django-parler" }, { name = "django-redis" }, + { name = "django-silk" }, { name = "django-storages", extra = ["s3"] }, { name = "django-timezone-field" }, { name = "django-treebeard" }, @@ -1013,6 +1037,7 @@ requires-dist = [ { name = "django-lasuite", extras = ["all"], specifier = "==0.0.27" }, { name = "django-parler", specifier = "==2.4" }, { name = "django-redis", specifier = "==7.0.0" }, + { name = "django-silk", specifier = "==5.5.2" }, { name = "django-storages", extras = ["s3"], specifier = "==1.14.6" }, { name = "django-test-migrations", marker = "extra == 'dev'", specifier = "==1.5.0" }, { name = "django-timezone-field", specifier = ">=5.1" },