From 558239a94324516496cecc3dea2a6362b47232c2 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Mon, 10 Aug 2026 11:58:29 +0200 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F(backend)=20seed=20the=20cont?= =?UTF-8?q?ent=20of=20the=20demo=20documents=20using=20yhub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit seed the content of the demo documents in the collaboration server: `create_demo` no longer writes it to the object storage, which nothing reads anymore, and fails with an explicit message when the collaboration server is not running rather than building a corpus of documents that would open empty --- CHANGELOG.md | 5 + .../demo/management/commands/create_demo.py | 112 ++++++++++++++++-- .../demo/tests/test_commands_create_demo.py | 56 ++++++++- 3 files changed, 163 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 656facc67..ea45d6915 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -168,6 +168,11 @@ and this project adheres to since 0.5.0, so the custom `get-ydoc` endpoint it used to need is gone. `create-ydoc` stays, no built-in offers what it does — a strict create, and content credited to the user rather than to the backend +- ♻️(backend) seed the content of the demo documents in the collaboration + server: `create_demo` no longer writes it to the object storage, which + nothing reads anymore, and fails with an explicit message when the + collaboration server is not running rather than building a corpus of + documents that would open empty - 🔥(backend) remove the unused `CollaborationService` - 💥(backend) remove the `documents/{id}/content/` endpoint - 💥(backend) remove the `documents/{id}/can-edit/` endpoint diff --git a/src/backend/demo/management/commands/create_demo.py b/src/backend/demo/management/commands/create_demo.py index e216edf94..ace042e2b 100644 --- a/src/backend/demo/management/commands/create_demo.py +++ b/src/backend/demo/management/commands/create_demo.py @@ -1,12 +1,12 @@ # ruff: noqa: S311, S106 """create_demo management command""" -import base64 import logging import math import random import time from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed from uuid import uuid4 from django import db @@ -17,6 +17,7 @@ import pycrdt from faker import Faker from core import models +from core.services.yhub_services import YHubError, YHubService from demo import defaults @@ -31,14 +32,101 @@ def random_true_with_probability(probability): return random.random() < probability -def get_ydoc_for_text(text): - """Return a ydoc from plain text for demo purposes.""" +# The collaboration server is a service of its own: seeding a corpus one +# document at a time would make the demo wait on the network for most of its +# run, so a few seeds are in flight at once. +SEED_CONCURRENCY = 10 + + +def create_block(kind, text, **attributes): + """ + Build a BlockNote block, inside the container the editor addresses it by. + + Every block lives in a `blockContainer` carrying its id and its colors: + that is the structure the editor writes, and the one the exports and the + search indexer read back. + """ + block = pycrdt.XmlElement( + kind, {"textAlignment": "left", **attributes}, [pycrdt.XmlText(text)] + ) + + return pycrdt.XmlElement( + "blockContainer", + {"id": str(uuid4()), "textColor": "default", "backgroundColor": "default"}, + [block], + ) + + +def create_section_blocks(writer): + """Return the blocks of one section: a title, some prose, sometimes a list.""" + blocks = [create_block("heading", writer.sentence(nb_words=4).rstrip("."), level=2)] + blocks += [ + create_block("paragraph", writer.paragraph(nb_sentences=random.randint(3, 8))) + for _ in range(random.randint(1, 3)) + ] + + if random_true_with_probability(0.4): + kind = random.choice(["bulletListItem", "numberedListItem"]) + blocks += [ + create_block(kind, writer.sentence(nb_words=random.randint(4, 10))) + for _ in range(random.randint(2, 5)) + ] + + if random_true_with_probability(0.2): + blocks.append(create_block("quote", writer.sentence(nb_words=12))) + + return blocks + + +def get_ydoc_for_document(title): + """ + Return the raw Yjs update of a document that reads like a real one. + + Faker writes the prose and the sections, in the structure BlockNote stores, + so a demo document is something to render, to export and to index rather + than the one line it used to be. A single language per document: the corpus + is multilingual, the documents are not. + """ + writer = fake[random.choice(fake.locales)] + + blocks = [create_block("heading", title, level=1)] + for _ in range(random.randint(2, 5)): + blocks.extend(create_section_blocks(writer)) + ydoc = pycrdt.Doc() - paragraph = pycrdt.XmlElement("p", {}, [pycrdt.XmlText(text)]) - fragment = pycrdt.XmlFragment([paragraph]) - ydoc["document-store"] = fragment - update = ydoc.get_update() - return base64.b64encode(update).decode("utf-8") + ydoc["document-store"] = pycrdt.XmlFragment( + [pycrdt.XmlElement("blockGroup", {}, blocks)] + ) + + return ydoc.get_update() + + +def seed_contents(stdout, contents): + """ + Seed the content of the demo documents in the collaboration server. + + It owns the content of the documents, so a demo document without content + there is an empty document — the object storage Django used to write it to + is not read by anything anymore. + """ + service = YHubService() + + with ThreadPoolExecutor(max_workers=SEED_CONCURRENCY) as pool: + seeds = { + pool.submit(service.create_ydoc, document, update): document + for document, update in contents + } + for seed in as_completed(seeds): + try: + seed.result() + except YHubError as err: + # nothing to fall back on: the demo would build a corpus of + # empty documents and look like it worked + raise CommandError( + f"Could not seed the content of document {seeds[seed].id}: {err}. " + "Is the collaboration server running?" + ) from err + stdout.write(".", ending="") class BulkQueue: @@ -150,6 +238,7 @@ def create_demo(stdout): users_ids = list(models.User.objects.values_list("id", flat=True)) with Timeit(stdout, "Creating documents"): + contents = [] for i in range(defaults.NB_OBJECTS["docs"]): # pylint: disable=protected-access key = models.Document._int2str(i) # noqa: SLF001 @@ -165,11 +254,16 @@ def create_demo(stdout): if random_true_with_probability(0.5) else random.choice(models.LinkReachChoices.values), ) - document.save_content(get_ydoc_for_text(f"Content for {title:s}")) + contents.append((document, get_ydoc_for_document(title))) queue.push(document) queue.flush() + # after the flush: a room seeded for a document the database ended up + # without would be content nothing points to + with Timeit(stdout, "Seeding document contents"): + seed_contents(stdout, contents) + with Timeit(stdout, "Creating docs accesses"): docs_ids = list(models.Document.objects.values_list("id", flat=True)) for doc_id in docs_ids: diff --git a/src/backend/demo/tests/test_commands_create_demo.py b/src/backend/demo/tests/test_commands_create_demo.py index 522333013..8c2e58566 100644 --- a/src/backend/demo/tests/test_commands_create_demo.py +++ b/src/backend/demo/tests/test_commands_create_demo.py @@ -3,15 +3,30 @@ from unittest import mock from django.core.management import call_command +from django.core.management.base import CommandError from django.test import override_settings import pytest from core import models +from core.services.yhub_services import ServiceUnavailableError, YHubService +from core.utils.yjs import yjs_to_text, yjs_to_xml pytestmark = pytest.mark.django_db +@pytest.fixture(name="collaboration_server", autouse=True) +def collaboration_server_fixture(): + """ + Take the content of the demo documents, as the collaboration server does. + + It owns the content now, so building the demo corpus calls it once per + document. + """ + with mock.patch.object(YHubService, "create_ydoc") as mock_create_ydoc: + yield mock_create_ydoc + + @mock.patch( "demo.defaults.NB_OBJECTS", { @@ -21,7 +36,7 @@ pytestmark = pytest.mark.django_db }, ) @override_settings(DEBUG=True) -def test_commands_create_demo(): +def test_commands_create_demo(collaboration_server): """The create_demo management command should create objects as expected.""" call_command("create_demo") @@ -29,6 +44,27 @@ def test_commands_create_demo(): assert models.Document.objects.count() >= 10 assert models.DocumentAccess.objects.count() > 10 + # every document was seeded with its content in the collaboration server, + # and nothing was written to the object storage + assert collaboration_server.call_count == 10 + seeded = {call.args[0].id for call in collaboration_server.call_args_list} + assert seeded == set(models.Document.objects.values_list("id", flat=True)) + for call in collaboration_server.call_args_list: + document, update = call.args + assert document.content is None + + # the structure BlockNote stores, so the editor opens a real document + xml = yjs_to_xml(update) + assert xml.startswith(" 3 + # a title, as the number BlockNote reads a heading level as + assert " len(document.title) + # assert dev users have doc accesses user = models.User.objects.get(email="impress@impress.world") assert models.DocumentAccess.objects.filter(user=user).exists() @@ -38,3 +74,21 @@ def test_commands_create_demo(): assert models.DocumentAccess.objects.filter(user=user).exists() user = models.User.objects.get(email="user.test@chromium.test") assert models.DocumentAccess.objects.filter(user=user).exists() + + +@mock.patch( + "demo.defaults.NB_OBJECTS", + {"users": 2, "docs": 2, "max_users_per_document": 1}, +) +@override_settings(DEBUG=True) +def test_commands_create_demo_without_collaboration_server(collaboration_server): + """ + A demo of empty documents is not a demo: the command should say what is wrong. + + Nothing else holds the content, so a failure to seed it cannot be shrugged + off as it could when Django still wrote it to its object storage. + """ + collaboration_server.side_effect = ServiceUnavailableError("yhub is unreachable") + + with pytest.raises(CommandError, match="Is the collaboration server running?"): + call_command("create_demo")