♻️(backend) seed the content of the demo documents using yhub

seed the content of the demo documents in the collaboration
server: `create_demo` no longer writes it to the object storage, which
nothing reads anymore, and fails with an explicit message when the
collaboration server is not running rather than building a corpus of
documents that would open empty

CHANGELOG.md

@@ -168,6 +168,11 @@ and this project adheres to
   since 0.5.0, so the custom `get-ydoc` endpoint it used to need is gone.
   `create-ydoc` stays, no built-in offers what it does — a strict create, and
   content credited to the user rather than to the backend
+- ♻️(backend) seed the content of the demo documents in the collaboration
+  server: `create_demo` no longer writes it to the object storage, which
+  nothing reads anymore, and fails with an explicit message when the
+  collaboration server is not running rather than building a corpus of
+  documents that would open empty
 - 🔥(backend) remove the unused `CollaborationService`
 - 💥(backend) remove the `documents/{id}/content/` endpoint
 - 💥(backend) remove the `documents/{id}/can-edit/` endpoint

src/backend/demo/management/commands/create_demo.py

@@ -1,12 +1,12 @@
 # ruff: noqa: S311, S106
 """create_demo management command"""
 
-import base64
 import logging
 import math
 import random
 import time
 from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from uuid import uuid4
 
 from django import db
@@ -17,6 +17,7 @@ import pycrdt
 from faker import Faker
 
 from core import models
+from core.services.yhub_services import YHubError, YHubService
 
 from demo import defaults
 
@@ -31,14 +32,101 @@ def random_true_with_probability(probability):
     return random.random() < probability
 
 
-def get_ydoc_for_text(text):
-    """Return a ydoc from plain text for demo purposes."""
+# The collaboration server is a service of its own: seeding a corpus one
+# document at a time would make the demo wait on the network for most of its
+# run, so a few seeds are in flight at once.
+SEED_CONCURRENCY = 10
+
+
+def create_block(kind, text, **attributes):
+    """
+    Build a BlockNote block, inside the container the editor addresses it by.
+
+    Every block lives in a `blockContainer` carrying its id and its colors:
+    that is the structure the editor writes, and the one the exports and the
+    search indexer read back.
+    """
+    block = pycrdt.XmlElement(
+        kind, {"textAlignment": "left", **attributes}, [pycrdt.XmlText(text)]
+    )
+
+    return pycrdt.XmlElement(
+        "blockContainer",
+        {"id": str(uuid4()), "textColor": "default", "backgroundColor": "default"},
+        [block],
+    )
+
+
+def create_section_blocks(writer):
+    """Return the blocks of one section: a title, some prose, sometimes a list."""
+    blocks = [create_block("heading", writer.sentence(nb_words=4).rstrip("."), level=2)]
+    blocks += [
+        create_block("paragraph", writer.paragraph(nb_sentences=random.randint(3, 8)))
+        for _ in range(random.randint(1, 3))
+    ]
+
+    if random_true_with_probability(0.4):
+        kind = random.choice(["bulletListItem", "numberedListItem"])
+        blocks += [
+            create_block(kind, writer.sentence(nb_words=random.randint(4, 10)))
+            for _ in range(random.randint(2, 5))
+        ]
+
+    if random_true_with_probability(0.2):
+        blocks.append(create_block("quote", writer.sentence(nb_words=12)))
+
+    return blocks
+
+
+def get_ydoc_for_document(title):
+    """
+    Return the raw Yjs update of a document that reads like a real one.
+
+    Faker writes the prose and the sections, in the structure BlockNote stores,
+    so a demo document is something to render, to export and to index rather
+    than the one line it used to be. A single language per document: the corpus
+    is multilingual, the documents are not.
+    """
+    writer = fake[random.choice(fake.locales)]
+
+    blocks = [create_block("heading", title, level=1)]
+    for _ in range(random.randint(2, 5)):
+        blocks.extend(create_section_blocks(writer))
+
     ydoc = pycrdt.Doc()
-    paragraph = pycrdt.XmlElement("p", {}, [pycrdt.XmlText(text)])
-    fragment = pycrdt.XmlFragment([paragraph])
-    ydoc["document-store"] = fragment
-    update = ydoc.get_update()
-    return base64.b64encode(update).decode("utf-8")
+    ydoc["document-store"] = pycrdt.XmlFragment(
+        [pycrdt.XmlElement("blockGroup", {}, blocks)]
+    )
+
+    return ydoc.get_update()
+
+
+def seed_contents(stdout, contents):
+    """
+    Seed the content of the demo documents in the collaboration server.
+
+    It owns the content of the documents, so a demo document without content
+    there is an empty document — the object storage Django used to write it to
+    is not read by anything anymore.
+    """
+    service = YHubService()
+
+    with ThreadPoolExecutor(max_workers=SEED_CONCURRENCY) as pool:
+        seeds = {
+            pool.submit(service.create_ydoc, document, update): document
+            for document, update in contents
+        }
+        for seed in as_completed(seeds):
+            try:
+                seed.result()
+            except YHubError as err:
+                # nothing to fall back on: the demo would build a corpus of
+                # empty documents and look like it worked
+                raise CommandError(
+                    f"Could not seed the content of document {seeds[seed].id}: {err}. "
+                    "Is the collaboration server running?"
+                ) from err
+            stdout.write(".", ending="")
 
 
 class BulkQueue:
@@ -150,6 +238,7 @@ def create_demo(stdout):
     users_ids = list(models.User.objects.values_list("id", flat=True))
 
     with Timeit(stdout, "Creating documents"):
+        contents = []
         for i in range(defaults.NB_OBJECTS["docs"]):
             # pylint: disable=protected-access
             key = models.Document._int2str(i)  # noqa: SLF001
@@ -165,11 +254,16 @@ def create_demo(stdout):
                 if random_true_with_probability(0.5)
                 else random.choice(models.LinkReachChoices.values),
             )
-            document.save_content(get_ydoc_for_text(f"Content for {title:s}"))
+            contents.append((document, get_ydoc_for_document(title)))
             queue.push(document)
 
         queue.flush()
 
+    # after the flush: a room seeded for a document the database ended up
+    # without would be content nothing points to
+    with Timeit(stdout, "Seeding document contents"):
+        seed_contents(stdout, contents)
+
     with Timeit(stdout, "Creating docs accesses"):
         docs_ids = list(models.Document.objects.values_list("id", flat=True))
         for doc_id in docs_ids:

src/backend/demo/tests/test_commands_create_demo.py

@@ -3,15 +3,30 @@
 from unittest import mock
 
 from django.core.management import call_command
+from django.core.management.base import CommandError
 from django.test import override_settings
 
 import pytest
 
 from core import models
+from core.services.yhub_services import ServiceUnavailableError, YHubService
+from core.utils.yjs import yjs_to_text, yjs_to_xml
 
 pytestmark = pytest.mark.django_db
 
 
+@pytest.fixture(name="collaboration_server", autouse=True)
+def collaboration_server_fixture():
+    """
+    Take the content of the demo documents, as the collaboration server does.
+
+    It owns the content now, so building the demo corpus calls it once per
+    document.
+    """
+    with mock.patch.object(YHubService, "create_ydoc") as mock_create_ydoc:
+        yield mock_create_ydoc
+
+
 @mock.patch(
     "demo.defaults.NB_OBJECTS",
     {
@@ -21,7 +36,7 @@ pytestmark = pytest.mark.django_db
     },
 )
 @override_settings(DEBUG=True)
-def test_commands_create_demo():
+def test_commands_create_demo(collaboration_server):
     """The create_demo management command should create objects as expected."""
     call_command("create_demo")
 
@@ -29,6 +44,27 @@ def test_commands_create_demo():
     assert models.Document.objects.count() >= 10
     assert models.DocumentAccess.objects.count() > 10
 
+    # every document was seeded with its content in the collaboration server,
+    # and nothing was written to the object storage
+    assert collaboration_server.call_count == 10
+    seeded = {call.args[0].id for call in collaboration_server.call_args_list}
+    assert seeded == set(models.Document.objects.values_list("id", flat=True))
+    for call in collaboration_server.call_args_list:
+        document, update = call.args
+        assert document.content is None
+
+        # the structure BlockNote stores, so the editor opens a real document
+        xml = yjs_to_xml(update)
+        assert xml.startswith("<blockGroup><blockContainer")
+        assert xml.count("<blockContainer") > 3
+        # a title, as the number BlockNote reads a heading level as
+        assert "<heading" in xml and 'level="1"' in xml
+
+        # it opens on the title of the document, and says a lot more than it
+        text = yjs_to_text(update)
+        assert text.startswith(document.title)
+        assert len(text) > len(document.title)
+
     # assert dev users have doc accesses
     user = models.User.objects.get(email="impress@impress.world")
     assert models.DocumentAccess.objects.filter(user=user).exists()
@@ -38,3 +74,21 @@ def test_commands_create_demo():
     assert models.DocumentAccess.objects.filter(user=user).exists()
     user = models.User.objects.get(email="user.test@chromium.test")
     assert models.DocumentAccess.objects.filter(user=user).exists()
+
+
+@mock.patch(
+    "demo.defaults.NB_OBJECTS",
+    {"users": 2, "docs": 2, "max_users_per_document": 1},
+)
+@override_settings(DEBUG=True)
+def test_commands_create_demo_without_collaboration_server(collaboration_server):
+    """
+    A demo of empty documents is not a demo: the command should say what is wrong.
+
+    Nothing else holds the content, so a failure to seed it cannot be shrugged
+    off as it could when Django still wrote it to its object storage.
+    """
+    collaboration_server.side_effect = ServiceUnavailableError("yhub is unreachable")
+
+    with pytest.raises(CommandError, match="Is the collaboration server running?"):
+        call_command("create_demo")