🐛(backend) replace document creation table locks with retry strategy

We have situation where the number of locks in the database can increase
dangerously creating deadlock situation. To remove this situation we
decided to change the strategy to manage document creation concurrency.
We decided to use a retry strategy, trying to create the document
multiple times while a usable path is found. To avoid having an
inifinite loop, we use a max_attempts counter configurable using the
setting TREEBEARD_PATH_COMPUTE_RETRY_MAX_ATTEMPTS

src/backend/core/models.py

@@ -19,7 +19,7 @@ from django.core.cache import cache
 from django.core.files.base import ContentFile
 from django.core.files.storage import default_storage
 from django.core.mail import send_mail
-from django.db import connection, models, transaction
+from django.db import models, transaction
 from django.db.models.functions import Left, Length
 from django.template.loader import render_to_string
 from django.utils import timezone
@@ -39,6 +39,7 @@ from core.choices import (
     RoleChoices,
     get_equivalent_link_definition,
 )
+from core.utils.treebeard import create_tree_node_with_retry
 from core.validators import sub_validator
 
 logger = getLogger(__name__)
@@ -265,8 +266,6 @@ class User(AbstractBaseUser, BaseModel, auth_models.PermissionsMixin):
         duplicate the sandbox document for the user
         """
         if settings.USER_ONBOARDING_SANDBOX_DOCUMENT:
-            # transaction.atomic is used in a context manager to avoid a transaction if
-            # the settings USER_ONBOARDING_SANDBOX_DOCUMENT is unused
             sandbox_id = settings.USER_ONBOARDING_SANDBOX_DOCUMENT
             try:
                 template_document = Document.objects.get(id=sandbox_id)
@@ -276,20 +275,15 @@ class User(AbstractBaseUser, BaseModel, auth_models.PermissionsMixin):
                     sandbox_id,
                 )
                 return
-
             with transaction.atomic():
-                # locks the table to ensure safe concurrent access
-                with connection.cursor() as cursor:
-                    cursor.execute(
-                        f'LOCK TABLE "{Document._meta.db_table}" '  # noqa: SLF001
-                        "IN SHARE ROW EXCLUSIVE MODE;"
+                sandbox_document = create_tree_node_with_retry(
+                    lambda: Document.add_root(
+                        title=template_document.title,
+                        content=template_document.content,
+                        attachments=template_document.attachments,
+                        duplicated_from=template_document,
+                        creator=self,
                     )
-                sandbox_document = Document.add_root(
-                    title=template_document.title,
-                    content=template_document.content,
-                    attachments=template_document.attachments,
-                    duplicated_from=template_document,
-                    creator=self,
                 )
 
                 DocumentAccess.objects.create(