mirror of
https://github.com/suitenumerique/docs.git
synced 2026-10-01 05:55:16 +02:00
✨(backend) expose /documents/import-zip/ API endpoint
POST /api/v1.0/documents/import-zip/ accepts a multipart ZIP file, and: - parses it with the zip_import service - uploads embedded media to S3 - converts each page from Markdown to YJS - persists the document tree. It then returns the count of imported pages and the list of root documents. Signed-off-by: Mathieu Agopian <mathieu@agopian.info>
This commit is contained in:
Mathieu Agopian
@@ -64,6 +64,7 @@ and this project adheres to
|
||||
offline
|
||||
- 🐛(frontend) stop the service worker from caching the collaboration server's
|
||||
rest api
|
||||
- ✨(backend) import documents from a ZIP file exported from Outline or Notion
|
||||
|
||||
### Changed
|
||||
|
||||
|
||||
@@ -5,10 +5,14 @@
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
import uuid
|
||||
import zipfile
|
||||
from collections import defaultdict
|
||||
from functools import partial
|
||||
from io import BytesIO
|
||||
from pathlib import PurePosixPath
|
||||
from urllib.parse import unquote, urlencode, urlparse
|
||||
|
||||
from django.conf import settings
|
||||
@@ -69,6 +73,7 @@ from core.services.search_indexers import (
|
||||
get_visited_document_ids_of,
|
||||
)
|
||||
from core.services.yhub_services import YHubError, YHubService
|
||||
from core.services.zip_import import parse_zip
|
||||
from core.tasks.access import reset_service_connections_in_cascade
|
||||
from core.tasks.documents import sync_service_deletions_in_cascade
|
||||
from core.tasks.mail import send_ask_for_access_mail
|
||||
@@ -436,6 +441,89 @@ class DocumentMetadata(drf.metadata.SimpleMetadata):
|
||||
return simple_metadata
|
||||
|
||||
|
||||
def _import_zip_create_node(node, user, converter, parent_doc=None):
|
||||
"""Create a Document (and its subtree) from a DocumentNode parsed out of a ZIP export.
|
||||
|
||||
Returns (doc, count) where count includes the node itself and all descendants.
|
||||
"""
|
||||
attachment_keys = []
|
||||
ref_to_url = {}
|
||||
for ref, media_bytes in node.media.items():
|
||||
ext = PurePosixPath(ref).suffix.lstrip(".")
|
||||
file_uuid = uuid.uuid4()
|
||||
key = f"{node.id}/{enums.ATTACHMENTS_FOLDER}/{file_uuid}.{ext}"
|
||||
default_storage.connection.meta.client.upload_fileobj(
|
||||
BytesIO(media_bytes),
|
||||
default_storage.bucket_name,
|
||||
key,
|
||||
ExtraArgs={"ContentType": f"image/{ext}"},
|
||||
)
|
||||
attachment_keys.append(key)
|
||||
ref_to_url[ref] = f"{settings.MEDIA_URL}{key}"
|
||||
|
||||
yjs_content = None
|
||||
if node.content is not None:
|
||||
text = node.content.decode("utf-8", errors="replace")
|
||||
for ref, url in ref_to_url.items():
|
||||
# Match the ref URL inside a markdown image/link, capturing
|
||||
# anything between the URL and the closing paren (e.g. an
|
||||
# Outline-style size hint: `" =1200x1600"`).
|
||||
text = re.sub(
|
||||
r"""
|
||||
\]\( # closing bracket + opening paren
|
||||
"""
|
||||
+ re.escape(ref)
|
||||
+ r"""
|
||||
([^)]*) # optional title / hint before closing paren
|
||||
\) # closing paren
|
||||
""",
|
||||
lambda m, _url=url: f"]({_url}{m.group(1)})",
|
||||
text,
|
||||
flags=re.VERBOSE,
|
||||
)
|
||||
try:
|
||||
yjs_content = converter.convert(
|
||||
text.encode("utf-8"),
|
||||
content_type=mime_types.MARKDOWN,
|
||||
accept=mime_types.YJS,
|
||||
)
|
||||
except (ConversionError, YProviderServiceUnavailableError) as err:
|
||||
logger.warning(
|
||||
"ZIP import: conversion failed for '%s': %s", node.title, err
|
||||
)
|
||||
|
||||
doc_kwargs = {"id": node.id, "title": node.title, "creator": user}
|
||||
if attachment_keys:
|
||||
doc_kwargs["attachments"] = attachment_keys
|
||||
|
||||
if parent_doc is None:
|
||||
doc = create_tree_node_with_retry(
|
||||
lambda _kw=doc_kwargs: models.Document.add_root(**_kw)
|
||||
)
|
||||
models.DocumentAccess.objects.create(
|
||||
document=doc, user=user, role=models.RoleChoices.OWNER
|
||||
)
|
||||
else:
|
||||
doc = create_tree_node_with_retry(
|
||||
lambda _parent=parent_doc, _kw=doc_kwargs: _parent.add_child(**_kw)
|
||||
)
|
||||
|
||||
if yjs_content is not None:
|
||||
try:
|
||||
YHubService(user=user).create_ydoc(doc, yjs_content)
|
||||
except YHubError as err:
|
||||
logger.warning(
|
||||
"ZIP import: could not seed content for '%s': %s", node.title, err
|
||||
)
|
||||
|
||||
count = 1
|
||||
for child in node.children:
|
||||
_, child_count = _import_zip_create_node(child, user, converter, doc)
|
||||
count += child_count
|
||||
|
||||
return doc, count
|
||||
|
||||
|
||||
# pylint: disable=too-many-public-methods
|
||||
class DocumentViewSet(
|
||||
SerializerPerActionMixin,
|
||||
@@ -1854,6 +1942,54 @@ class DocumentViewSet(
|
||||
status=drf.status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
@drf.decorators.action(detail=False, methods=["post"], url_path="import-zip")
|
||||
def import_zip(self, request, *args, **kwargs):
|
||||
"""Import a tree of documents from a ZIP export (Outline, Notion, ...)."""
|
||||
if not request.user.is_authenticated:
|
||||
raise drf.exceptions.NotAuthenticated()
|
||||
|
||||
if not settings.CONVERSION_UPLOAD_ENABLED:
|
||||
raise drf.exceptions.ValidationError({"zip": ["ZIP import is not allowed"]})
|
||||
|
||||
zip_file = request.data.get("zip")
|
||||
if not zip_file:
|
||||
raise drf.exceptions.ValidationError({"zip": ["This field is required."]})
|
||||
|
||||
try:
|
||||
zf = zipfile.ZipFile(BytesIO(zip_file.read()))
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise drf.exceptions.ValidationError(
|
||||
{"zip": ["Invalid ZIP file."]}
|
||||
) from exc
|
||||
|
||||
with zf:
|
||||
nodes = parse_zip(zf)
|
||||
|
||||
user = request.user
|
||||
converter = Converter()
|
||||
|
||||
root_docs = []
|
||||
pages = 0
|
||||
with transaction.atomic():
|
||||
for node in nodes:
|
||||
doc, count = _import_zip_create_node(node, user, converter)
|
||||
root_docs.append(doc)
|
||||
pages += count - 1 # root containers are not imported pages
|
||||
|
||||
posthog_capture(
|
||||
PosthogEventName.DOC_IMPORTED,
|
||||
user,
|
||||
{"format": "zip", "count": pages},
|
||||
)
|
||||
|
||||
return drf.response.Response(
|
||||
{
|
||||
"count": pages,
|
||||
"roots": [{"id": str(doc.id), "title": doc.title} for doc in root_docs],
|
||||
},
|
||||
status=drf.status.HTTP_201_CREATED,
|
||||
)
|
||||
|
||||
@drf.decorators.action(detail=True, methods=["post"], url_path="attachment-upload")
|
||||
def attachment_upload(self, request, *args, **kwargs):
|
||||
"""Upload a file related to a given document"""
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Tests for the POST /documents/import-zip/ endpoint."""
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.core.files.storage import default_storage
|
||||
|
||||
import pytest
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from core import factories, models
|
||||
from core.services.converter_services import ConversionError
|
||||
from core.utils.analytics import PosthogEventName
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
FIXTURES = Path(__file__).parent.parent / "fixtures"
|
||||
URL = "/api/v1.0/documents/import-zip/"
|
||||
YJS = "fakeyjs"
|
||||
|
||||
|
||||
def _make_zip(*entries):
|
||||
"""Return BytesIO of a zip containing the given (name, bytes) entries."""
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as zf:
|
||||
for name, content in entries:
|
||||
zf.writestr(name, content)
|
||||
buf.seek(0)
|
||||
return buf
|
||||
|
||||
|
||||
def test_api_documents_import_zip_anonymous():
|
||||
"""Anonymous users cannot import a ZIP."""
|
||||
response = APIClient().post(URL, {"zip": _make_zip()}, format="multipart")
|
||||
assert response.status_code == 401
|
||||
assert not models.Document.objects.exists()
|
||||
|
||||
|
||||
def test_api_documents_import_zip_disabled(settings):
|
||||
"""Returns 400 when CONVERSION_UPLOAD_ENABLED is False."""
|
||||
settings.CONVERSION_UPLOAD_ENABLED = False
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
response = client.post(URL, {"zip": _make_zip()}, format="multipart")
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == {"zip": ["ZIP import is not allowed"]}
|
||||
assert not models.Document.objects.exists()
|
||||
|
||||
|
||||
def test_api_documents_import_zip_missing_field(settings):
|
||||
"""Returns 400 when no zip field is provided."""
|
||||
settings.CONVERSION_UPLOAD_ENABLED = True
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
response = client.post(URL, {}, format="multipart")
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == {"zip": ["This field is required."]}
|
||||
|
||||
|
||||
def test_api_documents_import_zip_not_a_zip(settings):
|
||||
"""Returns 400 when the uploaded file is not a valid ZIP."""
|
||||
settings.CONVERSION_UPLOAD_ENABLED = True
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
not_a_zip = io.BytesIO(b"this is not a zip file")
|
||||
not_a_zip.name = "export.zip"
|
||||
|
||||
response = client.post(URL, {"zip": not_a_zip}, format="multipart")
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == {"zip": ["Invalid ZIP file."]}
|
||||
|
||||
|
||||
@patch("core.api.viewsets.YHubService")
|
||||
@patch("core.services.converter_services.Converter.convert")
|
||||
def test_api_documents_import_zip_success(mock_convert, mock_yhub, settings):
|
||||
"""201 with correct count and root list; documents and accesses created in DB."""
|
||||
settings.CONVERSION_UPLOAD_ENABLED = True
|
||||
mock_convert.return_value = YJS
|
||||
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
with patch("core.api.viewsets.posthog_capture") as mock_capture:
|
||||
with open(FIXTURES / "outline-export.zip", "rb") as f:
|
||||
response = client.post(URL, {"zip": f}, format="multipart")
|
||||
|
||||
assert response.status_code == 201, response.json()
|
||||
data = response.json()
|
||||
|
||||
# 4 children + 2 grandchildren; root "Welcome" is a container, not a page
|
||||
assert data["count"] == 6
|
||||
assert len(data["roots"]) == 1
|
||||
assert data["roots"][0]["title"] == "Welcome"
|
||||
|
||||
assert models.Document.objects.count() == 7
|
||||
|
||||
root = models.Document.objects.get(id=data["roots"][0]["id"])
|
||||
assert root.is_root()
|
||||
assert root.get_children().count() == 4
|
||||
assert models.DocumentAccess.objects.filter(
|
||||
document=root, user=user, role=models.RoleChoices.OWNER
|
||||
).exists()
|
||||
|
||||
# Only the root gets an explicit access record; children inherit via the tree
|
||||
assert models.DocumentAccess.objects.count() == 1
|
||||
|
||||
getting_started = root.get_children().get(title="Getting Started")
|
||||
mock_yhub.return_value.create_ydoc.assert_any_call(getting_started, YJS)
|
||||
assert getting_started.get_children().count() == 1
|
||||
assert getting_started.get_children().first().title == "rich nested doc"
|
||||
|
||||
what_is_outline = root.get_children().get(title="What is Outline")
|
||||
assert what_is_outline.get_children().count() == 1
|
||||
assert what_is_outline.get_children().first().title == "nested doc"
|
||||
|
||||
mock_capture.assert_called_once_with(
|
||||
PosthogEventName.DOC_IMPORTED,
|
||||
user,
|
||||
{"format": "zip", "count": 6},
|
||||
)
|
||||
|
||||
|
||||
@patch("core.api.viewsets.YHubService")
|
||||
@patch("core.services.converter_services.Converter.convert")
|
||||
def test_api_documents_import_zip_media_uploaded(mock_convert, mock_yhub, settings):
|
||||
"""Media files referenced in .md content are uploaded to S3 and recorded on the document."""
|
||||
settings.CONVERSION_UPLOAD_ENABLED = True
|
||||
# Pass the markdown through unchanged so the rewritten S3 URL is visible in content.
|
||||
mock_convert.side_effect = lambda content, **_: content.decode("utf-8")
|
||||
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
with open(FIXTURES / "outline-export.zip", "rb") as f:
|
||||
response = client.post(URL, {"zip": f}, format="multipart")
|
||||
|
||||
assert response.status_code == 201
|
||||
|
||||
rich = models.Document.objects.get(title="rich nested doc")
|
||||
assert len(rich.attachments) == 1
|
||||
|
||||
key = rich.attachments[0]
|
||||
assert key.startswith(f"{rich.id}/attachments/")
|
||||
assert key.endswith(".jpeg")
|
||||
|
||||
# File is actually in S3
|
||||
head = default_storage.connection.meta.client.head_object(
|
||||
Bucket=default_storage.bucket_name, Key=key
|
||||
)
|
||||
assert head["ContentType"] == "image/jpeg"
|
||||
|
||||
# The media URL was rewritten before being handed to the collaboration server
|
||||
calls_by_doc = {
|
||||
c.args[0]: c.args[1]
|
||||
for c in mock_yhub.return_value.create_ydoc.call_args_list
|
||||
}
|
||||
assert f"/media/{key}" in calls_by_doc[rich]
|
||||
|
||||
|
||||
@patch("core.services.converter_services.Converter.convert")
|
||||
def test_api_documents_import_zip_conversion_failure_continues(mock_convert, settings):
|
||||
"""A conversion failure on one node is logged and skipped; other nodes are still created."""
|
||||
settings.CONVERSION_UPLOAD_ENABLED = True
|
||||
mock_convert.side_effect = ConversionError("boom")
|
||||
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
buf = _make_zip(
|
||||
("Col/a.md", b"# A"),
|
||||
("Col/b.md", b"# B"),
|
||||
)
|
||||
buf.name = "export.zip"
|
||||
|
||||
response = client.post(URL, {"zip": buf}, format="multipart")
|
||||
|
||||
assert response.status_code == 201
|
||||
assert response.json()["count"] == 2 # a + b; Col is a root container
|
||||
assert models.Document.objects.count() == 3
|
||||
|
||||
|
||||
@patch("core.services.converter_services.Converter.convert")
|
||||
def test_api_documents_import_zip_empty_zip(mock_convert, settings):
|
||||
"""An empty ZIP returns 201 with zero documents created."""
|
||||
settings.CONVERSION_UPLOAD_ENABLED = True
|
||||
|
||||
user = factories.UserFactory()
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
response = client.post(URL, {"zip": _make_zip()}, format="multipart")
|
||||
|
||||
assert response.status_code == 201
|
||||
assert response.json() == {"count": 0, "roots": []}
|
||||
assert not models.Document.objects.exists()
|
||||
mock_convert.assert_not_called()
|
||||
Reference in New Issue
Block a user