From 45e8e31911fe2dd4bd3c4f741be02fd24483f63f Mon Sep 17 00:00:00 2001 From: Mathieu Agopian Date: Thu, 24 Sep 2026 18:37:12 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(backend)=20expose=20/documents/import?= =?UTF-8?q?-zip/=20API=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /api/v1.0/documents/import-zip/ accepts a multipart ZIP file, and: - parses it with the zip_import service - uploads embedded media to S3 - converts each page from Markdown to YJS - persists the document tree. It then returns the count of imported pages and the list of root documents. Signed-off-by: Mathieu Agopian --- CHANGELOG.md | 1 + src/backend/core/api/viewsets.py | 136 ++++++++++++ .../test_api_documents_import_zip.py | 210 ++++++++++++++++++ 3 files changed, 347 insertions(+) create mode 100644 src/backend/core/tests/documents/test_api_documents_import_zip.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ef9fac21..94ddd23cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ and this project adheres to offline - 🐛(frontend) stop the service worker from caching the collaboration server's rest api +- ✨(backend) import documents from a ZIP file exported from Outline or Notion ### Changed diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 8a452936e..472967545 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -5,10 +5,14 @@ import ipaddress import json import logging +import re import socket import uuid +import zipfile from collections import defaultdict from functools import partial +from io import BytesIO +from pathlib import PurePosixPath from urllib.parse import unquote, urlencode, urlparse from django.conf import settings @@ -69,6 +73,7 @@ from core.services.search_indexers import ( get_visited_document_ids_of, ) from core.services.yhub_services import YHubError, YHubService +from core.services.zip_import import parse_zip from core.tasks.access import reset_service_connections_in_cascade from core.tasks.documents import sync_service_deletions_in_cascade from core.tasks.mail import send_ask_for_access_mail @@ -436,6 +441,89 @@ class DocumentMetadata(drf.metadata.SimpleMetadata): return simple_metadata +def _import_zip_create_node(node, user, converter, parent_doc=None): + """Create a Document (and its subtree) from a DocumentNode parsed out of a ZIP export. + + Returns (doc, count) where count includes the node itself and all descendants. + """ + attachment_keys = [] + ref_to_url = {} + for ref, media_bytes in node.media.items(): + ext = PurePosixPath(ref).suffix.lstrip(".") + file_uuid = uuid.uuid4() + key = f"{node.id}/{enums.ATTACHMENTS_FOLDER}/{file_uuid}.{ext}" + default_storage.connection.meta.client.upload_fileobj( + BytesIO(media_bytes), + default_storage.bucket_name, + key, + ExtraArgs={"ContentType": f"image/{ext}"}, + ) + attachment_keys.append(key) + ref_to_url[ref] = f"{settings.MEDIA_URL}{key}" + + yjs_content = None + if node.content is not None: + text = node.content.decode("utf-8", errors="replace") + for ref, url in ref_to_url.items(): + # Match the ref URL inside a markdown image/link, capturing + # anything between the URL and the closing paren (e.g. an + # Outline-style size hint: `" =1200x1600"`). + text = re.sub( + r""" + \]\( # closing bracket + opening paren + """ + + re.escape(ref) + + r""" + ([^)]*) # optional title / hint before closing paren + \) # closing paren + """, + lambda m, _url=url: f"]({_url}{m.group(1)})", + text, + flags=re.VERBOSE, + ) + try: + yjs_content = converter.convert( + text.encode("utf-8"), + content_type=mime_types.MARKDOWN, + accept=mime_types.YJS, + ) + except (ConversionError, YProviderServiceUnavailableError) as err: + logger.warning( + "ZIP import: conversion failed for '%s': %s", node.title, err + ) + + doc_kwargs = {"id": node.id, "title": node.title, "creator": user} + if attachment_keys: + doc_kwargs["attachments"] = attachment_keys + + if parent_doc is None: + doc = create_tree_node_with_retry( + lambda _kw=doc_kwargs: models.Document.add_root(**_kw) + ) + models.DocumentAccess.objects.create( + document=doc, user=user, role=models.RoleChoices.OWNER + ) + else: + doc = create_tree_node_with_retry( + lambda _parent=parent_doc, _kw=doc_kwargs: _parent.add_child(**_kw) + ) + + if yjs_content is not None: + try: + YHubService(user=user).create_ydoc(doc, yjs_content) + except YHubError as err: + logger.warning( + "ZIP import: could not seed content for '%s': %s", node.title, err + ) + + count = 1 + for child in node.children: + _, child_count = _import_zip_create_node(child, user, converter, doc) + count += child_count + + return doc, count + + # pylint: disable=too-many-public-methods class DocumentViewSet( SerializerPerActionMixin, @@ -1854,6 +1942,54 @@ class DocumentViewSet( status=drf.status.HTTP_200_OK, ) + @drf.decorators.action(detail=False, methods=["post"], url_path="import-zip") + def import_zip(self, request, *args, **kwargs): + """Import a tree of documents from a ZIP export (Outline, Notion, ...).""" + if not request.user.is_authenticated: + raise drf.exceptions.NotAuthenticated() + + if not settings.CONVERSION_UPLOAD_ENABLED: + raise drf.exceptions.ValidationError({"zip": ["ZIP import is not allowed"]}) + + zip_file = request.data.get("zip") + if not zip_file: + raise drf.exceptions.ValidationError({"zip": ["This field is required."]}) + + try: + zf = zipfile.ZipFile(BytesIO(zip_file.read())) + except zipfile.BadZipFile as exc: + raise drf.exceptions.ValidationError( + {"zip": ["Invalid ZIP file."]} + ) from exc + + with zf: + nodes = parse_zip(zf) + + user = request.user + converter = Converter() + + root_docs = [] + pages = 0 + with transaction.atomic(): + for node in nodes: + doc, count = _import_zip_create_node(node, user, converter) + root_docs.append(doc) + pages += count - 1 # root containers are not imported pages + + posthog_capture( + PosthogEventName.DOC_IMPORTED, + user, + {"format": "zip", "count": pages}, + ) + + return drf.response.Response( + { + "count": pages, + "roots": [{"id": str(doc.id), "title": doc.title} for doc in root_docs], + }, + status=drf.status.HTTP_201_CREATED, + ) + @drf.decorators.action(detail=True, methods=["post"], url_path="attachment-upload") def attachment_upload(self, request, *args, **kwargs): """Upload a file related to a given document""" diff --git a/src/backend/core/tests/documents/test_api_documents_import_zip.py b/src/backend/core/tests/documents/test_api_documents_import_zip.py new file mode 100644 index 000000000..90bcc5379 --- /dev/null +++ b/src/backend/core/tests/documents/test_api_documents_import_zip.py @@ -0,0 +1,210 @@ +"""Tests for the POST /documents/import-zip/ endpoint.""" + +import io +import zipfile +from pathlib import Path +from unittest.mock import patch + +from django.core.files.storage import default_storage + +import pytest +from rest_framework.test import APIClient + +from core import factories, models +from core.services.converter_services import ConversionError +from core.utils.analytics import PosthogEventName + +pytestmark = pytest.mark.django_db + +FIXTURES = Path(__file__).parent.parent / "fixtures" +URL = "/api/v1.0/documents/import-zip/" +YJS = "fakeyjs" + + +def _make_zip(*entries): + """Return BytesIO of a zip containing the given (name, bytes) entries.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for name, content in entries: + zf.writestr(name, content) + buf.seek(0) + return buf + + +def test_api_documents_import_zip_anonymous(): + """Anonymous users cannot import a ZIP.""" + response = APIClient().post(URL, {"zip": _make_zip()}, format="multipart") + assert response.status_code == 401 + assert not models.Document.objects.exists() + + +def test_api_documents_import_zip_disabled(settings): + """Returns 400 when CONVERSION_UPLOAD_ENABLED is False.""" + settings.CONVERSION_UPLOAD_ENABLED = False + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + response = client.post(URL, {"zip": _make_zip()}, format="multipart") + + assert response.status_code == 400 + assert response.json() == {"zip": ["ZIP import is not allowed"]} + assert not models.Document.objects.exists() + + +def test_api_documents_import_zip_missing_field(settings): + """Returns 400 when no zip field is provided.""" + settings.CONVERSION_UPLOAD_ENABLED = True + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + response = client.post(URL, {}, format="multipart") + + assert response.status_code == 400 + assert response.json() == {"zip": ["This field is required."]} + + +def test_api_documents_import_zip_not_a_zip(settings): + """Returns 400 when the uploaded file is not a valid ZIP.""" + settings.CONVERSION_UPLOAD_ENABLED = True + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + not_a_zip = io.BytesIO(b"this is not a zip file") + not_a_zip.name = "export.zip" + + response = client.post(URL, {"zip": not_a_zip}, format="multipart") + + assert response.status_code == 400 + assert response.json() == {"zip": ["Invalid ZIP file."]} + + +@patch("core.api.viewsets.YHubService") +@patch("core.services.converter_services.Converter.convert") +def test_api_documents_import_zip_success(mock_convert, mock_yhub, settings): + """201 with correct count and root list; documents and accesses created in DB.""" + settings.CONVERSION_UPLOAD_ENABLED = True + mock_convert.return_value = YJS + + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + with patch("core.api.viewsets.posthog_capture") as mock_capture: + with open(FIXTURES / "outline-export.zip", "rb") as f: + response = client.post(URL, {"zip": f}, format="multipart") + + assert response.status_code == 201, response.json() + data = response.json() + + # 4 children + 2 grandchildren; root "Welcome" is a container, not a page + assert data["count"] == 6 + assert len(data["roots"]) == 1 + assert data["roots"][0]["title"] == "Welcome" + + assert models.Document.objects.count() == 7 + + root = models.Document.objects.get(id=data["roots"][0]["id"]) + assert root.is_root() + assert root.get_children().count() == 4 + assert models.DocumentAccess.objects.filter( + document=root, user=user, role=models.RoleChoices.OWNER + ).exists() + + # Only the root gets an explicit access record; children inherit via the tree + assert models.DocumentAccess.objects.count() == 1 + + getting_started = root.get_children().get(title="Getting Started") + mock_yhub.return_value.create_ydoc.assert_any_call(getting_started, YJS) + assert getting_started.get_children().count() == 1 + assert getting_started.get_children().first().title == "rich nested doc" + + what_is_outline = root.get_children().get(title="What is Outline") + assert what_is_outline.get_children().count() == 1 + assert what_is_outline.get_children().first().title == "nested doc" + + mock_capture.assert_called_once_with( + PosthogEventName.DOC_IMPORTED, + user, + {"format": "zip", "count": 6}, + ) + + +@patch("core.api.viewsets.YHubService") +@patch("core.services.converter_services.Converter.convert") +def test_api_documents_import_zip_media_uploaded(mock_convert, mock_yhub, settings): + """Media files referenced in .md content are uploaded to S3 and recorded on the document.""" + settings.CONVERSION_UPLOAD_ENABLED = True + # Pass the markdown through unchanged so the rewritten S3 URL is visible in content. + mock_convert.side_effect = lambda content, **_: content.decode("utf-8") + + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + with open(FIXTURES / "outline-export.zip", "rb") as f: + response = client.post(URL, {"zip": f}, format="multipart") + + assert response.status_code == 201 + + rich = models.Document.objects.get(title="rich nested doc") + assert len(rich.attachments) == 1 + + key = rich.attachments[0] + assert key.startswith(f"{rich.id}/attachments/") + assert key.endswith(".jpeg") + + # File is actually in S3 + head = default_storage.connection.meta.client.head_object( + Bucket=default_storage.bucket_name, Key=key + ) + assert head["ContentType"] == "image/jpeg" + + # The media URL was rewritten before being handed to the collaboration server + calls_by_doc = { + c.args[0]: c.args[1] + for c in mock_yhub.return_value.create_ydoc.call_args_list + } + assert f"/media/{key}" in calls_by_doc[rich] + + +@patch("core.services.converter_services.Converter.convert") +def test_api_documents_import_zip_conversion_failure_continues(mock_convert, settings): + """A conversion failure on one node is logged and skipped; other nodes are still created.""" + settings.CONVERSION_UPLOAD_ENABLED = True + mock_convert.side_effect = ConversionError("boom") + + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + buf = _make_zip( + ("Col/a.md", b"# A"), + ("Col/b.md", b"# B"), + ) + buf.name = "export.zip" + + response = client.post(URL, {"zip": buf}, format="multipart") + + assert response.status_code == 201 + assert response.json()["count"] == 2 # a + b; Col is a root container + assert models.Document.objects.count() == 3 + + +@patch("core.services.converter_services.Converter.convert") +def test_api_documents_import_zip_empty_zip(mock_convert, settings): + """An empty ZIP returns 201 with zero documents created.""" + settings.CONVERSION_UPLOAD_ENABLED = True + + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + response = client.post(URL, {"zip": _make_zip()}, format="multipart") + + assert response.status_code == 201 + assert response.json() == {"count": 0, "roots": []} + assert not models.Document.objects.exists() + mock_convert.assert_not_called()