mirror of
https://github.com/suitenumerique/docs.git
synced 2026-10-01 05:55:16 +02:00
✨(backend) parse ZIP exports into a document tree
Add a `parse_zip` service that reads `.md` files from a ZIP archive and builds a tree of `DocumentNode` objects from the directory hierarchy. Folders and same-name `.md` files at the same level merge into a single node: the file provides content, the folder provides children. Non-markdown files (images, CSVs, ...) are silently skipped (for now). Signed-off-by: Mathieu Agopian <mathieu@agopian.info>
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
"""Parse ZIP exports into a document tree."""
|
||||
|
||||
import zipfile
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
|
||||
@dataclass
|
||||
class DocumentNode:
|
||||
"""A document node in the import tree."""
|
||||
|
||||
title: str
|
||||
content: bytes | None = None
|
||||
children: list["DocumentNode"] = field(default_factory=list)
|
||||
|
||||
|
||||
def parse_zip(zf: zipfile.ZipFile) -> list[DocumentNode]:
|
||||
"""Return root DocumentNodes parsed from a ZIP export."""
|
||||
return _build_tree(_read_md_files(zf))
|
||||
|
||||
|
||||
def _read_md_files(zf: zipfile.ZipFile) -> dict[str, bytes]:
|
||||
"""Read .md files from the zip."""
|
||||
result = {}
|
||||
for info in zf.infolist():
|
||||
if not info.filename.lower().endswith(".md"):
|
||||
continue
|
||||
result[info.filename] = zf.read(info.filename)
|
||||
return result
|
||||
|
||||
|
||||
def _build_tree(md_files: dict[str, bytes]) -> list[DocumentNode]:
|
||||
"""
|
||||
Build a DocumentNode tree from a flat dict of path -> markdown content.
|
||||
|
||||
A folder and a .md file with the same name at the same level merge into a
|
||||
single node: the .md provides content, the folder provides children.
|
||||
Folders without a matching .md become container nodes with no content.
|
||||
"""
|
||||
paths = {PurePosixPath(k): v for k, v in md_files.items()}
|
||||
|
||||
all_dirs: set[PurePosixPath] = set()
|
||||
for path in paths:
|
||||
for ancestor in path.parents:
|
||||
if str(ancestor) != ".":
|
||||
all_dirs.add(ancestor)
|
||||
|
||||
by_parent: dict[PurePosixPath, dict[str, bytes]] = defaultdict(dict)
|
||||
for path, content in paths.items():
|
||||
by_parent[path.parent][path.stem] = content
|
||||
|
||||
dirs_by_parent: dict[PurePosixPath, set[str]] = defaultdict(set)
|
||||
for d in all_dirs:
|
||||
dirs_by_parent[d.parent].add(d.name)
|
||||
|
||||
def build_children(parent: PurePosixPath) -> list[DocumentNode]:
|
||||
files = by_parent.get(parent, {})
|
||||
subdirs = dirs_by_parent.get(parent, set())
|
||||
nodes = []
|
||||
for name in sorted(set(files) | subdirs):
|
||||
node = DocumentNode(title=name, content=files.get(name))
|
||||
subdir = parent / name
|
||||
if subdir in all_dirs:
|
||||
node.children = build_children(subdir)
|
||||
nodes.append(node)
|
||||
return nodes
|
||||
|
||||
return build_children(PurePosixPath("."))
|
||||
Binary file not shown.
@@ -0,0 +1,83 @@
|
||||
"""Unit tests for the zip_import service."""
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from core.services.zip_import import parse_zip
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def test_parse_empty_zip():
|
||||
"""An empty zip returns no nodes."""
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w"):
|
||||
pass
|
||||
buf.seek(0)
|
||||
with zipfile.ZipFile(buf) as zf:
|
||||
assert not parse_zip(zf)
|
||||
|
||||
|
||||
def test_parse_zip_ignores_non_md_files():
|
||||
"""Only .md files produce nodes; images, CSVs etc. are silently skipped."""
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as zf:
|
||||
zf.writestr("doc.md", b"# Hello")
|
||||
zf.writestr("image.png", b"\x89PNG")
|
||||
zf.writestr("data.csv", b"a,b,c")
|
||||
buf.seek(0)
|
||||
with zipfile.ZipFile(buf) as zf:
|
||||
nodes = parse_zip(zf)
|
||||
assert len(nodes) == 1
|
||||
assert nodes[0].title == "doc"
|
||||
|
||||
|
||||
def test_parse_zip_root_collection():
|
||||
"""A top-level folder with no matching .md becomes a container node."""
|
||||
with zipfile.ZipFile(FIXTURES / "outline-export.zip") as zf:
|
||||
roots = parse_zip(zf)
|
||||
|
||||
assert len(roots) == 1
|
||||
assert roots[0].title == "Welcome"
|
||||
assert roots[0].content is None
|
||||
assert len(roots[0].children) == 4
|
||||
|
||||
|
||||
def test_parse_zip_nested_children():
|
||||
"""A subfolder alongside a .md of the same name yields children on that node."""
|
||||
with zipfile.ZipFile(FIXTURES / "outline-export.zip") as zf:
|
||||
roots = parse_zip(zf)
|
||||
|
||||
by_title = {c.title: c for c in roots[0].children}
|
||||
getting_started = by_title["Getting Started"]
|
||||
assert len(getting_started.children) == 1
|
||||
assert getting_started.children[0].title == "rich nested doc"
|
||||
|
||||
|
||||
def test_parse_zip_content_present():
|
||||
"""Nodes built from .md files carry their raw bytes as content."""
|
||||
with zipfile.ZipFile(FIXTURES / "outline-export.zip") as zf:
|
||||
roots = parse_zip(zf)
|
||||
|
||||
by_title = {c.title: c for c in roots[0].children}
|
||||
assert by_title["Our Editor"].content is not None
|
||||
assert b"editor" in by_title["Our Editor"].content.lower()
|
||||
|
||||
|
||||
def test_parse_folder_alongside_md_merges():
|
||||
"""A folder and a same-name .md at the same level merge into one node."""
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as zf:
|
||||
zf.writestr("Parent/Doc.md", b"# Doc content")
|
||||
zf.writestr("Parent/Doc/Child.md", b"# Child")
|
||||
buf.seek(0)
|
||||
with zipfile.ZipFile(buf) as zf:
|
||||
roots = parse_zip(zf)
|
||||
|
||||
assert len(roots) == 1
|
||||
doc = roots[0].children[0]
|
||||
assert doc.title == "Doc"
|
||||
assert doc.content == b"# Doc content"
|
||||
assert len(doc.children) == 1
|
||||
assert doc.children[0].title == "Child"
|
||||
Reference in New Issue
Block a user