From 4b7ebd0bf46e226872254189174cf89b02dea4b2 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Thu, 11 Jan 2024 13:00:12 +0700 Subject: [PATCH] UBERF-4604 Load ydoc from minio (#4338) Signed-off-by: Alexander Onnikov --- common/config/rush/pnpm-lock.yaml | 20 +++---- dev/docker-compose.yaml | 2 +- packages/text-editor/package.json | 3 +- .../src/components/Collaboration.svelte | 2 +- .../CollaborativeAttributeBox.svelte | 2 +- .../components/CollaborativeTextEditor.svelte | 36 +++++++----- .../src/components/CollaboratorEditor.svelte | 2 +- .../node-view/svelte-node-view-renderer.ts | 14 +++++ packages/text-editor/src/index.ts | 8 +-- packages/text-editor/src/provider/minio.ts | 55 +++++++++++++++++++ .../src/{provider.ts => provider/tiptap.ts} | 19 +------ packages/text-editor/src/provider/utils.ts | 32 +++++++++++ packages/text-editor/src/utils.ts | 2 +- 13 files changed, 144 insertions(+), 53 deletions(-) create mode 100644 packages/text-editor/src/provider/minio.ts rename packages/text-editor/src/{provider.ts => provider/tiptap.ts} (83%) create mode 100644 packages/text-editor/src/provider/utils.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index c61eb973f0..1a8a68a587 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -998,6 +998,9 @@ dependencies: lexorank: specifier: ~1.0.4 version: 1.0.5 + lib0: + specifier: ^0.2.88 + version: 0.2.88 libphonenumber-js: specifier: ^1.9.46 version: 1.10.47 @@ -12802,14 +12805,6 @@ packages: resolution: {integrity: sha512-K1B/Yr/gIU0wm68hk/yB0p/mv6xM3ShD5aci42vOwcjof8slG8Kpo3Q7+1WTv7DaRHKWRgLPqrFDt+4GtuFAtA==} dev: false - /lib0@0.2.86: - resolution: {integrity: sha512-kxigQTM4Q7NwJkEgdqQvU21qiR37twcqqLmh+/SbiGbRLfPlLVbHyY9sWp7PwXh0Xus9ELDSjsUOwcrdt5yZ4w==} - engines: {node: '>=16'} - hasBin: true - dependencies: - isomorphic.js: 0.2.5 - dev: false - /lib0@0.2.88: resolution: {integrity: sha512-KyroiEvCeZcZEMx5Ys+b4u4eEBbA1ch7XUaBhYpwa/nPMrzTjUhI4RfcytmQfYoTBPcdyx+FX6WFNIoNuJzJfQ==} engines: {node: '>=16'} @@ -17241,7 +17236,7 @@ packages: y-protocols: ^1.0.1 yjs: ^13.5.38 dependencies: - lib0: 0.2.86 + lib0: 0.2.88 prosemirror-model: 1.19.3 yjs: 13.6.8 dev: false @@ -17262,7 +17257,7 @@ packages: peerDependencies: yjs: ^13.5.6 dependencies: - lib0: 0.2.86 + lib0: 0.2.88 lodash.debounce: 4.0.8 y-protocols: 1.0.6(yjs@13.6.8) yjs: 13.6.8 @@ -17327,7 +17322,7 @@ packages: resolution: {integrity: sha512-ZPq0hpJQb6f59B++Ngg4cKexDJTvfOgeiv0sBc4sUm8CaBWH7OQC4kcCgrqbjJ/B2+6vO49exvTmYfdlPtcjbg==} engines: {node: '>=16.0.0', npm: '>=8.0.0'} dependencies: - lib0: 0.2.86 + lib0: 0.2.88 dev: false /ylru@1.3.2: @@ -23653,7 +23648,7 @@ packages: dev: false file:projects/text-editor.tgz(@types/node@16.11.68)(bufferutil@4.0.7)(esbuild@0.16.17)(postcss-load-config@4.0.1)(postcss@8.4.31)(prosemirror-model@1.19.3)(ts-node@10.9.1): - resolution: {integrity: sha512-QgUDUW3v5zIVHETcfC/UsoZ8EPeauNcSyGO+I/J6rXLG2ndtNKIWRdl7Herj480/Q6JjPjWmm5RTIRAnz8lZXQ==, tarball: file:projects/text-editor.tgz} + resolution: {integrity: sha512-q4IthAiGXw8l3HkEitwxvXppLyB/mszif5wHdy0jcaz7Yc0moJOzS6ElJ3AgA/0IQHxKB7ladDAkU/wklo7icw==, tarball: file:projects/text-editor.tgz} id: file:projects/text-editor.tgz name: '@rush-temp/text-editor' version: 0.0.0 @@ -23696,6 +23691,7 @@ packages: eslint-plugin-promise: 6.1.1(eslint@8.54.0) eslint-plugin-svelte: 2.35.0(eslint@8.54.0)(svelte@4.2.5)(ts-node@10.9.1) jest: 29.7.0(@types/node@16.11.68)(ts-node@10.9.1) + lib0: 0.2.88 prettier: 3.1.0 prettier-plugin-svelte: 3.1.0(prettier@3.1.0)(svelte@4.2.5) prosemirror-codemark: 0.4.2(prosemirror-model@1.19.3) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index ee902e1810..d78ddcfe0b 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -102,7 +102,7 @@ services: environment: - COLLABORATOR_PORT=3078 - SECRET=secret - - TRANSACTOR_URL=ws://localhost:3333 + - TRANSACTOR_URL=ws://transactor:3333 - UPLOAD_URL=/files - MONGO_URL=mongodb://mongodb:27017 - MINIO_ENDPOINT=minio diff --git a/packages/text-editor/package.json b/packages/text-editor/package.json index 4d7c641252..4eca9395ed 100644 --- a/packages/text-editor/package.json +++ b/packages/text-editor/package.json @@ -76,6 +76,7 @@ "y-prosemirror": "^1.2.1", "rfc6902": "^5.0.1", "diff": "^5.1.0", - "slugify": "^1.6.6" + "slugify": "^1.6.6", + "lib0": "^0.2.88" } } diff --git a/packages/text-editor/src/components/Collaboration.svelte b/packages/text-editor/src/components/Collaboration.svelte index 53b0078427..a6e312dac5 100644 --- a/packages/text-editor/src/components/Collaboration.svelte +++ b/packages/text-editor/src/components/Collaboration.svelte @@ -20,7 +20,7 @@ import presentation from '@hcengineering/presentation' import textEditorPlugin from '../plugin' - import { DocumentId, TiptapCollabProvider, createTiptapCollaborationData } from '../provider' + import { DocumentId, TiptapCollabProvider, createTiptapCollaborationData } from '../provider/tiptap' import { CollaborationIds } from '../types' export let documentId: DocumentId diff --git a/packages/text-editor/src/components/CollaborativeAttributeBox.svelte b/packages/text-editor/src/components/CollaborativeAttributeBox.svelte index 1fd0e13dcb..e55b00a7f5 100644 --- a/packages/text-editor/src/components/CollaborativeAttributeBox.svelte +++ b/packages/text-editor/src/components/CollaborativeAttributeBox.svelte @@ -21,7 +21,7 @@ import { FocusExtension } from './extension/focus' import { FileAttachFunction } from './extension/imageExt' import textEditorPlugin from '../plugin' - import { minioDocumentId, mongodbDocumentId, platformDocumentId } from '../provider' + import { minioDocumentId, mongodbDocumentId, platformDocumentId } from '../provider/utils' import { RefAction, TextNodeAction } from '../types' export let object: Doc diff --git a/packages/text-editor/src/components/CollaborativeTextEditor.svelte b/packages/text-editor/src/components/CollaborativeTextEditor.svelte index 04cf5f37f0..1cc88e8707 100644 --- a/packages/text-editor/src/components/CollaborativeTextEditor.svelte +++ b/packages/text-editor/src/components/CollaborativeTextEditor.svelte @@ -29,7 +29,8 @@ import { textEditorCommandHandler } from '../commands' import { EditorKit } from '../kits/editor-kit' import textEditorPlugin from '../plugin' - import { DocumentId, TiptapCollabProvider } from '../provider' + import { MinioProvider } from '../provider/minio' + import { DocumentId, TiptapCollabProvider } from '../provider/tiptap' import { CollaborationIds, RefAction, @@ -90,7 +91,9 @@ const ydoc = getContext(CollaborationIds.Doc) ?? new YDoc() const contextProvider = getContext(CollaborationIds.Provider) - const provider: TiptapCollabProvider = + const localProvider = contextProvider === undefined ? new MinioProvider(documentId, ydoc) : undefined + + const remoteProvider: TiptapCollabProvider = contextProvider ?? new TiptapCollabProvider({ url: collaboratorURL, @@ -103,8 +106,14 @@ } }) - let loading = true - void provider.loaded.then(() => (loading = false)) + let localSynced = false + let remoteSynced = false + + $: loading = !localSynced && !remoteSynced + $: editable = !readonly && remoteSynced + + void localProvider?.loaded.then(() => (localSynced = true)) + void remoteProvider.loaded.then(() => (remoteSynced = true)) let editor: Editor let element: HTMLElement @@ -142,11 +151,11 @@ } export function takeSnapshot (snapshotId: string): void { - copyDocumentContent(documentId, snapshotId, { provider }, initialContentId) + copyDocumentContent(documentId, snapshotId, { provider: remoteProvider }, initialContentId) } export function copyField (srcFieldId: string, dstFieldId: string): void { - copyDocumentField(documentId, srcFieldId, dstFieldId, { provider }, initialContentId) + copyDocumentField(documentId, srcFieldId, dstFieldId, { provider: remoteProvider }, initialContentId) } export function isEditable (): boolean { @@ -179,11 +188,11 @@ } $: if (editor !== undefined) { - editor.setEditable(!readonly) + editor.setEditable(editable, true) } $: showTextStyleToolbar = - (!readonly || textFormatCategories.length > 0 || textNodeActions.length > 0) && canShowPopups + ((editable && textFormatCategories.length > 0) || textNodeActions.length > 0) && canShowPopups $: tippyOptions = { zIndex: 100000, @@ -239,7 +248,7 @@ appendTo: () => boundary ?? element }, shouldShow: ({ editor }) => { - if (readonly || !canShowPopups) { + if (!editable || !canShowPopups) { return false } return editor.isActive('image') @@ -250,7 +259,7 @@ field }), CollaborationCursor.configure({ - provider, + provider: remoteProvider, user, render: renderCursor, selectionRender: noSelectionRender @@ -295,10 +304,11 @@ try { editor.destroy() } catch (err: any) {} - if (contextProvider === undefined) { - provider.destroy() - } } + if (contextProvider === undefined) { + remoteProvider.destroy() + } + localProvider?.destroy() }) diff --git a/packages/text-editor/src/components/CollaboratorEditor.svelte b/packages/text-editor/src/components/CollaboratorEditor.svelte index 48ad9af70d..5766378f57 100644 --- a/packages/text-editor/src/components/CollaboratorEditor.svelte +++ b/packages/text-editor/src/components/CollaboratorEditor.svelte @@ -21,7 +21,7 @@ import { TextSelection } from '@tiptap/pm/state' import textEditorPlugin from '../plugin' - import { DocumentId } from '../provider' + import { DocumentId } from '../provider/tiptap' import { TextEditorCommandHandler, TextFormatCategory, TextNodeAction } from '../types' import CollaborativeTextEditor from './CollaborativeTextEditor.svelte' diff --git a/packages/text-editor/src/components/node-view/svelte-node-view-renderer.ts b/packages/text-editor/src/components/node-view/svelte-node-view-renderer.ts index d0fd01df7e..514f4968cb 100644 --- a/packages/text-editor/src/components/node-view/svelte-node-view-renderer.ts +++ b/packages/text-editor/src/components/node-view/svelte-node-view-renderer.ts @@ -47,6 +47,8 @@ class SvelteNodeView extends NodeView void +} + +async function fetchContent (doc: YDoc, name: string): Promise { + const frontUrl = getMetadata(presentation.metadata.FrontUrl) ?? window.location.origin + + const res = await fetch(concatLink(frontUrl, `/files?file=${name}`)) + + if (res.ok) { + const blob = await res.blob() + const buffer = await blob.arrayBuffer() + applyUpdate(doc, new Uint8Array(buffer)) + } +} + +export class MinioProvider extends Observable { + loaded: Promise + + constructor (name: string, doc: YDoc) { + super() + + if (name.startsWith('minio://')) { + name = name.split('://', 2)[1] + } + + void fetchContent(doc, name).then(() => { + this.emit('synced', [this]) + }) + + this.loaded = new Promise((resolve) => { + this.on('synced', resolve) + }) + } +} diff --git a/packages/text-editor/src/provider.ts b/packages/text-editor/src/provider/tiptap.ts similarity index 83% rename from packages/text-editor/src/provider.ts rename to packages/text-editor/src/provider/tiptap.ts index d64e32f896..8b0165ba61 100644 --- a/packages/text-editor/src/provider.ts +++ b/packages/text-editor/src/provider/tiptap.ts @@ -13,10 +13,10 @@ // limitations under the License. // import { Doc as Ydoc } from 'yjs' -import type { Doc, Ref } from '@hcengineering/core' -import { type KeyedAttribute, getClient } from '@hcengineering/presentation' import { HocuspocusProvider, type HocuspocusProviderConfiguration } from '@hocuspocus/provider' +export type DocumentId = string + export type TiptapCollabProviderConfiguration = HocuspocusProviderConfiguration & Required> & Omit & { @@ -28,21 +28,6 @@ export interface TiptapCollabProviderURLParameters { targetContentId?: DocumentId } -export type DocumentId = string - -export function minioDocumentId (docId: Ref, attr?: KeyedAttribute): DocumentId { - return attr !== undefined ? `minio://${docId}%${attr.key}` : `minio://${docId}` -} - -export function platformDocumentId (docId: Ref, attr: KeyedAttribute): DocumentId { - return `platform://${attr.attr.attributeOf}/${docId}/${attr.key}` -} - -export function mongodbDocumentId (docId: Ref, attr: KeyedAttribute): DocumentId { - const domain = getClient().getHierarchy().getDomain(attr.attr.attributeOf) - return `mongodb://${domain}/${docId}/${attr.key}` -} - export class TiptapCollabProvider extends HocuspocusProvider { loaded: Promise diff --git a/packages/text-editor/src/provider/utils.ts b/packages/text-editor/src/provider/utils.ts new file mode 100644 index 0000000000..6e9b230a5f --- /dev/null +++ b/packages/text-editor/src/provider/utils.ts @@ -0,0 +1,32 @@ +// +// Copyright © 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import type { Doc, Ref } from '@hcengineering/core' +import { type KeyedAttribute, getClient } from '@hcengineering/presentation' + +import { type DocumentId } from './tiptap' + +export function minioDocumentId (docId: Ref, attr?: KeyedAttribute): DocumentId { + return attr !== undefined ? `minio://${docId}%${attr.key}` : `minio://${docId}` +} + +export function platformDocumentId (docId: Ref, attr: KeyedAttribute): DocumentId { + return `platform://${attr.attr.attributeOf}/${docId}/${attr.key}` +} + +export function mongodbDocumentId (docId: Ref, attr: KeyedAttribute): DocumentId { + const domain = getClient().getHierarchy().getDomain(attr.attr.attributeOf) + return `mongodb://${domain}/${docId}/${attr.key}` +} diff --git a/packages/text-editor/src/utils.ts b/packages/text-editor/src/utils.ts index 72a4cf48b1..957a1ffc5f 100644 --- a/packages/text-editor/src/utils.ts +++ b/packages/text-editor/src/utils.ts @@ -28,7 +28,7 @@ import { themeStore } from '@hcengineering/ui' -import { type DocumentId, TiptapCollabProvider } from './provider' +import { type DocumentId, TiptapCollabProvider } from './provider/tiptap' import { type CollaborationUser } from './types' type ProviderData = (