From 2b4b97732efaf8f6e0af22b52295b9a47dcc676d Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Mon, 25 Dec 2023 16:49:08 +0700 Subject: [PATCH] UBERF-4569 Collaborative editors for Markup fields (#4247) Signed-off-by: Alexander Onnikov --- .vscode/launch.json | 4 +- common/config/rush/pnpm-lock.yaml | 67 ++- dev/docker-compose.yaml | 2 + models/core/src/core.ts | 4 + models/core/src/index.ts | 2 + models/lead/src/index.ts | 3 +- models/recruit/src/index.ts | 3 +- models/tracker/src/types.ts | 7 +- models/view/src/index.ts | 5 + models/view/src/plugin.ts | 1 + packages/core/lang/en.json | 1 + packages/core/lang/ru.json | 1 + packages/core/src/component.ts | 3 + packages/model/src/dsl.ts | 7 + packages/presentation/src/utils.ts | 3 + .../src/components/Collaboration.svelte | 15 +- .../CollaborativeAttributeBox.svelte | 99 +++++ .../CollaborativeAttributeSectionBox.svelte | 44 ++ .../components/CollaborativeTextEditor.svelte | 393 ++++++++++++++++++ .../src/components/CollaboratorEditor.svelte | 379 +++-------------- .../src/components/ReferenceInput.svelte | 26 +- .../src/components/StyledTextEditor.svelte | 26 +- .../src/components/TextEditor.svelte | 63 +-- .../components/TextEditorStyleToolbar.svelte | 4 +- .../src/components/editor/actions.ts | 82 ++-- .../src/components/extension/submit.ts | 38 ++ packages/text-editor/src/index.ts | 12 +- packages/text-editor/src/provider.ts | 65 ++- packages/text-editor/src/types.ts | 1 + packages/text-editor/src/utils.ts | 20 +- packages/text/src/html.ts | 4 +- packages/ui/src/components/Section.svelte | 79 ++-- plugins/activity-assets/lang/ru.json | 2 +- .../src/activityMessagesUtils.ts | 1 + .../attributes/SetAttributesPresenter.svelte | 5 +- plugins/activity-resources/src/utils.ts | 3 +- .../AttachmentStyleBoxCollabEditor.svelte | 255 ++++++++++++ .../src/components/AttachmentStyledBox.svelte | 136 ++---- .../src/components/AttachmentsGrid.svelte | 93 +++++ plugins/attachment-resources/src/index.ts | 2 + .../src/components/Members.svelte | 59 ++- .../src/components/MembersBox.svelte | 18 +- .../src/components/DepartmentStaff.svelte | 59 +-- .../src/components/Applications.svelte | 64 ++- .../src/components/EditVacancy.svelte | 16 +- .../components/issues/edit/EditIssue.svelte | 35 +- .../templates/EditIssueTemplate.svelte | 26 +- .../components/CollaborativeHTMLEditor.svelte | 27 ++ .../src/components/EditDoc.svelte | 2 +- plugins/view-resources/src/index.ts | 2 + pods/collaborator/package.json | 3 +- pods/collaborator/src/__start.ts | 2 +- server-plugins/chunter-resources/src/index.ts | 15 +- server/collaborator/package.json | 11 +- server/collaborator/src/__start.ts | 2 +- server/collaborator/src/config.ts | 4 + server/collaborator/src/context.ts | 21 +- server/collaborator/src/extensions/action.ts | 92 +++- server/collaborator/src/extensions/storage.ts | 189 ++++----- server/collaborator/src/platform.ts | 43 ++ server/collaborator/src/server.ts | 51 ++- server/collaborator/src/starter.ts | 12 +- server/collaborator/src/storage/adapter.ts | 25 ++ server/collaborator/src/storage/minio.ts | 117 ++++++ server/collaborator/src/storage/mongodb.ts | 78 ++++ server/collaborator/src/storage/platform.ts | 120 ++++++ server/collaborator/src/storage/router.ts | 53 +++ server/collaborator/src/transformers/html.ts | 41 ++ server/collaborator/src/types.ts | 12 +- server/core/src/indexer/summary.ts | 2 +- tests/docker-compose.yaml | 2 + 71 files changed, 2227 insertions(+), 936 deletions(-) create mode 100644 packages/text-editor/src/components/CollaborativeAttributeBox.svelte create mode 100644 packages/text-editor/src/components/CollaborativeAttributeSectionBox.svelte create mode 100644 packages/text-editor/src/components/CollaborativeTextEditor.svelte create mode 100644 packages/text-editor/src/components/extension/submit.ts create mode 100644 plugins/attachment-resources/src/components/AttachmentStyleBoxCollabEditor.svelte create mode 100644 plugins/attachment-resources/src/components/AttachmentsGrid.svelte create mode 100644 plugins/view-resources/src/components/CollaborativeHTMLEditor.svelte create mode 100644 server/collaborator/src/platform.ts create mode 100644 server/collaborator/src/storage/adapter.ts create mode 100644 server/collaborator/src/storage/minio.ts create mode 100644 server/collaborator/src/storage/mongodb.ts create mode 100644 server/collaborator/src/storage/platform.ts create mode 100644 server/collaborator/src/storage/router.ts create mode 100644 server/collaborator/src/transformers/html.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index f0d21f08d3..2f3ef800b6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -87,8 +87,10 @@ "request": "launch", "args": ["src/__start.ts"], "env": { - "SERVER_SECRET": "secret", + "SECRET": "secret", + "METRICS_CONSOLE": "true", "TRANSACTOR_URL": "ws://localhost:3333", + "MONGO_URL": "mongodb://localhost:27017", "MINIO_ACCESS_KEY": "minioadmin", "MINIO_SECRET_KEY": "minioadmin", "MINIO_ENDPOINT": "localhost" diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index b9fee7c314..c308b140be 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -17,6 +17,9 @@ dependencies: '@hocuspocus/server': specifier: ^2.5.0 version: 2.8.1(bufferutil@4.0.7)(yjs@13.6.8) + '@hocuspocus/transformer': + specifier: ^2.5.0 + version: 2.8.1(@tiptap/pm@2.1.12)(y-prosemirror@1.2.1)(yjs@13.6.8) '@koa/cors': specifier: ^3.1.0 version: 3.4.3 @@ -91,7 +94,7 @@ dependencies: version: file:projects/client-resources.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1) '@rush-temp/collaborator': specifier: file:./projects/collaborator.tgz - version: file:projects/collaborator.tgz(bufferutil@4.0.7)(svelte@4.2.5) + version: file:projects/collaborator.tgz(@tiptap/pm@2.1.12)(bufferutil@4.0.7)(prosemirror-model@1.19.3)(svelte@4.2.5) '@rush-temp/contact': specifier: file:./projects/contact.tgz version: file:projects/contact.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1) @@ -3686,6 +3689,20 @@ packages: - utf-8-validate dev: false + /@hocuspocus/transformer@2.8.1(@tiptap/pm@2.1.12)(y-prosemirror@1.2.1)(yjs@13.6.8): + resolution: {integrity: sha512-0WtE/fxGwD/BEdl22udznD+skggvF6D0kEpw7jTEUdfvqwSJEyUxq3tm1S1hVmZyQI21qDw6yTTgP7Ujf1WbSg==} + peerDependencies: + '@tiptap/pm': ^2.1.12 + y-prosemirror: ^1.2.1 + yjs: ^13.6.8 + dependencies: + '@tiptap/core': 2.1.12(@tiptap/pm@2.1.12) + '@tiptap/pm': 2.1.12 + '@tiptap/starter-kit': 2.1.12(@tiptap/pm@2.1.12) + y-prosemirror: 1.2.1(prosemirror-model@1.19.3)(yjs@13.6.8) + yjs: 13.6.8 + dev: false + /@humanwhocodes/config-array@0.11.11: resolution: {integrity: sha512-N2brEuAadi0CcdeMXUkhbZB84eskAc8MEX1By6qEchoVywSgXPIjou4rYsl0V3Hj0ZnuGycGCjdNgockbzeWNA==} engines: {node: '>=10.10.0'} @@ -17202,7 +17219,7 @@ packages: yjs: ^13.0.0 dependencies: level: 6.0.1 - lib0: 0.2.86 + lib0: 0.2.88 yjs: 13.6.8 dev: false optional: true @@ -17227,7 +17244,7 @@ packages: peerDependencies: yjs: ^13.0.0 dependencies: - lib0: 0.2.86 + lib0: 0.2.88 yjs: 13.6.8 dev: false @@ -17406,7 +17423,7 @@ packages: dev: false file:projects/activity-resources.tgz(@types/node@16.11.68)(esbuild@0.16.17)(postcss-load-config@4.0.1)(postcss@8.4.31)(ts-node@10.9.1): - resolution: {integrity: sha512-F4DXewrthjnPi36RuMEgym+/VACMDSyCmP8QvXHUPvwu7x113/rtbhzOhSPSaj+jwZCzMiKTKXOU68B8Q1j+fw==, tarball: file:projects/activity-resources.tgz} + resolution: {integrity: sha512-S3azsw6dQwDQHF9dH26Ffp/Yje78ZWyL2Gv1xw9FRphd7bwsG8cz3/KgKHVjJ52Bg8lsuSAQZsUbUJqmn+U+QQ==, tarball: file:projects/activity-resources.tgz} id: file:projects/activity-resources.tgz name: '@rush-temp/activity-resources' version: 0.0.0 @@ -18143,13 +18160,16 @@ packages: - ts-node dev: false - file:projects/collaborator.tgz(bufferutil@4.0.7)(svelte@4.2.5): - resolution: {integrity: sha512-Aqxt81MeApDyPNFAK2c+YaKTe5kbBZdHrL/7UgrXClC3hqN7tVepxeWK77TaSz/Sk9/1+dOYphDBMBYK5jOxGQ==, tarball: file:projects/collaborator.tgz} + file:projects/collaborator.tgz(@tiptap/pm@2.1.12)(bufferutil@4.0.7)(prosemirror-model@1.19.3)(svelte@4.2.5): + resolution: {integrity: sha512-37tcslNB1OuZncrABis1O54JHogvoU8oT3FYo91epmBvzoriNrS6wYuV2Odc+UI4frX6bZGQJEIYxmNYLNMajQ==, tarball: file:projects/collaborator.tgz} id: file:projects/collaborator.tgz name: '@rush-temp/collaborator' version: 0.0.0 dependencies: '@hocuspocus/server': 2.8.1(bufferutil@4.0.7)(yjs@13.6.8) + '@hocuspocus/transformer': 2.8.1(@tiptap/pm@2.1.12)(y-prosemirror@1.2.1)(yjs@13.6.8) + '@tiptap/core': 2.1.12(@tiptap/pm@2.1.12) + '@tiptap/html': 2.1.12(@tiptap/core@2.1.12)(@tiptap/pm@2.1.12) '@types/body-parser': 1.19.3 '@types/compression': 1.7.3 '@types/cors': 2.8.14 @@ -18171,22 +18191,29 @@ packages: eslint-plugin-promise: 6.1.1(eslint@8.54.0) express: 4.18.2 jest: 29.7.0(@types/node@16.11.68)(ts-node@10.9.1) + mongodb: 4.17.1 prettier: 3.1.0 prettier-plugin-svelte: 3.1.0(prettier@3.1.0)(svelte@4.2.5) ts-jest: 29.1.1(esbuild@0.16.17)(jest@29.7.0)(typescript@5.2.2) ts-node: 10.9.1(@types/node@16.11.68)(typescript@5.2.2) typescript: 5.2.2 ws: 8.14.2(bufferutil@4.0.7) + y-prosemirror: 1.2.1(prosemirror-model@1.19.3)(yjs@13.6.8) yjs: 13.6.8 transitivePeerDependencies: - '@babel/core' - '@jest/types' - '@swc/core' - '@swc/wasm' + - '@tiptap/pm' + - aws-crt - babel-jest - babel-plugin-macros - bufferutil - node-notifier + - prosemirror-model + - prosemirror-state + - prosemirror-view - supports-color - svelte - utf-8-validate @@ -19447,7 +19474,7 @@ packages: dev: false file:projects/model-all.tgz(svelte@4.2.5)(typescript@5.2.2): - resolution: {integrity: sha512-PEl4zAGjvSsQ5stSRUuj2PYcZ0BdWTkxyaJNP63dbX4U6QA5sf7BuXamJRgETTD74jA40pGRXUAMnprS6t9z4w==, tarball: file:projects/model-all.tgz} + resolution: {integrity: sha512-bqRngFhfXoPedb0IMaXya9LnZ5agvGBsd49EUmpJuM/BWIDMW8Oj4iU2fnkTVYQEiBsprhXX1POk4chGR6dHvA==, tarball: file:projects/model-all.tgz} id: file:projects/model-all.tgz name: '@rush-temp/model-all' version: 0.0.0 @@ -19811,7 +19838,7 @@ packages: dev: false file:projects/model-server-activity.tgz(svelte@4.2.5)(typescript@5.2.2): - resolution: {integrity: sha512-ddPj4Y8/1G2ItDMmgLbaasdkjTTY66I3XKmyV3KfZNjeRq7DJZquF5VG1GM09TpfAkQkLJBJcoHoqtqQ9OgluA==, tarball: file:projects/model-server-activity.tgz} + resolution: {integrity: sha512-n6L6M7urgihLXLCFZWoh8LlVt+8dQHiGaehmAG4KJ1X3pz8xmSnZAjUAeLdXZvuRHFcfTPyHv0YzDBJ+npXxHg==, tarball: file:projects/model-server-activity.tgz} id: file:projects/model-server-activity.tgz name: '@rush-temp/model-server-activity' version: 0.0.0 @@ -20021,7 +20048,7 @@ packages: dev: false file:projects/model-server-notification.tgz(svelte@4.2.5)(typescript@5.2.2): - resolution: {integrity: sha512-BmWHvQtuRizjJE87k3thPWrhHJBfOEx1tg62HmPHUMsWxoS6nBKVzTa4zzwFqHPb4tlzBS7S8ux1n5tTwtJiMQ==, tarball: file:projects/model-server-notification.tgz} + resolution: {integrity: sha512-csNOi/1jDDopp9M7/9nlneRsjWLZ8vMj16CsAYirsR7PtPRYXoC0jLwzIUCnscdJry8KH7l9oZm8DbszdunklA==, tarball: file:projects/model-server-notification.tgz} id: file:projects/model-server-notification.tgz name: '@rush-temp/model-server-notification' version: 0.0.0 @@ -20294,7 +20321,7 @@ packages: dev: false file:projects/model-survey.tgz(svelte@4.2.5)(typescript@5.2.2): - resolution: {integrity: sha512-Q15RJl+fvfRN7hsIQUZ4UzcqF524kBjhNiaFimmUJCkTlOjhUhhn04pjcSFsh7g+7KbVfZW3CyD7L9ZQVqHp4w==, tarball: file:projects/model-survey.tgz} + resolution: {integrity: sha512-HqvLuhQJO8PpOeuDEPr7dcrxA2xZKXp7FLIwdckKONYx506mZVzErCiU3eXy3M4YyLXIQCTy272vnxJZnemJTw==, tarball: file:projects/model-survey.tgz} id: file:projects/model-survey.tgz name: '@rush-temp/model-survey' version: 0.0.0 @@ -20336,7 +20363,7 @@ packages: dev: false file:projects/model-task.tgz(svelte@4.2.5)(typescript@5.2.2): - resolution: {integrity: sha512-56qSfouGLl0idf3X7qFQVjeAmCT0eg2iDPoErUQzeQpaYgJ2sbI1wp0Czj3hDm83NlA0eiIYkOIgqtHuSULM7w==, tarball: file:projects/model-task.tgz} + resolution: {integrity: sha512-P62KW6rPhZBuQdKMpqqnyMoKzw3Zd2057wyJxWQ0jOB9q9dd0XgFtsmMWAX/18O83aFY92sbqoPg/EPGTLV6zA==, tarball: file:projects/model-task.tgz} id: file:projects/model-task.tgz name: '@rush-temp/model-task' version: 0.0.0 @@ -20977,7 +21004,7 @@ packages: dev: false file:projects/pod-server.tgz(svelte@4.2.5): - resolution: {integrity: sha512-7Griqc83Xh/jubUWlO8qA+Cx1+2pjzZgouGteeQZXfCokol6X813h3O5vkKO3fenjg6doFII5zI+DJx1R4jBnQ==, tarball: file:projects/pod-server.tgz} + resolution: {integrity: sha512-P3qTbPOzsBBFCp/JsSNERRx6ITah+3bfpBDqPgP4dzydWPEcbW9AALPYjrYviv/GRZuFzgcBu32ZxrUMCZ6DwA==, tarball: file:projects/pod-server.tgz} id: file:projects/pod-server.tgz name: '@rush-temp/pod-server' version: 0.0.0 @@ -21125,7 +21152,7 @@ packages: dev: false file:projects/prod.tgz(bufferutil@4.0.7)(esbuild@0.16.17)(sass@1.69.0)(ts-node@10.9.1)(typescript@5.2.2): - resolution: {integrity: sha512-AMQUJ8O71HSvI9iGDSj/32iyRs75Qepku6ir0Xq6n2/IwVW4BE3n0Fzhh5tfFTsbPtjL2p4qcA0mmJ0zg28D+g==, tarball: file:projects/prod.tgz} + resolution: {integrity: sha512-RpgrKzlPUDZ5L+pT2NPQmXxUomiw84N6898NdEYNbbXERliDL6II+7Qw9LwoEBHtVDChgnU8ez+5WcuknHrjlw==, tarball: file:projects/prod.tgz} id: file:projects/prod.tgz name: '@rush-temp/prod' version: 0.0.0 @@ -21526,7 +21553,7 @@ packages: dev: false file:projects/server-activity.tgz(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1): - resolution: {integrity: sha512-DneWyxHhW1vXrtKGwcLBuE7tkcbRkQVg4L/X4YamviGOuSWNSwUMb4KeJk7tqRhNwt0ldg3GcXvP7p01oPyVnA==, tarball: file:projects/server-activity.tgz} + resolution: {integrity: sha512-VDiyr4T43YaFFBX7WNa9eesnQK7F2dij3fbm62nMM+RV4iQBzOMJwBWAZAGEnHAkPQNMKSwnzqUf2N0barvTjg==, tarball: file:projects/server-activity.tgz} id: file:projects/server-activity.tgz name: '@rush-temp/server-activity' version: 0.0.0 @@ -23169,7 +23196,7 @@ packages: dev: false file:projects/survey-assets.tgz(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1)(typescript@5.2.2): - resolution: {integrity: sha512-ERl1tCl6UFla9peQ8yjEtMIgDr6AhFGxfp5qEH7yi7H0CFrBYddSj5euGsvHwYnowtshRsAI5YKw4sUhEeTVxA==, tarball: file:projects/survey-assets.tgz} + resolution: {integrity: sha512-KqJbIkEBJ7f4dqGEd1ztPbMCP6l6l+wdYOTr83WN37YOUj8ucMuTcDWcZv1aCMZl/T/C6EUIDf5TgYvl1CHjcw==, tarball: file:projects/survey-assets.tgz} id: file:projects/survey-assets.tgz name: '@rush-temp/survey-assets' version: 0.0.0 @@ -23201,7 +23228,7 @@ packages: dev: false file:projects/survey-resources.tgz(@types/node@16.11.68)(esbuild@0.16.17)(postcss-load-config@4.0.1)(postcss@8.4.31)(svelte@4.2.5)(ts-node@10.9.1)(typescript@5.2.2): - resolution: {integrity: sha512-7TYqfVbpbPrAl/+fBGHT78DkkQo5+ZdIO7sIf+Z7UVEoNYzmnZnM0VCmnzdpAeOjZ64Z+kLG3onaLYbFf6awDQ==, tarball: file:projects/survey-resources.tgz} + resolution: {integrity: sha512-S4t+6+J3q1YibQwIzBOYpeWlSPTiFyvVhnhw07Eqnxy1z3ADHLlHpsxCXUI+zQQQzzo9zvGLRoK1hSYmtcPGfA==, tarball: file:projects/survey-resources.tgz} id: file:projects/survey-resources.tgz name: '@rush-temp/survey-resources' version: 0.0.0 @@ -23246,7 +23273,7 @@ packages: dev: false file:projects/survey.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1)(typescript@5.2.2): - resolution: {integrity: sha512-meZo7FpLblppewqgFsGx3Mh6AS7deFcxC8D9vnRRWliUuGpYfq+7K3/T++j38D2RHZVGYnJTjHJnCMXTnTSPfA==, tarball: file:projects/survey.tgz} + resolution: {integrity: sha512-VQ5Ne0MuO8zCvfvMqROddHTfinHEmiybMnz3XM/YfxOTxU1RRwZDGSoaixBU/tK98BURHp+NDnCiD5X3Q48DtQ==, tarball: file:projects/survey.tgz} id: file:projects/survey.tgz name: '@rush-temp/survey' version: 0.0.0 @@ -23420,7 +23447,7 @@ packages: dev: false file:projects/task-resources.tgz(@types/node@16.11.68)(esbuild@0.16.17)(postcss-load-config@4.0.1)(postcss@8.4.31)(ts-node@10.9.1)(typescript@5.2.2): - resolution: {integrity: sha512-rmEmBZU8K4rPiePhlw99tsYnCiYoXlymTECYCm29m3zOEcKsLptEL5EPcI1YgJd6+83LOfAEHxYFLXcFgXWe+A==, tarball: file:projects/task-resources.tgz} + resolution: {integrity: sha512-sXLi/UksIrEaJL0K+aSQcqUSm8jJv9qeyEEJMiENDxjwvCDzONTz8HPwFjRVUdPSH6Mnz9YpNRwQyIDOpni0qg==, tarball: file:projects/task-resources.tgz} id: file:projects/task-resources.tgz name: '@rush-temp/task-resources' version: 0.0.0 @@ -23466,7 +23493,7 @@ packages: dev: false file:projects/task.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1): - resolution: {integrity: sha512-IH/xqCjSTuVcbWA+pbE/dcZeChW7N4+jJiVCbwGDB605yipH/IDYSz1fsf+evXbkBm8lSHBnrzQf2ovVWk+Iig==, tarball: file:projects/task.tgz} + resolution: {integrity: sha512-qtjMbTtDdi6Zz2GIi6HXzFOVPYjrC35i3m/3E6cJcbFG7LzRv/QU9guhZl5M3zuSmymP0m6wxtJ3C4CiboCMJQ==, tarball: file:projects/task.tgz} id: file:projects/task.tgz name: '@rush-temp/task' version: 0.0.0 @@ -23746,7 +23773,7 @@ packages: dev: false file:projects/text-editor.tgz(@types/node@16.11.68)(bufferutil@4.0.7)(esbuild@0.16.17)(postcss-load-config@4.0.1)(postcss@8.4.31)(prosemirror-model@1.19.3)(ts-node@10.9.1): - resolution: {integrity: sha512-s3a0AnlJS8WBnLjA5ggHaLcbSh9tT9DA6Szy9wYWGJzt9R82/nfhh4qMOLWbvHbTgOu/ieWGrhG2FpWAzHSwzw==, tarball: file:projects/text-editor.tgz} + resolution: {integrity: sha512-42rs06vaqb/uM49NzAf4ggZxatYbBqfT2Qy0rZLsK72tSUAxZC86E97pxUjYaLKguUoAGkou/bQf2iOFySZQwA==, tarball: file:projects/text-editor.tgz} id: file:projects/text-editor.tgz name: '@rush-temp/text-editor' version: 0.0.0 diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 5ab0f4de31..1e147ee4a4 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -94,6 +94,7 @@ services: collaborator: image: hardcoreeng/collaborator links: + - mongodb - minio - transactor ports: @@ -102,6 +103,7 @@ services: - COLLABORATOR_PORT=3078 - SECRET=secret - TRANSACTOR_URL=ws://localhost:3333 + - MONGO_URL=mongodb://mongodb:27017 - MINIO_ENDPOINT=minio - MINIO_ACCESS_KEY=minioadmin - MINIO_SECRET_KEY=minioadmin diff --git a/models/core/src/core.ts b/models/core/src/core.ts index 942db4dbe6..0215f7b635 100644 --- a/models/core/src/core.ts +++ b/models/core/src/core.ts @@ -199,6 +199,10 @@ export class TTypeNumber extends TType {} @Model(core.class.TypeMarkup, core.class.Type) export class TTypeMarkup extends TType {} +@UX(core.string.Collaborative) +@Model(core.class.TypeCollaborativeMarkup, core.class.Type) +export class TTypeCollaborativeMarkup extends TType {} + @UX(core.string.Ref) @Model(core.class.RefTo, core.class.Type) export class TRefTo extends TType implements RefTo { diff --git a/models/core/src/index.ts b/models/core/src/index.ts index 53d9c5924e..ac4e906398 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -51,6 +51,7 @@ import { TType, TTypeAttachment, TTypeBoolean, + TTypeCollaborativeMarkup, TTypeDate, TTypeHyperlink, TTypeIntlString, @@ -108,6 +109,7 @@ export function createModel (builder: Builder): void { TType, TEnumOf, TTypeMarkup, + TTypeCollaborativeMarkup, TArrOf, TRefTo, TTypeDate, diff --git a/models/lead/src/index.ts b/models/lead/src/index.ts index 038a923d43..764e27f014 100644 --- a/models/lead/src/index.ts +++ b/models/lead/src/index.ts @@ -25,6 +25,7 @@ import { Model, Prop, ReadOnly, + TypeCollaborativeMarkup, TypeDate, TypeMarkup, TypeRef, @@ -94,7 +95,7 @@ export class TCustomer extends TContact implements Customer { @Prop(Collection(lead.class.Lead), lead.string.Leads) leads?: number - @Prop(TypeMarkup(), core.string.Description) + @Prop(TypeCollaborativeMarkup(), core.string.Description) @Index(IndexKind.FullText) description!: string } diff --git a/models/recruit/src/index.ts b/models/recruit/src/index.ts index 07bcef106c..9b329b73ee 100644 --- a/models/recruit/src/index.ts +++ b/models/recruit/src/index.ts @@ -25,6 +25,7 @@ import { Prop, ReadOnly, TypeBoolean, + TypeCollaborativeMarkup, TypeDate, TypeMarkup, TypeRef, @@ -69,7 +70,7 @@ export { default } from './plugin' @Model(recruit.class.Vacancy, task.class.Project) @UX(recruit.string.Vacancy, recruit.icon.Vacancy, 'VCN', 'name') export class TVacancy extends TProject implements Vacancy { - @Prop(TypeMarkup(), recruit.string.FullDescription) + @Prop(TypeCollaborativeMarkup(), recruit.string.FullDescription) @Index(IndexKind.FullText) fullDescription?: string diff --git a/models/tracker/src/types.ts b/models/tracker/src/types.ts index aac243f985..71f1feb48c 100644 --- a/models/tracker/src/types.ts +++ b/models/tracker/src/types.ts @@ -13,6 +13,7 @@ // limitations under the License. // +import chunter from '@hcengineering/chunter' import contact, { type Employee, type Person } from '@hcengineering/contact' import { DOMAIN_MODEL, @@ -33,6 +34,7 @@ import { Model, Prop, ReadOnly, + TypeCollaborativeMarkup, TypeDate, TypeMarkup, TypeNumber, @@ -64,7 +66,6 @@ import { type TimeSpendReport } from '@hcengineering/tracker' import tracker from './plugin' -import chunter from '@hcengineering/chunter' export const DOMAIN_TRACKER = 'tracker' as Domain @@ -170,7 +171,7 @@ export class TIssue extends TTask implements Issue { @Index(IndexKind.FullText) title!: string - @Prop(TypeMarkup(), tracker.string.Description) + @Prop(TypeCollaborativeMarkup(), tracker.string.Description) @Index(IndexKind.FullText) description!: Markup @@ -258,7 +259,7 @@ export class TIssueTemplate extends TDoc implements IssueTemplate { @Index(IndexKind.FullText) title!: string - @Prop(TypeMarkup(), tracker.string.Description) + @Prop(TypeCollaborativeMarkup(), tracker.string.Description) @Index(IndexKind.FullText) description!: Markup diff --git a/models/view/src/index.ts b/models/view/src/index.ts index f26068bc68..0c0f4f2208 100644 --- a/models/view/src/index.ts +++ b/models/view/src/index.ts @@ -477,6 +477,11 @@ export function createModel (builder: Builder): void { editor: view.component.HTMLEditor }) + classPresenter(builder, core.class.TypeCollaborativeMarkup, view.component.MarkupPresenter) + builder.mixin(core.class.TypeCollaborativeMarkup, core.class.Class, view.mixin.InlineAttributEditor, { + editor: view.component.CollaborativeHTMLEditor + }) + classPresenter(builder, core.class.TypeBoolean, view.component.BooleanPresenter, view.component.BooleanEditor) classPresenter( builder, diff --git a/models/view/src/plugin.ts b/models/view/src/plugin.ts index 1588df2b7d..cf2b9d6011 100644 --- a/models/view/src/plugin.ts +++ b/models/view/src/plugin.ts @@ -68,6 +68,7 @@ export default mergeIds(viewId, view, { EnumEditor: '' as AnyComponent, EnumArrayEditor: '' as AnyComponent, HTMLEditor: '' as AnyComponent, + CollaborativeHTMLEditor: '' as AnyComponent, MarkupEditor: '' as AnyComponent, MarkupEditorPopup: '' as AnyComponent, ListView: '' as AnyComponent, diff --git a/packages/core/lang/en.json b/packages/core/lang/en.json index 4df0182a32..e74635eda0 100644 --- a/packages/core/lang/en.json +++ b/packages/core/lang/en.json @@ -27,6 +27,7 @@ "Enum": "Enum", "Members": "Members", "Hyperlink": "URL", + "Collaborative": "Collaborative", "Object": "Object", "System": "System", "CreatedBy": "Created by", diff --git a/packages/core/lang/ru.json b/packages/core/lang/ru.json index eca5f476ca..f3a44e448a 100644 --- a/packages/core/lang/ru.json +++ b/packages/core/lang/ru.json @@ -27,6 +27,7 @@ "Enum": "Справочник", "Members": "Участники", "Hyperlink": "URL", + "Collaborative": "Коллаборативный", "Object": "Объект", "System": "Система", "CreatedBy": "Создан", diff --git a/packages/core/src/component.ts b/packages/core/src/component.ts index 2634c36ebe..66da30d6ce 100644 --- a/packages/core/src/component.ts +++ b/packages/core/src/component.ts @@ -35,6 +35,7 @@ import type { IndexStageState, IndexingConfiguration, Interface, + Markup, MigrationState, Obj, PluginConfiguration, @@ -102,6 +103,7 @@ export default plugin(coreId, { TypeBoolean: '' as Ref>>, TypeTimestamp: '' as Ref>>, TypeDate: '' as Ref>>, + TypeCollaborativeMarkup: '' as Ref>>, RefTo: '' as Ref>>, ArrOf: '' as Ref>>, Enum: '' as Ref>, @@ -158,6 +160,7 @@ export default plugin(coreId, { String: '' as IntlString, Record: '' as IntlString, Markup: '' as IntlString, + Collaborative: '' as IntlString, Number: '' as IntlString, Boolean: '' as IntlString, Timestamp: '' as IntlString, diff --git a/packages/model/src/dsl.ts b/packages/model/src/dsl.ts index b51264a8c0..1c101f68cf 100644 --- a/packages/model/src/dsl.ts +++ b/packages/model/src/dsl.ts @@ -392,6 +392,13 @@ export function TypeMarkup (): Type { return { _class: core.class.TypeMarkup, label: core.string.Markup } } +/** + * @public + */ +export function TypeCollaborativeMarkup (): Type { + return { _class: core.class.TypeCollaborativeMarkup, label: core.string.Collaborative } +} + /** * @public */ diff --git a/packages/presentation/src/utils.ts b/packages/presentation/src/utils.ts index cacfa8d957..a5596daa7c 100644 --- a/packages/presentation/src/utils.ts +++ b/packages/presentation/src/utils.ts @@ -373,6 +373,9 @@ export function getAttributePresenterClass ( if (hierarchy.isDerived(attrClass, core.class.TypeMarkup)) { category = 'inplace' } + if (hierarchy.isDerived(attrClass, core.class.TypeCollaborativeMarkup)) { + category = 'inplace' + } if (hierarchy.isDerived(attrClass, core.class.Collection)) { attrClass = (attribute.type as Collection).of category = 'collection' diff --git a/packages/text-editor/src/components/Collaboration.svelte b/packages/text-editor/src/components/Collaboration.svelte index 133a2b5e54..53b0078427 100644 --- a/packages/text-editor/src/components/Collaboration.svelte +++ b/packages/text-editor/src/components/Collaboration.svelte @@ -16,15 +16,19 @@ --> + + diff --git a/packages/text-editor/src/components/CollaborativeAttributeSectionBox.svelte b/packages/text-editor/src/components/CollaborativeAttributeSectionBox.svelte new file mode 100644 index 0000000000..be8733485a --- /dev/null +++ b/packages/text-editor/src/components/CollaborativeAttributeSectionBox.svelte @@ -0,0 +1,44 @@ + + + +
+
+
+ +
+ + +
+ +
diff --git a/packages/text-editor/src/components/CollaborativeTextEditor.svelte b/packages/text-editor/src/components/CollaborativeTextEditor.svelte new file mode 100644 index 0000000000..1c090503e6 --- /dev/null +++ b/packages/text-editor/src/components/CollaborativeTextEditor.svelte @@ -0,0 +1,393 @@ + + + + + +
(needFocus = true)} +> + {#if loading} +
+ +
+ {/if} + +
+ {#if showTextStyleToolbar} + + {/if} +
+ +
+ +
+ +
+
+
+ + {#if refActions.length > 0} +
+
+ {#each refActions as a} +
+ {/if} +
+ + diff --git a/packages/text-editor/src/components/CollaboratorEditor.svelte b/packages/text-editor/src/components/CollaboratorEditor.svelte index 1d52cd96f4..73d53a27aa 100644 --- a/packages/text-editor/src/components/CollaboratorEditor.svelte +++ b/packages/text-editor/src/components/CollaboratorEditor.svelte @@ -15,50 +15,30 @@ // --> - - -{#if loading} -
- -
-{/if} - -{#if visible} - {#if $$slots.tools} -
-
- -
-
- {/if} - - - - - -
-
-
-{/if} +
+ { + editor = evt.detail + }} + /> +
diff --git a/packages/text-editor/src/components/ReferenceInput.svelte b/packages/text-editor/src/components/ReferenceInput.svelte index 4aca118f42..8d6e938d72 100644 --- a/packages/text-editor/src/components/ReferenceInput.svelte +++ b/packages/text-editor/src/components/ReferenceInput.svelte @@ -13,8 +13,7 @@ // limitations under the License. --> -
- { - needFocus = true - }} - /> +
+
-
- { - needFocus = true - }} - /> +
+
diff --git a/plugins/activity-assets/lang/ru.json b/plugins/activity-assets/lang/ru.json index e52187ab23..c3296135f2 100644 --- a/plugins/activity-assets/lang/ru.json +++ b/plugins/activity-assets/lang/ru.json @@ -1,7 +1,7 @@ { "string": { "Activity": "Активность", - "Added": "Добавила(а)", + "Added": "Добавил(а)", "All": "Все", "AllActivity": "Вся активнось", "Attributes": "Атрибуты", diff --git a/plugins/activity-resources/src/activityMessagesUtils.ts b/plugins/activity-resources/src/activityMessagesUtils.ts index 1f5d5a52ff..8c17751d28 100644 --- a/plugins/activity-resources/src/activityMessagesUtils.ts +++ b/plugins/activity-resources/src/activityMessagesUtils.ts @@ -51,6 +51,7 @@ const valueTypes: ReadonlyArray>> = [ core.class.TypeNumber, core.class.TypeDate, core.class.TypeMarkup, + core.class.TypeCollaborativeMarkup, core.class.TypeHyperlink ] diff --git a/plugins/activity-resources/src/components/doc-update-message/attributes/SetAttributesPresenter.svelte b/plugins/activity-resources/src/components/doc-update-message/attributes/SetAttributesPresenter.svelte index be1f854c83..6bf907df39 100644 --- a/plugins/activity-resources/src/components/doc-update-message/attributes/SetAttributesPresenter.svelte +++ b/plugins/activity-resources/src/components/doc-update-message/attributes/SetAttributesPresenter.svelte @@ -32,7 +32,10 @@ $: isTextType = getIsTextType(attributeModel) function getIsTextType (attributeModel: AttributeModel): boolean { - return attributeModel.attribute?.type?._class === core.class.TypeMarkup + return ( + attributeModel.attribute?.type?._class === core.class.TypeMarkup || + attributeModel.attribute?.type?._class === core.class.TypeCollaborativeMarkup + ) } let isDiffShown = false diff --git a/plugins/activity-resources/src/utils.ts b/plugins/activity-resources/src/utils.ts index 132d0904ab..d296249e2f 100644 --- a/plugins/activity-resources/src/utils.ts +++ b/plugins/activity-resources/src/utils.ts @@ -33,7 +33,8 @@ const valueTypes: ReadonlyArray>> = [ core.class.EnumOf, core.class.TypeNumber, core.class.TypeDate, - core.class.TypeMarkup + core.class.TypeMarkup, + core.class.TypeCollaborativeMarkup ] export type TxDisplayViewlet = diff --git a/plugins/attachment-resources/src/components/AttachmentStyleBoxCollabEditor.svelte b/plugins/attachment-resources/src/components/AttachmentStyleBoxCollabEditor.svelte new file mode 100644 index 0000000000..6d56675a33 --- /dev/null +++ b/plugins/attachment-resources/src/components/AttachmentStyleBoxCollabEditor.svelte @@ -0,0 +1,255 @@ + + + + + +{#key object?._id} + +
pasteAction(ev)} + on:dragover|preventDefault={() => {}} + on:dragleave={() => {}} + on:drop|preventDefault|stopPropagation={(ev) => { + void fileDrop(ev) + }} + > + { + return await createAttachment(file) + }} + on:open-document={async (event) => { + const doc = await client.findOne(event.detail._class, { _id: event.detail._id }) + if (doc != null) { + const location = await getObjectLinkFragment(client.getHierarchy(), doc, {}, view.component.EditDoc) + navigate(location) + } + }} + on:focus + on:blur + on:update + /> + {#if (attachments.length > 0 && enableAttachments) || progress} + { + if (evt.detail !== undefined) { + await removeAttachment(evt.detail) + } + }} + /> + {/if} +
+{/key} diff --git a/plugins/attachment-resources/src/components/AttachmentStyledBox.svelte b/plugins/attachment-resources/src/components/AttachmentStyledBox.svelte index 9625b2dddc..82d3aff36c 100644 --- a/plugins/attachment-resources/src/components/AttachmentStyledBox.svelte +++ b/plugins/attachment-resources/src/components/AttachmentStyledBox.svelte @@ -19,12 +19,10 @@ import { IntlString, setPlatformStatus, unknownError } from '@hcengineering/platform' import { createQuery, DraftController, draftsStore, getClient } from '@hcengineering/presentation' import textEditor, { AttachIcon, type RefAction, StyledTextBox } from '@hcengineering/text-editor' - import { ButtonSize, Loading, updatePopup, Scroller } from '@hcengineering/ui' - import { ListSelectionProvider, SelectDirection } from '@hcengineering/view-resources' + import { ButtonSize } from '@hcengineering/ui' import attachment from '../plugin' import { deleteFile, uploadFile } from '../utils' - import AttachmentPresenter from './AttachmentPresenter.svelte' - import AttachmentPreview from './AttachmentPreview.svelte' + import AttachmentsGrid from './AttachmentsGrid.svelte' export let objectId: Ref | undefined = undefined export let space: Ref | undefined = undefined @@ -54,18 +52,6 @@ $: draftKey = objectId ? `${objectId}_attachments` : undefined const dispatch = createEventDispatcher() - let attachmentPopupId: string = '' - const listProvider = new ListSelectionProvider((offset: 1 | -1 | 0, of?: Doc, dir?: SelectDirection) => { - const currentAttachmentIndex = listProvider.current() - if (currentAttachmentIndex === undefined) return - const selected = currentAttachmentIndex + offset - const sel = listProvider.docs()[selected] as Attachment - if (sel !== undefined && attachmentPopupId !== '') { - listProvider.updateFocus(sel) - updatePopup(attachmentPopupId, { props: { file: sel.file, name: sel.name, contentType: sel.type } }) - } - }) - $: listProvider.update(Array.from(attachments.values()).filter((attachment) => attachment.type.startsWith('image/'))) export function focus (): void { refInput.focus() @@ -358,7 +344,6 @@ extraActions = [] } - let element: HTMLElement let progressItems: Ref[] = [] @@ -382,90 +367,41 @@ fileDrop(ev) }} > -
- { - return await createAttachment(file) + { + return await createAttachment(file) + }} + /> + {#if (attachments.size > 0 && enableAttachments) || progress} + { + if (evt.detail !== undefined) { + await removeAttachment(evt.detail) + } }} /> -
- {#if (attachments.size && enableAttachments) || progress} -
- -
- {#each Array.from(attachments.values()) as attachment, index} - {#if useAttachmentPreview} - (attachmentPopupId = res.detail)} - /> - {:else} - { - if (result !== undefined) { - removeAttachment(attachment) - } - }} - /> - {/if} - {/each} - {#if progress} -
- { - element.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'center' }) - }} - /> -
- {/if} -
- -
{/if}
- - diff --git a/plugins/attachment-resources/src/components/AttachmentsGrid.svelte b/plugins/attachment-resources/src/components/AttachmentsGrid.svelte new file mode 100644 index 0000000000..ee9afa5fbd --- /dev/null +++ b/plugins/attachment-resources/src/components/AttachmentsGrid.svelte @@ -0,0 +1,93 @@ + + + +
+ +
+ {#each attachments as attachment (attachment._id)} + {#if useAttachmentPreview} + (attachmentPopupId = res.detail)} /> + {:else} + + {/if} + {/each} + {#if progress} +
+ { + element.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'center' }) + }} + /> +
+ {/if} +
+ +
+ + diff --git a/plugins/attachment-resources/src/index.ts b/plugins/attachment-resources/src/index.ts index 4516d482e7..575e478880 100644 --- a/plugins/attachment-resources/src/index.ts +++ b/plugins/attachment-resources/src/index.ts @@ -35,6 +35,7 @@ import FileDownload from './components/icons/FileDownload.svelte' import Photos from './components/Photos.svelte' import AttachmentStyledBox from './components/AttachmentStyledBox.svelte' import AttachmentStyleBoxEditor from './components/AttachmentStyleBoxEditor.svelte' +import AttachmentStyleBoxCollabEditor from './components/AttachmentStyleBoxCollabEditor.svelte' import AccordionEditor from './components/AccordionEditor.svelte' import IconUploadDuo from './components/icons/UploadDuo.svelte' import IconAttachment from './components/icons/Attachment.svelte' @@ -56,6 +57,7 @@ export { FileBrowser, AttachmentStyledBox, AttachmentStyleBoxEditor, + AttachmentStyleBoxCollabEditor, AccordionEditor, IconUploadDuo, IconAttachment, diff --git a/plugins/contact-resources/src/components/Members.svelte b/plugins/contact-resources/src/components/Members.svelte index 891f99751c..1a5c65464a 100644 --- a/plugins/contact-resources/src/components/Members.svelte +++ b/plugins/contact-resources/src/components/Members.svelte @@ -16,7 +16,7 @@ import { Member } from '@hcengineering/contact' import type { Class, Doc, Ref, Space } from '@hcengineering/core' import { createQuery, getClient } from '@hcengineering/presentation' - import { Button, Icon, IconAdd, Label, showPopup } from '@hcengineering/ui' + import { Button, IconAdd, Label, Section, showPopup } from '@hcengineering/ui' import { Viewlet, ViewletPreference } from '@hcengineering/view' import { Table, ViewletSelector, ViewletSettingButton } from '@hcengineering/view-resources' import contact from '../plugin' @@ -66,14 +66,8 @@ let preference: ViewletPreference | undefined -
-
-
- -
- - +
+
-
- {#if members > 0 && viewlet} - - {:else} -
- - - - - - -
- {/if} - + + + + {#if members > 0 && viewlet} +
+ {:else} +
+ + + + + + +
+ {/if} + + diff --git a/plugins/contact-resources/src/components/MembersBox.svelte b/plugins/contact-resources/src/components/MembersBox.svelte index 455976f8bd..eaee512874 100644 --- a/plugins/contact-resources/src/components/MembersBox.svelte +++ b/plugins/contact-resources/src/components/MembersBox.svelte @@ -15,7 +15,7 @@ -
-
-
- -
- - -
- -
+
+ + + +
diff --git a/plugins/hr-resources/src/components/DepartmentStaff.svelte b/plugins/hr-resources/src/components/DepartmentStaff.svelte index e19e0bf9eb..f9ca02ebd4 100644 --- a/plugins/hr-resources/src/components/DepartmentStaff.svelte +++ b/plugins/hr-resources/src/components/DepartmentStaff.svelte @@ -18,7 +18,7 @@ import { Ref, WithLookup } from '@hcengineering/core' import { Department, Staff } from '@hcengineering/hr' import { createQuery, getClient } from '@hcengineering/presentation' - import { Button, IconAdd, Label, Scroller, eventToHTMLElement, showPopup } from '@hcengineering/ui' + import { Button, IconAdd, Label, Scroller, Section, eventToHTMLElement, showPopup } from '@hcengineering/ui' import { Viewlet, ViewletPreference } from '@hcengineering/view' import { Table, ViewletSelector, ViewletSettingButton } from '@hcengineering/view-resources' import hr from '../plugin' @@ -67,11 +67,8 @@ let viewlet: WithLookup | undefined -
-
- - +
+
-
- {#if (value?.members.length ?? 0) > 0} - -
- - {:else} -
- - - - - -
- {/if} - + + + + {#if (value?.members.length ?? 0) > 0} + +
+ + {:else} +
+ + + + + + +
+ {/if} + + diff --git a/plugins/recruit-resources/src/components/Applications.svelte b/plugins/recruit-resources/src/components/Applications.svelte index 21ce90c71c..490e05fd87 100644 --- a/plugins/recruit-resources/src/components/Applications.svelte +++ b/plugins/recruit-resources/src/components/Applications.svelte @@ -14,7 +14,7 @@ --> -
-
-
- -
- - +
+
-
- {#if applications > 0 && viewlet && !loading} - -
- - {:else} -
-
- + + + + {#if applications > 0 && viewlet && !loading} + +
+ + {:else} +
+
+ +
+ + + + + +
- - - - - - - {/if} - + {/if} + + diff --git a/plugins/recruit-resources/src/components/EditVacancy.svelte b/plugins/recruit-resources/src/components/EditVacancy.svelte index 6fd418140b..d929afe629 100644 --- a/plugins/recruit-resources/src/components/EditVacancy.svelte +++ b/plugins/recruit-resources/src/components/EditVacancy.svelte @@ -14,7 +14,7 @@ // limitations under the License. --> + +{#key object._id} + +{/key} diff --git a/plugins/view-resources/src/components/EditDoc.svelte b/plugins/view-resources/src/components/EditDoc.svelte index 53b724e034..bf0dcec71b 100644 --- a/plugins/view-resources/src/components/EditDoc.svelte +++ b/plugins/view-resources/src/components/EditDoc.svelte @@ -61,7 +61,7 @@ } onDestroy(async () => { - notificationClient.then(async (client) => { + await notificationClient.then(async (client) => { await client.read(_id) }) }) diff --git a/plugins/view-resources/src/index.ts b/plugins/view-resources/src/index.ts index 191f399483..226c56c336 100644 --- a/plugins/view-resources/src/index.ts +++ b/plugins/view-resources/src/index.ts @@ -42,6 +42,7 @@ import StringFilter from './components/filter/StringFilter.svelte' import StringFilterPresenter from './components/filter/StringFilterPresenter.svelte' import TimestampFilter from './components/filter/TimestampFilter.svelte' import ValueFilter from './components/filter/ValueFilter.svelte' +import CollaborativeHTMLEditor from './components/CollaborativeHTMLEditor.svelte' import HTMLEditor from './components/HTMLEditor.svelte' import HTMLPresenter from './components/HTMLPresenter.svelte' import HyperlinkPresenter from './components/HyperlinkPresenter.svelte' @@ -238,6 +239,7 @@ export default async (): Promise => ({ FilterTypePopup, ValueSelector, HTMLEditor, + CollaborativeHTMLEditor, ListView, GrowPresenter, DividerPresenter, diff --git a/pods/collaborator/package.json b/pods/collaborator/package.json index 3cefa1ed22..6f61b0323f 100644 --- a/pods/collaborator/package.json +++ b/pods/collaborator/package.json @@ -12,8 +12,7 @@ "docker:build": "docker build -t hardcoreeng/collaborator .", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator staging", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator", - "run-local": "cross-env TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin ts-node src/__start.ts", - "run-bundle": "cross-env TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin node ./bundle.js", + "run-local": "cross-env MONGO_URL=mongodb://localhost:27017 TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin ts-node src/__start.ts", "lint": "eslint src", "format": "format src", "test": "jest --passWithNoTests --silent" diff --git a/pods/collaborator/src/__start.ts b/pods/collaborator/src/__start.ts index 474f0679e7..a2a1c1557d 100644 --- a/pods/collaborator/src/__start.ts +++ b/pods/collaborator/src/__start.ts @@ -14,4 +14,4 @@ // import { startCollaborator } from '@hcengineering/collaborator' -startCollaborator() +void startCollaborator() diff --git a/server-plugins/chunter-resources/src/index.ts b/server-plugins/chunter-resources/src/index.ts index 8a39a4c7a6..6db2d648e7 100644 --- a/server-plugins/chunter-resources/src/index.ts +++ b/server-plugins/chunter-resources/src/index.ts @@ -42,7 +42,8 @@ import core, { TxFactory, TxProcessor, TxRemoveDoc, - TxUpdateDoc + TxUpdateDoc, + Type } from '@hcengineering/core' import notification, { Collaborators, NotificationType, NotificationContent } from '@hcengineering/notification' import { getMetadata, IntlString } from '@hcengineering/platform' @@ -54,6 +55,10 @@ import { getBacklinks, getBacklinksTxes } from './backlinks' export { getBacklinksTxes } from './backlinks' +function isMarkupType (type: Ref>>): boolean { + return type === core.class.TypeMarkup || type === core.class.TypeCollaborativeMarkup +} + function getCreateBacklinksTxes ( control: TriggerControl, txFactory: TxFactory, @@ -66,7 +71,7 @@ function getCreateBacklinksTxes ( const backlinks: Data[] = [] const attributes = control.hierarchy.getAllAttributes(doc._class) for (const attr of attributes.values()) { - if (attr.type._class === core.class.TypeMarkup) { + if (isMarkupType(attr.type._class)) { const content = (doc as any)[attr.name]?.toString() ?? '' const attrBacklinks = getBacklinks(backlinkId, backlinkClass, attachedDocId, content) backlinks.push(...attrBacklinks) @@ -90,7 +95,7 @@ async function getUpdateBacklinksTxes ( const backlinks: Data[] = [] const attributes = control.hierarchy.getAllAttributes(doc._class) for (const attr of attributes.values()) { - if (attr.type._class === core.class.TypeMarkup) { + if (isMarkupType(attr.type._class)) { hasBacklinkAttrs = true const content = (doc as any)[attr.name]?.toString() ?? '' const attrBacklinks = getBacklinks(backlinkId, backlinkClass, attachedDocId, content) @@ -318,7 +323,7 @@ async function BacklinksUpdate (tx: Tx, control: TriggerControl): Promise let hasUpdates = false const attributes = control.hierarchy.getAllAttributes(ctx.objectClass) for (const attr of attributes.values()) { - if (attr.type._class === core.class.TypeMarkup && attr.name in ctx.operations) { + if (isMarkupType(attr.type._class) && attr.name in ctx.operations) { hasUpdates = true break } @@ -349,7 +354,7 @@ async function BacklinksRemove (tx: Tx, control: TriggerControl): Promise let hasMarkdown = false const attributes = control.hierarchy.getAllAttributes(ctx.objectClass) for (const attr of attributes.values()) { - if (attr.type._class === core.class.TypeMarkup) { + if (isMarkupType(attr.type._class)) { hasMarkdown = true break } diff --git a/server/collaborator/package.json b/server/collaborator/package.json index 9f58b229df..c2f1ff0772 100644 --- a/server/collaborator/package.json +++ b/server/collaborator/package.json @@ -12,8 +12,7 @@ "docker:build": "docker build -t hardcoreeng/collaborator .", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator staging", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator", - "run-local": "cross-env TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin ts-node src/__start.ts", - "run-bundle": "cross-env TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin node ./bundle.js", + "run-local": "cross-env MONGO_URL=mongodb://localhost:27017 TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin ts-node src/__start.ts", "lint": "eslint src", "format": "format src", "test": "jest --passWithNoTests --silent" @@ -54,8 +53,14 @@ "@hcengineering/client": "^0.6.14", "@hcengineering/client-resources": "^0.6.23", "@hcengineering/minio": "^0.6.0", - "yjs": "^13.5.52", + "@hcengineering/text": "^0.6.1", "@hocuspocus/server": "^2.5.0", + "@hocuspocus/transformer": "^2.5.0", + "@tiptap/core": "^2.1.12", + "@tiptap/html": "^2.1.12", + "mongodb": "^4.11.0", + "yjs": "^13.5.52", + "y-prosemirror": "^1.2.1", "express": "^4.17.1", "body-parser": "~1.19.1", "cors": "^2.8.5", diff --git a/server/collaborator/src/__start.ts b/server/collaborator/src/__start.ts index 96bc1d1239..5b91ae1d98 100644 --- a/server/collaborator/src/__start.ts +++ b/server/collaborator/src/__start.ts @@ -15,4 +15,4 @@ // import { startCollaborator } from './starter' -startCollaborator() +void startCollaborator() diff --git a/server/collaborator/src/config.ts b/server/collaborator/src/config.ts index 8a4213c9ec..64077c98cf 100644 --- a/server/collaborator/src/config.ts +++ b/server/collaborator/src/config.ts @@ -25,6 +25,7 @@ export interface Config { Port: number TransactorUrl: string + MongoUrl: string MinioEndpoint: string MinioAccessKey: string @@ -37,6 +38,7 @@ const envMap: { [key in keyof Config]: string } = { Interval: 'INTERVAL', Port: 'COLLABORATOR_PORT', TransactorUrl: 'TRANSACTOR_URL', + MongoUrl: 'MONGO_URL', MinioEndpoint: 'MINIO_ENDPOINT', MinioAccessKey: 'MINIO_ACCESS_KEY', MinioSecretKey: 'MINIO_SECRET_KEY' @@ -47,6 +49,7 @@ const required: Array = [ 'ServiceID', 'Port', 'TransactorUrl', + 'MongoUrl', 'MinioEndpoint', 'MinioAccessKey', 'MinioSecretKey' @@ -59,6 +62,7 @@ const config: Config = (() => { Interval: parseInt(process.env[envMap.Interval] ?? '30000'), Port: parseInt(process.env[envMap.Port] ?? '3078'), TransactorUrl: process.env[envMap.TransactorUrl], + MongoUrl: process.env[envMap.MongoUrl], MinioEndpoint: process.env[envMap.MinioEndpoint], MinioAccessKey: process.env[envMap.MinioAccessKey], MinioSecretKey: process.env[envMap.MinioSecretKey] diff --git a/server/collaborator/src/context.ts b/server/collaborator/src/context.ts index 6cd38f2a46..ebfd8fa54f 100644 --- a/server/collaborator/src/context.ts +++ b/server/collaborator/src/context.ts @@ -13,21 +13,34 @@ // limitations under the License. // +import { generateId } from '@hcengineering/core' import { Token, decodeToken } from '@hcengineering/server-token' import { onAuthenticatePayload } from '@hocuspocus/server' export interface Context { - token: Token + connectionId: string + token: string + decodedToken: Token initialContentId: string + targetContentId: string +} + +export type withContext = Omit & { + context: Context } export function buildContext (data: onAuthenticatePayload): Context { - const token = decodeToken(data.token) + const connectionId = generateId() + const decodedToken = decodeToken(data.token) const initialContentId = data.requestParameters.get('initialContentId') as string + const targetContentId = data.requestParameters.get('targetContentId') as string const context: Context = { - token, - initialContentId: initialContentId ?? '' + connectionId, + decodedToken, + token: data.token, + initialContentId: initialContentId ?? '', + targetContentId: targetContentId ?? '' } return context diff --git a/server/collaborator/src/extensions/action.ts b/server/collaborator/src/extensions/action.ts index 5d88e4c86f..8c15812dc5 100644 --- a/server/collaborator/src/extensions/action.ts +++ b/server/collaborator/src/extensions/action.ts @@ -1,12 +1,46 @@ +// +// Copyright © 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { MeasureContext } from '@hcengineering/core' import { Connection, Document, Extension, Hocuspocus, onConfigurePayload, onStatelessPayload } from '@hocuspocus/server' +import { Transformer } from '@hocuspocus/transformer' import * as Y from 'yjs' import { Context } from '../context' -import { Action, ActionStatus, ActionStatusResponse, DocumentCopyAction, DocumentFieldCopyAction } from '../types' +import { + Action, + ActionStatus, + ActionStatusResponse, + DocumentContentAction, + DocumentCopyAction, + DocumentFieldCopyAction +} from '../types' + +export interface ActionsConfiguration { + ctx: MeasureContext + transformer: Transformer +} export class ActionsExtension implements Extension { + private readonly configuration: ActionsConfiguration instance!: Hocuspocus + constructor (configuration: ActionsConfiguration) { + this.configuration = configuration + } + async onConfigure ({ instance }: onConfigurePayload): Promise { this.instance = instance } @@ -14,21 +48,29 @@ export class ActionsExtension implements Extension { async onStateless (data: onStatelessPayload): Promise { try { const action = JSON.parse(data.payload) as Action - const context = data.connection.context as Context - const { connection } = data + const context = data.connection.context + const { connection, document, documentName } = data - switch (action.action) { - case 'document.copy': - await this.onCopyDocument(context, action) - this.sendActionStatus(connection, action, 'completed') - return - case 'document.field.copy': - await this.onCopyDocumentField(context, action) - this.sendActionStatus(connection, action, 'completed') - return - default: - console.error('unsupported action type', action) - } + console.log('process stateless message', action.action, documentName) + + await this.configuration.ctx.with(action.action, {}, async () => { + switch (action.action) { + case 'document.content': + await this.onDocumentContent(document, action) + this.sendActionStatus(connection, action, 'completed') + return + case 'document.copy': + await this.onCopyDocument(context, action) + this.sendActionStatus(connection, action, 'completed') + return + case 'document.field.copy': + await this.onCopyDocumentField(context, action) + this.sendActionStatus(connection, action, 'completed') + return + default: + console.error('unsupported action type', action) + } + }) } catch (err: any) { console.error('failed to process stateless message', err) } @@ -39,16 +81,23 @@ export class ActionsExtension implements Extension { connection.sendStateless(JSON.stringify(payload)) } + async onDocumentContent (document: Document, action: DocumentContentAction): Promise { + const { content, field } = action.params + if (!document.share.has(field)) { + const ydoc = this.configuration.transformer.toYdoc(content, field) + document.merge(ydoc) + } else { + console.warn('document has already been initialized') + } + } + async onCopyDocument (context: Context, action: DocumentCopyAction): Promise { const instance = this.instance const { sourceId, targetId } = action.params console.info(`copy document content ${sourceId} -> ${targetId}`) - const _context: Context = { - token: context.token, - initialContentId: '' - } + const _context: Context = { ...context, initialContentId: '', targetContentId: '' } let source: Document | null = null let target: Document | null = null @@ -102,10 +151,7 @@ export class ActionsExtension implements Extension { return } - const _context: Context = { - token: context.token, - initialContentId: '' - } + const _context: Context = { ...context, initialContentId: '', targetContentId: '' } let doc: Document | null = null diff --git a/server/collaborator/src/extensions/storage.ts b/server/collaborator/src/extensions/storage.ts index 51ee30dfaa..0f61aad03e 100644 --- a/server/collaborator/src/extensions/storage.ts +++ b/server/collaborator/src/extensions/storage.ts @@ -13,126 +13,117 @@ // limitations under the License. // -import attachment, { Attachment } from '@hcengineering/attachment' -import client from '@hcengineering/client' -import clientResources from '@hcengineering/client-resources' -import core, { Client, MeasureContext, Ref, TxOperations } from '@hcengineering/core' -import { MinioService } from '@hcengineering/minio' -import { setMetadata } from '@hcengineering/platform' -import { Token, generateToken } from '@hcengineering/server-token' -import { Extension, onLoadDocumentPayload, onStoreDocumentPayload } from '@hocuspocus/server' -import { applyUpdate, encodeStateAsUpdate } from 'yjs' -import config from '../config' -import { Context } from '../context' - -// eslint-disable-next-line -const WebSocket = require('ws') - -async function connect (transactorUrl: string, token: Token): Promise { - const encodedToken = generateToken(token.email, token.workspace) - // We need to override default factory with 'ws' one. - setMetadata(client.metadata.ClientSocketFactory, (url) => { - return new WebSocket(url, { - headers: { - 'User-Agent': config.ServiceID - } - }) - }) - return await (await clientResources()).function.GetClient(encodedToken, transactorUrl) -} +import { MeasureContext } from '@hcengineering/core' +import { + Document, + Extension, + afterUnloadDocumentPayload, + onChangePayload, + onDisconnectPayload, + onLoadDocumentPayload, + onStoreDocumentPayload +} from '@hocuspocus/server' +import { Doc as YDoc } from 'yjs' +import { Context, withContext } from '../context' +import { StorageAdapter } from '../storage/adapter' export interface StorageConfiguration { ctx: MeasureContext - minio: MinioService - transactorUrl: string + adapter: StorageAdapter } export class StorageExtension implements Extension { private readonly configuration: StorageConfiguration + private readonly collaborators = new Map>() constructor (configuration: StorageConfiguration) { this.configuration = configuration } - async getMinioDocument (documentId: string, token: Token): Promise { - const buffer = await this.configuration.minio.read(token.workspace, documentId) - return Buffer.concat(buffer) + async onChange ({ context, documentName }: withContext): Promise { + const collaborators = this.collaborators.get(documentName) ?? new Set() + collaborators.add(context.connectionId) + this.collaborators.set(documentName, collaborators) } - async onLoadDocument (data: onLoadDocumentPayload): Promise { - console.log('load document', data.documentName) + async onLoadDocument ({ context, documentName }: withContext): Promise { + return await this.configuration.ctx.with('load-document', {}, async () => { + return await this.loadDocument(documentName, context) + }) + } - const documentId = data.documentName - const { token, initialContentId } = data.context as Context + async onStoreDocument ({ context, documentName, document }: withContext): Promise { + const collaborators = this.collaborators.get(documentName) + if (collaborators === undefined || collaborators.size === 0) { + console.log('no changes for document', documentName) + return + } - await this.configuration.ctx.with('load-document', {}, async () => { - let minioDocument: Buffer | undefined + await this.configuration.ctx.with('store-document', {}, async () => { + this.collaborators.delete(documentName) + await this.storeDocument(documentName, document, context) + }) + } + + async onDisconnect ({ context, documentName, document }: withContext): Promise { + const { connectionId } = context + const collaborators = this.collaborators.get(documentName) + if (collaborators === undefined || !this.collaborators.has(connectionId)) { + console.log('no changes for document', documentName) + } + + await this.configuration.ctx.with('store-document', {}, async () => { + this.collaborators.get(documentName)?.delete(connectionId) + await this.storeDocument(documentName, document, context) + }) + } + + async afterUnloadDocument ({ documentName }: afterUnloadDocumentPayload): Promise { + this.collaborators.delete(documentName) + } + + async loadDocument (documentId: string, context: Context): Promise { + const { adapter } = this.configuration + + console.log('load document', documentId) + try { + const ydoc = await adapter.loadDocument(documentId, context) + if (ydoc !== undefined) { + return ydoc + } + } catch (err) { + console.error('failed to load document', documentId, err) + } + + const { initialContentId } = context + if (initialContentId !== undefined && initialContentId.length > 0) { + console.log('load document initial content', initialContentId) try { - minioDocument = await this.getMinioDocument(documentId, token) - } catch (err: any) { - if (initialContentId !== undefined && initialContentId.length > 0) { - try { - minioDocument = await this.getMinioDocument(initialContentId, token) - } catch (err: any) { - // Do nothing - // Initial content document also might not have been initialized in minio (e.g. if it's an empty template) - } - } + return await adapter.loadDocument(initialContentId, context) + } catch (err) { + console.error('failed to load document', initialContentId, err) } - - if (minioDocument !== undefined && minioDocument.length > 0) { - try { - const uint8arr = new Uint8Array(minioDocument) - applyUpdate(data.document, uint8arr) - } catch (err) { - console.error(err) - } - } - }) - - return data.document + } } - async onStoreDocument (data: onStoreDocumentPayload): Promise { - console.log('store document', data.documentName) + async storeDocument (documentId: string, document: Document, context: Context): Promise { + const { adapter } = this.configuration - const documentId = data.documentName - const { token } = data.context as Context + console.log('store document', documentId) + try { + await adapter.saveDocument(documentId, document, context) + } catch (err) { + console.error('failed to save document', documentId, err) + } - await this.configuration.ctx.with('store-document', {}, async (ctx) => { - const updates = encodeStateAsUpdate(data.document) - const buffer = Buffer.from(updates.buffer) - - // persist document to Minio - await ctx.with('minio', {}, async () => { - const metaData = { 'content-type': 'application/ydoc' } - await this.configuration.minio.put(token.workspace, documentId, buffer, buffer.length, metaData) - }) - - // notify platform about changes - await ctx.with('platform', {}, async () => { - try { - const connection = await connect(this.configuration.transactorUrl, token) - - // token belongs to the first user opened the document, this is not accurate, but - // since the document is collaborative, we need to choose some account to update the doc - const account = await connection.findOne(core.class.Account, { email: token.email }) - const accountId = account?._id ?? core.account.System - - const client = new TxOperations(connection, accountId, true) - const current = await client.findOne(attachment.class.Attachment, { _id: documentId as Ref }) - if (current !== undefined) { - console.debug('platform notification for document', documentId) - await client.update(current, { lastModified: Date.now(), size: buffer.length }) - } else { - console.debug('platform attachment document not found', documentId) - } - - await connection.close() - } catch (err: any) { - console.debug('failed to notify platform', documentId, err) - } - }) - }) + const { targetContentId } = context + if (targetContentId !== undefined && targetContentId.length > 0) { + console.log('store document target content', targetContentId) + try { + await adapter.saveDocument(targetContentId, document, context) + } catch (err) { + console.error('failed to save document', targetContentId, err) + } + } } } diff --git a/server/collaborator/src/platform.ts b/server/collaborator/src/platform.ts new file mode 100644 index 0000000000..068e076c0e --- /dev/null +++ b/server/collaborator/src/platform.ts @@ -0,0 +1,43 @@ +// +// Copyright © 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import client from '@hcengineering/client' +import clientResources from '@hcengineering/client-resources' +import core, { Client, TxOperations } from '@hcengineering/core' +import { setMetadata } from '@hcengineering/platform' +import { Token } from '@hcengineering/server-token' +import config from './config' + +// eslint-disable-next-line +const WebSocket = require('ws') + +export async function connect (transactorUrl: string, token: string): Promise { + // We need to override default factory with 'ws' one. + setMetadata(client.metadata.ClientSocketFactory, (url) => { + return new WebSocket(url, { + headers: { + 'User-Agent': config.ServiceID + } + }) + }) + return await (await clientResources()).function.GetClient(token, transactorUrl) +} + +export async function getTxOperations (client: Client, token: Token, isDerived: boolean = false): Promise { + const account = await client.findOne(core.class.Account, { email: token.email }) + const accountId = account?._id ?? core.account.System + + return new TxOperations(client, accountId, isDerived) +} diff --git a/server/collaborator/src/server.ts b/server/collaborator/src/server.ts index 0c2f42373c..048f9836cf 100644 --- a/server/collaborator/src/server.ts +++ b/server/collaborator/src/server.ts @@ -15,25 +15,42 @@ import { MeasureContext } from '@hcengineering/core' import { MinioService } from '@hcengineering/minio' +import { serverExtensions } from '@hcengineering/text' import { Hocuspocus, onAuthenticatePayload } from '@hocuspocus/server' import bp from 'body-parser' import compression from 'compression' import cors from 'cors' import express from 'express' import { IncomingMessage, createServer } from 'http' +import { MongoClient } from 'mongodb' import { WebSocket, WebSocketServer } from 'ws' import { Config } from './config' import { ActionsExtension } from './extensions/action' -import { StorageExtension } from './extensions/storage' import { Context, buildContext } from './context' +import { HtmlTransformer } from './transformers/html' +import { MinioStorageAdapter } from './storage/minio' +import { MongodbStorageAdapter } from './storage/mongodb' +import { PlatformStorageAdapter } from './storage/platform' +import { StorageExtension } from './extensions/storage' +import { RouterStorageAdapter } from './storage/router' const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0' /** * @public */ -export function start (ctx: MeasureContext, config: Config, minio: MinioService): () => void { +export type Shutdown = () => Promise + +/** + * @public + */ +export async function start ( + ctx: MeasureContext, + config: Config, + minio: MinioService, + mongo: MongoClient +): Promise { const port = config.Port console.log(`starting server on :${port} ...`) @@ -55,6 +72,9 @@ export function start (ctx: MeasureContext, config: Config, minio: MinioService) }) ) + const extensionsCtx = ctx.newChild('extensions', {}) + const storageCtx = ctx.newChild('storage', {}) + const hocuspocus = new Hocuspocus({ address: '0.0.0.0', port, @@ -89,11 +109,28 @@ export function start (ctx: MeasureContext, config: Config, minio: MinioService) unloadImmediately: false, extensions: [ - new ActionsExtension(), + new ActionsExtension({ + ctx: extensionsCtx.newChild('actions', {}), + transformer: new HtmlTransformer(serverExtensions) + }), new StorageExtension({ - ctx: ctx.newChild('minio', {}), - minio, - transactorUrl: config.TransactorUrl + ctx: extensionsCtx.newChild('storage', {}), + adapter: new RouterStorageAdapter( + { + minio: new MinioStorageAdapter(storageCtx.newChild('minio', {}), minio, config.TransactorUrl), + mongodb: new MongodbStorageAdapter( + storageCtx.newChild('mongodb', {}), + mongo, + new HtmlTransformer(serverExtensions) + ), + platform: new PlatformStorageAdapter( + storageCtx.newChild('platform', {}), + config.TransactorUrl, + new HtmlTransformer(serverExtensions) + ) + }, + 'minio' + ) }) ], @@ -138,7 +175,7 @@ export function start (ctx: MeasureContext, config: Config, minio: MinioService) server.listen(port) console.log(`started server on :${port}`) - return () => { + return async () => { server.close() } } diff --git a/server/collaborator/src/starter.ts b/server/collaborator/src/starter.ts index 0ac31d0c5a..31c0df3ea6 100644 --- a/server/collaborator/src/starter.ts +++ b/server/collaborator/src/starter.ts @@ -17,12 +17,13 @@ import { MinioService } from '@hcengineering/minio' import { setMetadata } from '@hcengineering/platform' import serverToken from '@hcengineering/server-token' +import { MongoClient } from 'mongodb' import config from './config' import { metricsContext } from './metrics' import { start } from './server' -export function startCollaborator (): void { +export async function startCollaborator (): Promise { setMetadata(serverToken.metadata.Secret, config.Secret) let minioPort = 9000 @@ -33,7 +34,7 @@ export function startCollaborator (): void { minioPort = parseInt(sp[1]) } - const minio = new MinioService({ + const minioClient = new MinioService({ endPoint: minioEndpoint, port: minioPort, useSSL: false, @@ -41,10 +42,13 @@ export function startCollaborator (): void { secretKey: config.MinioSecretKey }) - const server = start(metricsContext, config, minio) + const mongoClient = await MongoClient.connect(config.MongoUrl) + + const shutdown = await start(metricsContext, config, minioClient, mongoClient) const close = (): void => { - server() + void mongoClient.close() + void shutdown() } process.on('SIGINT', close) diff --git a/server/collaborator/src/storage/adapter.ts b/server/collaborator/src/storage/adapter.ts new file mode 100644 index 0000000000..1de5433aec --- /dev/null +++ b/server/collaborator/src/storage/adapter.ts @@ -0,0 +1,25 @@ +// +// Copyright © 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Document } from '@hocuspocus/server' +import { Doc as YDoc } from 'yjs' +import { Context } from '../context' + +export interface StorageAdapter { + loadDocument: (documentId: string, context: Context) => Promise + saveDocument: (documentId: string, document: Document, context: Context) => Promise +} + +export type StorageAdapters = Record diff --git a/server/collaborator/src/storage/minio.ts b/server/collaborator/src/storage/minio.ts new file mode 100644 index 0000000000..31e80f67f1 --- /dev/null +++ b/server/collaborator/src/storage/minio.ts @@ -0,0 +1,117 @@ +// +// Copyright © 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import attachment, { Attachment } from '@hcengineering/attachment' +import { MeasureContext, Ref } from '@hcengineering/core' +import { MinioService } from '@hcengineering/minio' +import { Document } from '@hocuspocus/server' +import { Doc as YDoc, applyUpdate, encodeStateAsUpdate } from 'yjs' + +import { Context } from '../context' + +import { StorageAdapter } from './adapter' +import { connect, getTxOperations } from '../platform' + +function maybePlatformDocumentId (documentId: string): boolean { + return !documentId.includes('%') +} + +export class MinioStorageAdapter implements StorageAdapter { + constructor ( + private readonly ctx: MeasureContext, + private readonly minio: MinioService, + private readonly transactorUrl: string + ) {} + + async loadDocument (documentId: string, context: Context): Promise { + const { + decodedToken: { workspace } + } = context + + return await this.ctx.with('load-document', {}, async (ctx) => { + const minioDocument = await ctx.with('query', {}, async () => { + try { + const buffer = await this.minio.read(workspace, documentId) + return Buffer.concat(buffer) + } catch { + return undefined + } + }) + + if (minioDocument === undefined) { + return undefined + } + + const ydoc = new YDoc() + + await ctx.with('transform', {}, () => { + try { + const uint8arr = new Uint8Array(minioDocument) + applyUpdate(ydoc, uint8arr) + } catch (err) { + console.error(err) + } + }) + + return ydoc + }) + } + + async saveDocument (documentId: string, document: Document, context: Context): Promise { + const { decodedToken, token } = context + + await this.ctx.with('save-document', {}, async (ctx) => { + const buffer = await ctx.with('transform', {}, () => { + const updates = encodeStateAsUpdate(document) + return Buffer.from(updates.buffer) + }) + + await ctx.with('update', {}, async () => { + const metadata = { 'content-type': 'application/ydoc' } + await this.minio.put(decodedToken.workspace, documentId, buffer, buffer.length, metadata) + }) + + // minio file is usually an attachment document + // we need to touch an attachment from here to notify platform about changes + + if (!maybePlatformDocumentId(documentId)) { + // documentId is not a platform document id, we can skip platform notification + return + } + + await ctx.with('platform', {}, async () => { + const connection = await ctx.with('connect', {}, async () => { + return await connect(this.transactorUrl, token) + }) + + try { + const client = await getTxOperations(connection, decodedToken, true) + + const current = await ctx.with('query', {}, async () => { + return await client.findOne(attachment.class.Attachment, { _id: documentId as Ref }) + }) + + if (current !== undefined) { + await ctx.with('update', {}, async () => { + await client.update(current, { lastModified: Date.now(), size: buffer.length }) + }) + } + } finally { + await connection.close() + } + }) + }) + } +} diff --git a/server/collaborator/src/storage/mongodb.ts b/server/collaborator/src/storage/mongodb.ts new file mode 100644 index 0000000000..7b4b7aadb5 --- /dev/null +++ b/server/collaborator/src/storage/mongodb.ts @@ -0,0 +1,78 @@ +// +// Copyright © 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { MeasureContext, toWorkspaceString } from '@hcengineering/core' +import { Document } from '@hocuspocus/server' +import { Transformer } from '@hocuspocus/transformer' +import { MongoClient } from 'mongodb' +import { Doc as YDoc } from 'yjs' + +import { Context } from '../context' + +import { StorageAdapter } from './adapter' + +interface MongodbDocumentId { + objectDomain: string + objectId: string + objectAttr: string +} + +function parseDocumentId (documentId: string): MongodbDocumentId { + const [objectDomain, objectId, objectAttr] = documentId.split('/') + return { + objectId: objectId ?? '', + objectDomain: objectDomain ?? '', + objectAttr: objectAttr ?? '' + } +} + +function isValidDocumentId (documentId: MongodbDocumentId): boolean { + return documentId.objectDomain !== '' && documentId.objectId !== '' && documentId.objectAttr !== '' +} + +export class MongodbStorageAdapter implements StorageAdapter { + constructor ( + private readonly ctx: MeasureContext, + private readonly mongodb: MongoClient, + private readonly transformer: Transformer + ) {} + + async loadDocument (documentId: string, context: Context): Promise { + const { decodedToken } = context + const { objectId, objectDomain, objectAttr } = parseDocumentId(documentId) + + if (!isValidDocumentId({ objectId, objectDomain, objectAttr })) { + console.warn('malformed document id', documentId) + return undefined + } + + return await this.ctx.with('load-document', {}, async (ctx) => { + const doc = await ctx.with('query', {}, async () => { + const db = this.mongodb.db(toWorkspaceString(decodedToken.workspace)) + return await db.collection(objectDomain).findOne({ _id: objectId }, { projection: { [objectAttr]: 1 } }) + }) + + const content = doc !== null && objectAttr in doc ? (doc[objectAttr] as string) : '' + + return await ctx.with('transform', {}, () => { + return this.transformer.toYdoc(content, objectAttr) + }) + }) + } + + async saveDocument (_documentId: string, _document: Document, _context: Context): Promise { + // do nothing, not supported + } +} diff --git a/server/collaborator/src/storage/platform.ts b/server/collaborator/src/storage/platform.ts new file mode 100644 index 0000000000..d01c60a743 --- /dev/null +++ b/server/collaborator/src/storage/platform.ts @@ -0,0 +1,120 @@ +// +// Copyright © 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Class, Doc, MeasureContext, Ref } from '@hcengineering/core' +import { Document } from '@hocuspocus/server' +import { Transformer } from '@hocuspocus/transformer' +import { Doc as YDoc } from 'yjs' + +import { Context } from '../context' +import { connect, getTxOperations } from '../platform' + +import { StorageAdapter } from './adapter' + +interface PlatformDocumentId { + objectClass: Ref> + objectId: Ref + objectAttr: string +} + +function parseDocumentId (documentId: string): PlatformDocumentId { + const [objectClass, objectId, objectAttr] = documentId.split('/') + return { + objectClass: (objectClass ?? '') as Ref>, + objectId: (objectId ?? '') as Ref, + objectAttr: objectAttr ?? '' + } +} + +function isValidDocumentId (documentId: PlatformDocumentId): boolean { + return documentId.objectClass !== '' && documentId.objectId !== '' && documentId.objectAttr !== '' +} + +export class PlatformStorageAdapter implements StorageAdapter { + constructor ( + private readonly ctx: MeasureContext, + private readonly transactorUrl: string, + private readonly transformer: Transformer + ) {} + + async loadDocument (documentId: string, context: Context): Promise { + const { token } = context + const { objectId, objectClass, objectAttr } = parseDocumentId(documentId) + + if (!isValidDocumentId({ objectId, objectClass, objectAttr })) { + console.warn('malformed document id', documentId) + return undefined + } + + return await this.ctx.with('load-document', {}, async (ctx) => { + let content = '' + + const client = await ctx.with('connect', {}, async () => { + return await connect(this.transactorUrl, token) + }) + + try { + const doc = await ctx.with('query', {}, async () => { + return await client.findOne(objectClass, { _id: objectId }, { projection: { [objectAttr]: 1 } }) + }) + if (doc !== undefined && objectAttr in doc) { + content = (doc as any)[objectAttr] as string + } + } finally { + await client.close() + } + + return await ctx.with('transform', {}, () => { + return this.transformer.toYdoc(content, objectAttr) + }) + }) + } + + async saveDocument (documentId: string, document: Document, context: Context): Promise { + const { decodedToken, token } = context + const { objectId, objectClass, objectAttr } = parseDocumentId(documentId) + + if (!isValidDocumentId({ objectId, objectClass, objectAttr })) { + console.warn('malformed document id', documentId) + return undefined + } + + await this.ctx.with('save-document', {}, async (ctx) => { + const connection = await ctx.with('connect', {}, async () => { + return await connect(this.transactorUrl, token) + }) + const client = await getTxOperations(connection, decodedToken) + + try { + const current = await ctx.with('query', {}, async () => { + return await client.findOne(objectClass, { _id: objectId }) + }) + + if (current !== undefined) { + const content = await ctx.with('transform', {}, () => { + return this.transformer.fromYdoc(document, objectAttr) + }) + await ctx.with('update', {}, async () => { + if ((current as any)[objectAttr] !== content) { + await client.update(current, { [objectAttr]: content }) + } + }) + } + } finally { + await connection.close() + } + }) + } +} diff --git a/server/collaborator/src/storage/router.ts b/server/collaborator/src/storage/router.ts new file mode 100644 index 0000000000..9a4dc5d8ad --- /dev/null +++ b/server/collaborator/src/storage/router.ts @@ -0,0 +1,53 @@ +// +// Copyright © 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Document } from '@hocuspocus/server' +import { Doc as YDoc } from 'yjs' + +import { Context } from '../context' + +import { StorageAdapter, StorageAdapters } from './adapter' + +function parseDocumentName (documentId: string): { schema: string, documentName: string } { + const [schema, documentName] = documentId.split('://', 2) + return documentName !== undefined ? { documentName, schema } : { documentName: documentId, schema: '' } +} + +export class RouterStorageAdapter implements StorageAdapter { + constructor ( + private readonly adapters: StorageAdapters, + private readonly defaultAdapter: string + ) {} + + getStorageAdapter (schema: string): StorageAdapter | undefined { + return schema in this.adapters + ? this.adapters[schema] + : this.defaultAdapter !== undefined + ? this.adapters[this.defaultAdapter] + : undefined + } + + async loadDocument (documentId: string, context: Context): Promise { + const { schema, documentName } = parseDocumentName(documentId) + const adapter = this.getStorageAdapter(schema) + return await adapter?.loadDocument?.(documentName, context) + } + + async saveDocument (documentId: string, document: Document, context: Context): Promise { + const { schema, documentName } = parseDocumentName(documentId) + const adapter = this.getStorageAdapter(schema) + await adapter?.saveDocument?.(documentName, document, context) + } +} diff --git a/server/collaborator/src/transformers/html.ts b/server/collaborator/src/transformers/html.ts new file mode 100644 index 0000000000..5918f880f3 --- /dev/null +++ b/server/collaborator/src/transformers/html.ts @@ -0,0 +1,41 @@ +// +// Copyright © 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { TiptapTransformer, Transformer } from '@hocuspocus/transformer' +import { Extensions } from '@tiptap/core' +import { generateHTML, generateJSON } from '@tiptap/html' +import { Doc } from 'yjs' + +export class HtmlTransformer implements Transformer { + transformer: Transformer + + constructor (private readonly extensions: Extensions) { + this.transformer = TiptapTransformer.extensions(extensions) + } + + fromYdoc (document: Doc, fieldName?: string | string[] | undefined): any { + const json = this.transformer.fromYdoc(document, fieldName) + return generateHTML(json, this.extensions) + } + + toYdoc (document: any, fieldName: string): Doc { + if (typeof document === 'string' && document !== '') { + const json = generateJSON(document, this.extensions) + return this.transformer.toYdoc(json, fieldName) + } + + return new Doc() + } +} diff --git a/server/collaborator/src/types.ts b/server/collaborator/src/types.ts index ef6fb14604..7c1ba35ee2 100644 --- a/server/collaborator/src/types.ts +++ b/server/collaborator/src/types.ts @@ -13,7 +13,17 @@ // limitations under the License. // -export type Action = DocumentCopyAction | DocumentFieldCopyAction +export type Action = DocumentCopyAction | DocumentFieldCopyAction | DocumentContentAction + +export type StorageType = 'minio' | 'platform' + +export interface DocumentContentAction { + action: 'document.content' + params: { + field: string + content: string + } +} export interface DocumentCopyAction { action: 'document.copy' diff --git a/server/core/src/indexer/summary.ts b/server/core/src/indexer/summary.ts index 95864121e9..a51bf42965 100644 --- a/server/core/src/indexer/summary.ts +++ b/server/core/src/indexer/summary.ts @@ -271,7 +271,7 @@ export async function extractIndexedValues ( continue } - if (keyAttr.type._class === core.class.TypeMarkup) { + if (keyAttr.type._class === core.class.TypeMarkup || keyAttr.type._class === core.class.TypeCollaborativeMarkup) { sourceContent = convert(sourceContent, { preserveNewlines: true, selectors: [{ selector: 'img', format: 'skip' }] diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index 1e65dcb021..15ffc296ca 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -102,6 +102,7 @@ services: collaborator: image: hardcoreeng/collaborator links: + - mongodb - minio - transactor ports: @@ -110,6 +111,7 @@ services: - COLLABORATOR_PORT=3078 - SECRET=secret - TRANSACTOR_URL=ws://localhost:3334 + - MONGO_URL=mongodb://mongodb:27018 - MINIO_ENDPOINT=minio - MINIO_ACCESS_KEY=minioadmin - MINIO_SECRET_KEY=minioadmin