diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 4306837270..6e09d7b3dc 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -189,7 +189,7 @@ importers: version: file:projects/collaboration.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/collaborator': specifier: file:./projects/collaborator.tgz - version: file:projects/collaborator.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@tiptap/pm@2.11.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(gcp-metadata@5.3.0(encoding@0.1.13))(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(snappy@7.2.2)(socks@2.8.3)(utf-8-validate@6.0.4)(y-protocols@1.0.6(yjs@13.6.23)) + version: file:projects/collaborator.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@tiptap/pm@2.11.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)(utf-8-validate@6.0.4)(y-prosemirror@1.2.15(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(y-protocols@1.0.6(yjs@13.6.23))(yjs@13.6.23))(y-protocols@1.0.6(yjs@13.6.23)) '@rush-temp/collaborator-client': specifier: file:./projects/collaborator-client.tgz version: file:projects/collaborator-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) @@ -1131,7 +1131,7 @@ importers: version: file:projects/tests-ws-sanity.tgz '@rush-temp/text': specifier: file:./projects/text.tgz - version: file:projects/text.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(prosemirror-inputrules@1.4.0)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4) + version: file:projects/text.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(prosemirror-inputrules@1.4.0)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/text-core': specifier: file:./projects/text-core.tgz version: file:projects/text-core.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4) @@ -1144,6 +1144,12 @@ importers: '@rush-temp/text-editor-resources': specifier: file:./projects/text-editor-resources.tgz version: file:projects/text-editor-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(highlight.js@11.11.1)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(prosemirror-inputrules@1.4.0)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4) + '@rush-temp/text-html': + specifier: file:./projects/text-html.tgz + version: file:projects/text-html.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + '@rush-temp/text-markdown': + specifier: file:./projects/text-markdown.tgz + version: file:projects/text-markdown.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/text-ydoc': specifier: file:./projects/text-ydoc.tgz version: file:projects/text-ydoc.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4)(y-protocols@1.0.6(yjs@13.6.23)) @@ -4021,7 +4027,7 @@ packages: version: 0.0.0 '@rush-temp/api-client@file:projects/api-client.tgz': - resolution: {integrity: sha512-SMKCkubeb8nE1zsYu0fsutpc9LJKiNOG0bn9NP+FxlSU4NGb8++EpZfb650lGYdvg2MljvrQD+kqYQDt1IRd1A==, tarball: file:projects/api-client.tgz} + resolution: {integrity: sha512-vHSAYOOfNXaxc1sk4qmXWIqodv5BpwMnNHCVw2FFl82nI8Ml8P7V6p9DNklf97UC2JpF6L7bA2olTKJjWueJyQ==, tarball: file:projects/api-client.tgz} version: 0.0.0 '@rush-temp/api-tests@file:projects/api-tests.tgz': @@ -4133,7 +4139,7 @@ packages: version: 0.0.0 '@rush-temp/collaborator@file:projects/collaborator.tgz': - resolution: {integrity: sha512-QNIE09ZcEdRCPC+oX1x4ltXrEJ+GDrz23EhmqdEZiiU3gLpj6RTVjGnGLeyKTjmXhPWorKWAGmfSwZ1VLBNGhQ==, tarball: file:projects/collaborator.tgz} + resolution: {integrity: sha512-58uWn3sX/3Q8iYkKKwysct13g+bT9rfTw8JaLFLH5rJFfSorpRjtypQ29SVTUwSO+e7xm8BkH5XVOaqk5b1OBQ==, tarball: file:projects/collaborator.tgz} version: 0.0.0 '@rush-temp/contact-assets@file:projects/contact-assets.tgz': @@ -4309,7 +4315,7 @@ packages: version: 0.0.0 '@rush-temp/importer@file:projects/importer.tgz': - resolution: {integrity: sha512-nd4QEoFM7LFj37X/9PCtKl2HTaQl3xnpCbJL+FBuYPJhimHzG4KTvb3E5vZ31OZxgAzYBBLZb1KsswqqlXAJ9A==, tarball: file:projects/importer.tgz} + resolution: {integrity: sha512-DecPpuedBGqrUXATVoljf098pHTQ9zqylXrUKbWiibuuFy5i9zzw/9HY8hM4Zg1Ku8ZOm1AGBUoSmgmGPJ13Cg==, tarball: file:projects/importer.tgz} version: 0.0.0 '@rush-temp/inventory-assets@file:projects/inventory-assets.tgz': @@ -4773,7 +4779,7 @@ packages: version: 0.0.0 '@rush-temp/pod-ai-bot@file:projects/pod-ai-bot.tgz': - resolution: {integrity: sha512-EPudGHMe7m4s6PeA23nmc25YVXpq62MfJV3ZZCxy8aD+xJA1mJ78c5QG+RPBnEe5mVeLlufLNPeWVlyVcPrmKQ==, tarball: file:projects/pod-ai-bot.tgz} + resolution: {integrity: sha512-XZBQM9sEPJVQyE/PPrSKZ7cH3k2UBx5PJU41K+kRxLbjF1nyRgDTO88PMdZk2wcUJXuj9TNIo1rtLn7cwXtgYA==, tarball: file:projects/pod-ai-bot.tgz} version: 0.0.0 '@rush-temp/pod-analytics-collector@file:projects/pod-analytics-collector.tgz': @@ -4801,7 +4807,7 @@ packages: version: 0.0.0 '@rush-temp/pod-github@file:projects/pod-github.tgz': - resolution: {integrity: sha512-Szu3mv8d6NPVJtTXbnxK4LvO0yCMhJeOjOxX/hQNjizR4bzuWA0RFmihTUxGiByqWhuwciVC5G7ZtjZdprATzQ==, tarball: file:projects/pod-github.tgz} + resolution: {integrity: sha512-fzeprH8VrdbEOKPKWmNQ2uzdSVDwhiOxaQEbx3A6WqmAiv0ftltnkBJsegz5Pa80BpG7rFi7nVUj3HTU+iSc+Q==, tarball: file:projects/pod-github.tgz} version: 0.0.0 '@rush-temp/pod-gmail@file:projects/pod-gmail.tgz': @@ -5025,7 +5031,7 @@ packages: version: 0.0.0 '@rush-temp/server-chunter-resources@file:projects/server-chunter-resources.tgz': - resolution: {integrity: sha512-kiR/o8nysn7cgV4YGZNzxBpqzpoTqXGPyF7fXO3MJ0aZPa46AWQPushZMQTnR9O7wFs3RUuHR7Bc6VsjlzqDqA==, tarball: file:projects/server-chunter-resources.tgz} + resolution: {integrity: sha512-nsXVE7tY5FkhaEpaGDygZAIYISVRaIubVZz3w0VqQMpbBStm+02ju1aRtdQr3Zob/JDbI9prkF3UGaWxsu7wyQ==, tarball: file:projects/server-chunter-resources.tgz} version: 0.0.0 '@rush-temp/server-chunter@file:projects/server-chunter.tgz': @@ -5237,7 +5243,7 @@ packages: version: 0.0.0 '@rush-temp/server-tool@file:projects/server-tool.tgz': - resolution: {integrity: sha512-XGLZjlEMa+MBHaEr8yUT/c2Rn41CSRIW6ihPJSwA9T0hYn+vkoIXKXV/n7+BnpomXBY58zxlOQE1rSYbBKrtDA==, tarball: file:projects/server-tool.tgz} + resolution: {integrity: sha512-BJVfBMcM1Zdyw8474Nt4IQnyohSpTUNR9bWR7Ho+Rkiz4qCoxuAifAtuS9P0sVi4Ua8YdGeVDnPELTnN1qHW0w==, tarball: file:projects/server-tool.tgz} version: 0.0.0 '@rush-temp/server-tracker-resources@file:projects/server-tracker-resources.tgz': @@ -5400,12 +5406,20 @@ packages: resolution: {integrity: sha512-jpmq/zjovw6L0F6HTliZnFFFBdUaFD3lSgs1KQsOIgDX9spZgBa5Pgei7ev2NPllNM2VbehHCOgx1k61lKfMbQ==, tarball: file:projects/text-editor.tgz} version: 0.0.0 + '@rush-temp/text-html@file:projects/text-html.tgz': + resolution: {integrity: sha512-tHs2WGLdf/d6zKV33Yf7YFPekyTXP4vm1iJVNrNlxgHC0uVWogwpAn+yoLq16ZksHMvA94uQw0/dJqD3EBpmDA==, tarball: file:projects/text-html.tgz} + version: 0.0.0 + + '@rush-temp/text-markdown@file:projects/text-markdown.tgz': + resolution: {integrity: sha512-ys7AkE1k1PmsgWwTMxLkuUnZZVynIzwdCzZgjm3Ox19+1aqXth2y/kD5VYhnJ/YcqFHJs3zF2LKEhQUk6DtfEg==, tarball: file:projects/text-markdown.tgz} + version: 0.0.0 + '@rush-temp/text-ydoc@file:projects/text-ydoc.tgz': - resolution: {integrity: sha512-GDg5LdJM76vYawA5nVZ4Ww9zdDjommvyizqS41jPZ/lahX1WYHvdE1+S1IQLjP6cku/vHPhdbJ9A8+q8tH0cnw==, tarball: file:projects/text-ydoc.tgz} + resolution: {integrity: sha512-m0xmzyPbWl++/4OPa2evroi1WqVKJXvPvosncUY39B+N39rcGpSKjcVMFtY8YHKJt9FBca2zCSPeq+TnNICjOQ==, tarball: file:projects/text-ydoc.tgz} version: 0.0.0 '@rush-temp/text@file:projects/text.tgz': - resolution: {integrity: sha512-8Q54dpYn+hO9JM+nWcc4zrikkwORS8C+gAuxFSA/CyVbxW4sNY/BI9Od8U5QrDgOPRO9AoQbVGEDldTOLZCp2g==, tarball: file:projects/text.tgz} + resolution: {integrity: sha512-MzCjKpbxV9a83Hnx+Ur5THHu4Uguk3iJPecgEEiPUZRJCnOaoSjnAL5dmi8lEU3N/x5hzoOVeKaVvnwA9Kareg==, tarball: file:projects/text.tgz} version: 0.0.0 '@rush-temp/theme@file:projects/theme.tgz': @@ -11316,8 +11330,8 @@ packages: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} - parse5@7.1.2: - resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} + parse5@7.2.1: + resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==} parseley@0.12.1: resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==} @@ -17354,7 +17368,7 @@ snapshots: - supports-color - ts-node - '@rush-temp/collaborator@file:projects/collaborator.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@tiptap/pm@2.11.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(gcp-metadata@5.3.0(encoding@0.1.13))(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(snappy@7.2.2)(socks@2.8.3)(utf-8-validate@6.0.4)(y-protocols@1.0.6(yjs@13.6.23))': + '@rush-temp/collaborator@file:projects/collaborator.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@tiptap/pm@2.11.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)(utf-8-validate@6.0.4)(y-prosemirror@1.2.15(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(y-protocols@1.0.6(yjs@13.6.23))(yjs@13.6.23))(y-protocols@1.0.6(yjs@13.6.23))': dependencies: '@hocuspocus/server': 2.15.1(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-protocols@1.0.6(yjs@13.6.23))(yjs@13.6.23) '@hocuspocus/transformer': 2.15.1(@tiptap/core@2.11.3(@tiptap/pm@2.11.3))(@tiptap/pm@2.11.3)(y-prosemirror@1.2.15(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(y-protocols@1.0.6(yjs@13.6.23))(yjs@13.6.23))(yjs@13.6.23) @@ -17386,7 +17400,6 @@ snapshots: ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) typescript: 5.3.3 ws: 8.18.0(bufferutil@4.0.8)(utf-8-validate@6.0.4) - y-prosemirror: 1.2.15(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(y-protocols@1.0.6(yjs@13.6.23))(yjs@13.6.23) yjs: 13.6.23 transitivePeerDependencies: - '@aws-sdk/credential-providers' @@ -17403,13 +17416,11 @@ snapshots: - kerberos - mongodb-client-encryption - node-notifier - - prosemirror-model - - prosemirror-state - - prosemirror-view - snappy - socks - supports-color - utf-8-validate + - y-prosemirror - y-protocols '@rush-temp/contact-assets@file:projects/contact-assets.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': @@ -25774,12 +25785,63 @@ snapshots: - supports-color - ts-node - '@rush-temp/text-ydoc@file:projects/text-ydoc.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4)(y-protocols@1.0.6(yjs@13.6.23))': + '@rush-temp/text-html@file:projects/text-html.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': + dependencies: + '@types/jest': 29.5.12 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + htmlparser2: 9.1.0 + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + prettier: 3.2.5 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) + typescript: 5.7.3 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - '@types/node' + - babel-jest + - babel-plugin-macros + - esbuild + - node-notifier + - supports-color + - ts-node + + '@rush-temp/text-markdown@file:projects/text-markdown.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: - '@tiptap/core': 2.11.3(@tiptap/pm@2.11.3) - '@tiptap/pm': 2.11.3 '@types/jest': 29.5.12 '@types/markdown-it': 13.0.8 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + fast-equals: 5.2.2 + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + markdown-it: 14.0.0 + prettier: 3.2.5 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) + typescript: 5.7.3 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - '@types/node' + - babel-jest + - babel-plugin-macros + - esbuild + - node-notifier + - supports-color + - ts-node + + '@rush-temp/text-ydoc@file:projects/text-ydoc.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4)(y-protocols@1.0.6(yjs@13.6.23))': + dependencies: + '@types/jest': 29.5.12 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) eslint: 8.56.0 @@ -25813,7 +25875,7 @@ snapshots: - utf-8-validate - y-protocols - '@rush-temp/text@file:projects/text.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(prosemirror-inputrules@1.4.0)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))(utf-8-validate@6.0.4)': + '@rush-temp/text@file:projects/text.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(prosemirror-inputrules@1.4.0)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@tiptap/core': 2.11.3(@tiptap/pm@2.11.3) '@tiptap/extension-code': 2.11.3(@tiptap/core@2.11.3(@tiptap/pm@2.11.3)) @@ -25849,8 +25911,6 @@ snapshots: eslint-plugin-promise: 6.1.1(eslint@8.56.0) fast-equals: 5.2.2 jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) - jest-environment-jsdom: 29.7.0(bufferutil@4.0.8)(utf-8-validate@6.0.4) - markdown-it: 14.0.0 prettier: 3.2.5 prosemirror-codemark: 0.4.2(prosemirror-inputrules@1.4.0)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2) ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.3.3) @@ -25861,8 +25921,6 @@ snapshots: - '@types/node' - babel-jest - babel-plugin-macros - - bufferutil - - canvas - esbuild - node-notifier - prosemirror-inputrules @@ -25871,7 +25929,6 @@ snapshots: - prosemirror-view - supports-color - ts-node - - utf-8-validate '@rush-temp/theme@file:projects/theme.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: @@ -27643,7 +27700,7 @@ snapshots: dependencies: '@types/node': 20.11.19 '@types/tough-cookie': 4.0.5 - parse5: 7.1.2 + parse5: 7.2.1 '@types/json-schema@7.0.15': {} @@ -32614,7 +32671,7 @@ snapshots: https-proxy-agent: 5.0.1 is-potential-custom-element-name: 1.0.1 nwsapi: 2.2.13 - parse5: 7.1.2 + parse5: 7.2.1 saxes: 6.0.0 symbol-tree: 3.2.4 tough-cookie: 4.1.4 @@ -33899,7 +33956,7 @@ snapshots: json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 - parse5@7.1.2: + parse5@7.2.1: dependencies: entities: 4.5.0 diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts index a42bddc9f4..64c97a5c21 100644 --- a/dev/tool/src/clean.ts +++ b/dev/tool/src/clean.ts @@ -73,7 +73,7 @@ import { generateToken } from '@hcengineering/server-token' import { connect } from '@hcengineering/server-tool' import tags, { type TagCategory, type TagElement, type TagReference } from '@hcengineering/tags' import task, { type ProjectType, type Task, type TaskType } from '@hcengineering/task' -import { updateYDocContent } from '@hcengineering/text-ydoc' +// import { updateYDocContent } from '@hcengineering/text-ydoc' import tracker from '@hcengineering/tracker' import { deepEqual } from 'fast-equals' import { type Db } from 'mongodb' diff --git a/packages/api-client/package.json b/packages/api-client/package.json index 7ad79ec3af..f6ab73b269 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -57,6 +57,7 @@ "@hcengineering/account-client": "^0.6.0", "@hcengineering/platform": "^0.6.11", "@hcengineering/text": "^0.6.5", + "@hcengineering/text-markdown": "^0.6.0", "snappyjs": "^0.7.0" }, "repository": "https://github.com/hcengineering/platform", diff --git a/packages/api-client/src/markup/client.ts b/packages/api-client/src/markup/client.ts index a999429bc3..5a783a2080 100644 --- a/packages/api-client/src/markup/client.ts +++ b/packages/api-client/src/markup/client.ts @@ -23,7 +23,8 @@ import { makeCollabId } from '@hcengineering/core' import { type CollaboratorClient, getClient } from '@hcengineering/collaborator-client' -import { parseMessageMarkdown, jsonToMarkup, markupToHTML, markupToMarkdown, htmlToMarkup } from '@hcengineering/text' +import { htmlToJSON, jsonToHTML, jsonToMarkup, markupToJSON } from '@hcengineering/text' +import { markdownToMarkup, markupToMarkdown } from '@hcengineering/text-markdown' import { type ServerConfig } from '../config' import { type MarkupOperations, type MarkupFormat, type MarkupRef } from './types' @@ -62,14 +63,15 @@ class MarkupOperationsImpl implements MarkupOperations { ): Promise { const collabId = makeCollabId(objectClass, objectId, objectAttr) const markup = await this.collaborator.getMarkup(collabId, doc) + const json = markupToJSON(markup) switch (format) { case 'markup': return markup case 'html': - return markupToHTML(markup) + return jsonToHTML(json) case 'markdown': - return await markupToMarkdown(markup, this.refUrl, this.imageUrl) + return markupToMarkdown(json, { refUrl: this.refUrl, imageUrl: this.imageUrl }) default: throw new Error('Unknown content format') } @@ -89,10 +91,10 @@ class MarkupOperationsImpl implements MarkupOperations { markup = value break case 'html': - markup = htmlToMarkup(value) + markup = jsonToMarkup(htmlToJSON(value)) break case 'markdown': - markup = jsonToMarkup(parseMessageMarkdown(value, this.imageUrl, this.refUrl)) + markup = jsonToMarkup(markdownToMarkup(value, { refUrl: this.refUrl, imageUrl: this.imageUrl })) break default: throw new Error('Unknown content format') diff --git a/packages/importer/package.json b/packages/importer/package.json index 9ac60f7e2e..3533bec328 100644 --- a/packages/importer/package.json +++ b/packages/importer/package.json @@ -55,6 +55,7 @@ "@hcengineering/server-core": "^0.6.1", "@hcengineering/task": "^0.6.20", "@hcengineering/text": "^0.6.5", + "@hcengineering/text-markdown": "^0.6.0", "@hcengineering/tracker": "^0.6.24", "@hcengineering/view": "^0.6.13", "commander": "^8.1.0", diff --git a/packages/importer/src/docx/docx.ts b/packages/importer/src/docx/docx.ts index 1670143e5c..eebf7a193e 100644 --- a/packages/importer/src/docx/docx.ts +++ b/packages/importer/src/docx/docx.ts @@ -13,7 +13,8 @@ // limitations under the License. // -import { defaultExtensions, htmlToJSON, MarkupNode, serializeMessage } from '@hcengineering/text' +import { MarkupNode, htmlToJSON } from '@hcengineering/text' +import { markupToMarkdown } from '@hcengineering/text-markdown' import { mkdir, readdir, readFile, writeFile } from 'fs/promises' import * as yaml from 'js-yaml' import { basename, dirname, extname, join, relative } from 'path' @@ -73,7 +74,7 @@ export class DocumentConverter { async processDocument (path: string, root: string): Promise { const htmlString = await this.options.htmlConverter(path) - const markup = htmlToJSON(htmlString, defaultExtensions) + const markup = htmlToJSON(htmlString) let document: DocumentState = { name: fileNameNoExt(path), @@ -105,7 +106,7 @@ export class DocumentConverter { } function compileMarkdown (file: DocumentState): string { - const markdown = serializeMessage(file.markup, 'ref://', '') + const markdown = markupToMarkdown(file.markup, { refUrl: 'ref://', imageUrl: '' }) const headerYaml = yaml.dump(file.header) const headerString = '---\n' + headerYaml + '---\n' diff --git a/packages/importer/src/huly/unified.ts b/packages/importer/src/huly/unified.ts index 6091dac6ac..e5fcc7559c 100644 --- a/packages/importer/src/huly/unified.ts +++ b/packages/importer/src/huly/unified.ts @@ -218,7 +218,7 @@ class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor { const sourcePath = this.getSourcePath(id) if (sourcePath == null) return - const href = decodeURI(mark.attrs.href) + const href = decodeURI(mark.attrs?.href ?? '') const fullPath = path.resolve(path.dirname(sourcePath), href) if (this.refMetaByPath.has(fullPath)) { diff --git a/packages/importer/src/importer/importer.ts b/packages/importer/src/importer/importer.ts index 81577e4d66..4a79ce908e 100644 --- a/packages/importer/src/importer/importer.ts +++ b/packages/importer/src/importer/importer.ts @@ -57,7 +57,8 @@ import task, { type TaskType, type TaskTypeWithFactory } from '@hcengineering/task' -import { jsonToMarkup, parseMessageMarkdown } from '@hcengineering/text' +import { jsonToMarkup } from '@hcengineering/text' +import { markdownToMarkup } from '@hcengineering/text-markdown' import tracker, { type Issue, type IssueParentInfo, @@ -644,7 +645,7 @@ export class WorkspaceImporter { } async createComment (issueId: Ref, comment: ImportComment, projectId: Ref): Promise { - const json = parseMessageMarkdown(comment.text ?? '', 'image://') + const json = markdownToMarkup(comment.text ?? '') const processedJson = this.preprocessor.process(json, issueId, projectId) const markup = jsonToMarkup(processedJson) @@ -760,7 +761,7 @@ export class WorkspaceImporter { content: string, spaceId: Ref ): Promise> { - const json = parseMessageMarkdown(content ?? '', 'image://') + const json = markdownToMarkup(content ?? '') const processedJson = this.preprocessor.process(json, id, spaceId) const markup = jsonToMarkup(processedJson) diff --git a/packages/importer/src/notion/notion.ts b/packages/importer/src/notion/notion.ts index c61b56f207..35f265cddf 100644 --- a/packages/importer/src/notion/notion.ts +++ b/packages/importer/src/notion/notion.ts @@ -28,10 +28,10 @@ import { MarkupMarkType, type MarkupNode, MarkupNodeType, - parseMessageMarkdown, traverseNode, traverseNodeMarks } from '@hcengineering/text' +import { markdownToMarkup } from '@hcengineering/text-markdown' import { type Attachment } from '@hcengineering/attachment' import attachment from '@hcengineering/model-attachment' @@ -436,7 +436,7 @@ async function importPageDocument ( documentMetaMap?: Map ): Promise { const md = data.toString() ?? '' - const json = parseMessageMarkdown(md ?? '', 'image://') + const json = markdownToMarkup(md ?? '') if (documentMetaMap !== undefined) { preProcessMarkdown(json, documentMetaMap, fileUploader) } @@ -487,7 +487,7 @@ function preProcessMarkdown ( } else { traverseNodeMarks(node, (mark) => { if (mark.type === MarkupMarkType.link) { - const href = mark.attrs.href + const href = mark.attrs?.href ?? '' switch (getLinkType(href)) { case NOTION_MD_LINK_TYPES.UNKNOWN: case NOTION_MD_LINK_TYPES.EXTERNAL_LINK: { diff --git a/packages/presentation/src/components/markup/Mark.svelte b/packages/presentation/src/components/markup/Mark.svelte index e2f47d1c25..bd0a98fb2d 100644 --- a/packages/presentation/src/components/markup/Mark.svelte +++ b/packages/presentation/src/components/markup/Mark.svelte @@ -23,7 +23,7 @@ function handleLink (e: MouseEvent): void { try { - const href = mark.attrs.href + const href = mark.attrs?.href if (href != null && href !== '') { const url = new URL(href) const frontUrl = getMetadata(presentation.metadata.FrontUrl) ?? window.location.origin diff --git a/packages/presentation/src/components/markup/NodeContent.svelte b/packages/presentation/src/components/markup/NodeContent.svelte index c1c0eabc9f..33a59cbe25 100644 --- a/packages/presentation/src/components/markup/NodeContent.svelte +++ b/packages/presentation/src/components/markup/NodeContent.svelte @@ -35,7 +35,7 @@ } function toString (value: AttrValue | undefined): string | undefined { - return value !== undefined ? `${value}` : undefined + return value != null ? `${value}` : undefined } function toNumber (value: AttrValue | undefined): number | undefined { @@ -43,7 +43,7 @@ return value ? 1 : 0 } - return value !== undefined ? (typeof value === 'string' ? parseInt(value) : value) : undefined + return value != null ? (typeof value === 'string' ? parseInt(value) : value) : undefined } const checkEmoji = (nodes: MarkupNode[]): boolean => { diff --git a/packages/presentation/src/components/markup/lite/LiteNodeContent.svelte b/packages/presentation/src/components/markup/lite/LiteNodeContent.svelte index 9ba7bfe94c..135ad1e531 100644 --- a/packages/presentation/src/components/markup/lite/LiteNodeContent.svelte +++ b/packages/presentation/src/components/markup/lite/LiteNodeContent.svelte @@ -34,7 +34,7 @@ } function toString (value: AttrValue | undefined): string | undefined { - return value !== undefined ? `${value}` : undefined + return value != null ? `${value}` : undefined } function toNumber (value: AttrValue | undefined): number | undefined { @@ -42,7 +42,7 @@ return value ? 1 : 0 } - return value !== undefined ? (typeof value === 'string' ? parseInt(value) : value) : undefined + return value != null ? (typeof value === 'string' ? parseInt(value) : value) : undefined } diff --git a/packages/text-core/src/markup/model.ts b/packages/text-core/src/markup/model.ts index 255a2f401e..a851787dcb 100644 --- a/packages/text-core/src/markup/model.ts +++ b/packages/text-core/src/markup/model.ts @@ -55,10 +55,10 @@ export enum MarkupMarkType { /** @public */ export interface MarkupMark { type: MarkupMarkType - attrs: Record // A map of attributes + attrs?: Record // A map of attributes } -export type AttrValue = string | number | boolean +export type AttrValue = string | number | boolean | null | undefined export type Attrs = Record /** @public */ diff --git a/packages/text-core/src/markup/utils.ts b/packages/text-core/src/markup/utils.ts index 01e7fa62b9..f19e154831 100644 --- a/packages/text-core/src/markup/utils.ts +++ b/packages/text-core/src/markup/utils.ts @@ -17,7 +17,7 @@ import { Markup } from '@hcengineering/core' import { deepEqual } from 'fast-equals' import { nodeDoc, nodeParagraph, nodeText } from './dsl' -import { MarkupMark, MarkupMarkType, MarkupNode, MarkupNodeType, emptyMarkupNode, type AttrValue } from './model' +import { MarkupMark, MarkupNode, MarkupNodeType, emptyMarkupNode } from './model' import { traverseNode } from './traverse' /** @public */ @@ -188,221 +188,21 @@ export function stripTags (markup: Markup, textLimit = 0): string { return result } -class NodeBuilder { - textParts: string[] = [] - - constructor (private readonly addTags: boolean) {} - - addText (text: string): void { - this.textParts.push(text) - } - - addTag (text: string, newLine: boolean = false): void { - if (this.addTags) { - this.textParts.push(text) - } - if (!this.addTags && newLine) { - this.textParts.push('\n') - } - } - - toText (): string { - return this.textParts.join('') - } -} - -function addMark (builder: NodeBuilder, mark?: MarkupMark, next?: () => void): void { - if (mark != null) { - const attrs = mark.attrs ?? {} - - if (mark.type === MarkupMarkType.bold) { - builder.addTag('') - next?.() - builder.addTag('') - } else if (mark.type === MarkupMarkType.code) { - builder.addTag('') - next?.() - builder.addTag('') - } else if (mark.type === MarkupMarkType.em) { - builder.addTag('') - next?.() - builder.addTag('') - } else if (mark.type === MarkupMarkType.link) { - builder.addTag(``) - next?.() - builder.addTag('') - } else if (mark.type === MarkupMarkType.strike) { - builder.addTag('') - next?.() - builder.addTag('') - } else if (mark.type === MarkupMarkType.underline) { - builder.addTag('') - next?.() - builder.addTag('') - } else { - builder.addTag(`unknown mark: "${mark.type as string}"`, false) - next?.() - } - } -} - -function addMarks (builder: NodeBuilder, marks: MarkupMark[], next?: () => void): void { - if (marks.length > 0) { - const mark = marks[0] - const others = marks.slice(1) - - if (others.length > 0) { - addMark(builder, mark, () => { - addMarks(builder, others, next) - }) - } else { - addMark(builder, mark, next) - } - } -} - -function addNodeContent (builder: NodeBuilder, node?: MarkupNode): void { - if (node == null) return - - const attrs = node.attrs ?? {} - const nodes = node.content ?? [] - - if (node.type === MarkupNodeType.doc) { - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - } else if (node.type === MarkupNodeType.text) { - builder.addText(node.text ?? '') - } else if (node.type === MarkupNodeType.paragraph) { - builder.addTag('

') - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag('

') - } else if (node.type === MarkupNodeType.blockquote) { - builder.addTag('
') - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag('
') - } else if (node.type === MarkupNodeType.horizontal_rule) { - builder.addTag('
') - } else if (node.type === MarkupNodeType.heading) { - const level = toNumber(node.attrs?.level) ?? 1 - builder.addTag(``) - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag(``) - } else if (node.type === MarkupNodeType.code_block) { - builder.addTag('
')
-    nodes.forEach((childNode) => {
-      addNode(builder, childNode)
-    })
-    builder.addTag('
') - } else if (node.type === MarkupNodeType.image) { - const src = toString(attrs.src) - const alt = toString(attrs.alt) - builder.addText(`${alt}`) - } else if (node.type === MarkupNodeType.reference) { - const label = toString(attrs.label) - builder.addTag( - `` - ) - builder.addText(label !== undefined ? `@${label}` : '') - builder.addTag('') - } else if (node.type === MarkupNodeType.hard_break) { - builder.addTag('
') - } else if (node.type === MarkupNodeType.ordered_list) { - builder.addTag('
    ') - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag('
') - } else if (node.type === MarkupNodeType.bullet_list) { - builder.addTag('
    ') - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag('
') - } else if (node.type === MarkupNodeType.list_item) { - builder.addTag('
  • ') - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag('
    • ') - } else if (node.type === MarkupNodeType.subLink) { - builder.addTag('') - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag('') - } else if (node.type === MarkupNodeType.table) { - builder.addTag('') - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag('
    ') - } else if (node.type === MarkupNodeType.table_row) { - builder.addTag('') - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag('') - } else if (node.type === MarkupNodeType.table_cell) { - builder.addTag('') - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag('') - } else if (node.type === MarkupNodeType.table_header) { - builder.addTag('') - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - builder.addTag('') - } else { - builder.addText(`unknown node: "${node.type}"`) - nodes.forEach((childNode) => { - addNode(builder, childNode) - }) - } -} - -function addNode (builder: NodeBuilder, node: MarkupNode): void { - const marks = node.marks ?? [] - - if (marks.length > 0) { - addMarks(builder, marks, () => { - addNodeContent(builder, node) - }) - } else { - addNodeContent(builder, node) - } -} - -function toString (value: AttrValue | undefined): string | undefined { - return value !== undefined ? `${value}` : undefined -} - -function toNumber (value: AttrValue | undefined): number | undefined { - if (typeof value === 'boolean') { - return value ? 1 : 0 - } - - return value !== undefined ? (typeof value === 'string' ? parseInt(value) : value) : undefined -} - -export function markupToHTML (markup: Markup): string { - const jsonModel = markupToJSON(markup) - const builder = new NodeBuilder(true) - addNode(builder, jsonModel) - return builder.toText() -} - export function markupToText (markup: Markup): string { const jsonModel = markupToJSON(markup) - const builder = new NodeBuilder(false) - addNode(builder, jsonModel) - return builder.toText() + const fragments: string[] = [] + + traverseNode(jsonModel, (node) => { + if (node.type === MarkupNodeType.text) { + const text = node.text ?? '' + if (node.text !== undefined && node.text.length > 0) { + fragments.push(text) + } + } else if (node.type === MarkupNodeType.paragraph) { + fragments.push('\n\n') + } + return true + }) + + return fragments.join('').trim() } diff --git a/packages/text-html/.eslintrc.js b/packages/text-html/.eslintrc.js new file mode 100644 index 0000000000..72235dc283 --- /dev/null +++ b/packages/text-html/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/packages/text-html/config/rig.json b/packages/text-html/config/rig.json new file mode 100644 index 0000000000..0110930f55 --- /dev/null +++ b/packages/text-html/config/rig.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig" +} diff --git a/packages/text-html/jest.config.js b/packages/text-html/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/packages/text-html/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/packages/text-html/package.json b/packages/text-html/package.json new file mode 100644 index 0000000000..3a1f29653c --- /dev/null +++ b/packages/text-html/package.json @@ -0,0 +1,47 @@ +{ + "name": "@hcengineering/text-html", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "files": [ + "lib/**/*", + "types/**/*", + "tsconfig.json" + ], + "author": "Anticrm Platform Contributors", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "test": "jest --passWithNoTests --silent", + "build:watch": "compile", + "format": "format src", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "^0.6.0", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "typescript": "^5.3.3", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5" + }, + "dependencies": { + "@hcengineering/text-core": "^0.6.0", + "htmlparser2": "^9.0.0" + }, + "repository": "https://github.com/hcengineering/platform", + "publishConfig": { + "registry": "https://npm.pkg.github.com" + } +} diff --git a/packages/text-html/src/__tests__/html.test.ts b/packages/text-html/src/__tests__/html.test.ts new file mode 100644 index 0000000000..6efe92eea2 --- /dev/null +++ b/packages/text-html/src/__tests__/html.test.ts @@ -0,0 +1,467 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { MarkupNode } from '@hcengineering/text-core' +import { htmlToMarkup, markupToHtml } from '..' + +const tests: Array<{ name: string, markup: object, html: string }> = [ + { + name: 'paragraph', + markup: { + type: 'doc', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'paragraph 1' + } + ] + } + ] + }, + html: '

    paragraph 1

    ' + }, + { + name: 'text alignment', + markup: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { + level: 1, + textAlign: 'left' + }, + content: [ + { + type: 'text', + text: 'heading 1' + } + ] + }, + { + type: 'paragraph', + attrs: { + textAlign: 'right' + }, + content: [ + { + type: 'text', + text: 'paragraph 1' + } + ] + } + ] + }, + html: '

    heading 1

    paragraph 1

    ' + }, + { + name: 'headings', + markup: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { + level: 1, + textAlign: null + }, + content: [ + { + type: 'text', + text: 'heading 1' + } + ] + }, + { + type: 'heading', + attrs: { + level: 2, + textAlign: null + }, + content: [ + { + type: 'text', + text: 'heading 2' + } + ] + }, + { + type: 'heading', + attrs: { + level: 3, + textAlign: null + }, + content: [ + { + type: 'text', + text: 'heading 3' + } + ] + }, + { + type: 'heading', + attrs: { + level: 4, + textAlign: null + }, + content: [ + { + type: 'text', + text: 'heading 4' + } + ] + }, + { + type: 'heading', + attrs: { + level: 5, + textAlign: null + }, + content: [ + { + type: 'text', + text: 'heading 5' + } + ] + }, + { + type: 'heading', + attrs: { + level: 6, + textAlign: null + }, + content: [ + { + type: 'text', + text: 'heading 6' + } + ] + } + ] + }, + html: '

    heading 1

    heading 2

    heading 3

    heading 4

    heading 5
    heading 6
    ' + }, + { + name: 'blockquote', + markup: { + type: 'doc', + content: [ + { + type: 'blockquote', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'Lorem ipsum dolor sit amet.' + } + ] + } + ] + } + ] + }, + html: '

    Lorem ipsum dolor sit amet.

    ' + }, + { + name: 'codeblock', + markup: { + type: 'doc', + content: [ + { + type: 'codeBlock', + attrs: { + language: 'typescript' + }, + content: [ + { + type: 'text', + text: 'const x: number = 42;' + } + ] + } + ] + }, + html: '
    const x: number = 42;
    ' + }, + { + name: 'hr', + markup: { + type: 'doc', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'paragraph 1' + } + ] + }, + { + type: 'horizontalRule' + }, + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'paragraph 2' + } + ] + } + ] + }, + html: '

    paragraph 1

    paragraph 2

    ' + }, + { + name: 'br', + markup: { + type: 'doc', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'line 1' + }, + { + type: 'hardBreak' + }, + { + type: 'text', + text: 'line 2' + } + ] + } + ] + }, + html: '

    line 1
    line 2

    ' + }, + { + name: 'ordered list', + markup: { + type: 'doc', + content: [ + { + type: 'orderedList', + attrs: { + start: 1 + }, + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'item 1' + } + ] + } + ] + }, + { + type: 'listItem', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'item 2' + } + ] + } + ] + }, + { + type: 'listItem', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'item 3' + } + ] + } + ] + } + ] + } + ] + }, + html: '

    1. item 1

    2. item 2

    3. item 3

    ' + }, + { + name: 'bullet list', + markup: { + type: 'doc', + content: [ + { + type: 'bulletList', + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'item 1' + } + ] + } + ] + }, + { + type: 'listItem', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'item 2' + } + ] + } + ] + }, + { + type: 'listItem', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'item 3' + } + ] + } + ] + } + ] + } + ] + }, + html: '

    • item 1

    • item 2

    • item 3

    ' + }, + { + name: 'ref', + markup: { + type: 'doc', + content: [ + { + type: 'paragraph', + attrs: { + textAlign: null + }, + content: [ + { + type: 'text', + text: 'hello ' + }, + { + type: 'reference', + attrs: { + id: '64708c79c8f2613474dea38b', + objectclass: 'contact:class:Person', + label: 'John Doe' + } + } + ] + } + ] + }, + html: '

    hello @John Doe

    ' + } +] + +describe('markupToHtml', () => { + describe('convert to html', () => { + tests.forEach(({ name, markup, html }) => { + it(name, () => { + const result = markupToHtml(markup as MarkupNode) + expect(result).toEqual(html) + }) + }) + }) +}) + +describe('htmlToMarkup', () => { + describe('convert to markup', () => { + tests.forEach(({ name, markup, html }) => { + it(name, () => { + const result = htmlToMarkup(html) + expect(result).toEqual(markup) + }) + }) + }) + + describe('convert to markup and back', () => { + tests.forEach(({ name, markup, html }) => { + it(name, () => { + const result = htmlToMarkup(html) + const serialized = markupToHtml(result) + expect(serialized).toEqual(html) + }) + }) + }) +}) diff --git a/packages/text-html/src/index.ts b/packages/text-html/src/index.ts new file mode 100644 index 0000000000..1462f0008a --- /dev/null +++ b/packages/text-html/src/index.ts @@ -0,0 +1,28 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type MarkupNode } from '@hcengineering/text-core' +import { type HtmlParserOptions, HtmlParser } from './parser' +import { type HtmlSerializerOptions, HtmlSerializer } from './serializer' + +export function markupToHtml (markup: MarkupNode, options: HtmlSerializerOptions = {}): string { + const serializer = new HtmlSerializer(options) + return serializer.serialize(markup) +} + +export function htmlToMarkup (html: string, options: HtmlParserOptions = {}): MarkupNode { + const parser = new HtmlParser(options) + return parser.parse(html) +} diff --git a/packages/text-html/src/parser.ts b/packages/text-html/src/parser.ts new file mode 100644 index 0000000000..1435ea111a --- /dev/null +++ b/packages/text-html/src/parser.ts @@ -0,0 +1,564 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type AttrValue, + type MarkupMark, + type MarkupNode, + MarkupMarkType, + MarkupNodeType +} from '@hcengineering/text-core' +import { Parser } from 'htmlparser2' + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface HtmlParserOptions {} + +interface HtmlTagHandler { + handleOpenTag: (state: HtmlParseState, tag: string, attributes: Record) => void + handleCloseTag: (state: HtmlParseState, tag: string) => void +} + +interface HtmlNodeRule { + node: MarkupNodeType + getAttrs?: Record | ((attrs: Record) => Record | undefined) + wrapNode?: boolean + wrapContent?: boolean +} + +interface HtmlMarkRule { + mark: MarkupMarkType + getAttrs?: Record | ((attrs: Record) => Record | undefined) +} + +interface HtmlSpecialRule { + handleOpenTag: (state: HtmlParseState, tag: string, attributes: Record) => void + handleCloseTag: (state: HtmlParseState, tag: string) => void +} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +interface HtmlIgnoreRule {} + +interface HtmlStyleRule { + style: string + getAttrs?: (value: string) => Record | undefined +} + +class HtmlParseState { + private readonly stack: MarkupNode[] = [] + private readonly marks: MarkupMark[] = [] + + constructor ( + readonly root: MarkupNode, + readonly handlers: Record + ) { + this.stack.push(root) + } + + top (): MarkupNode | undefined { + return this.stack[this.stack.length - 1] + } + + addText (text: string): void { + const top = this.top() + if (top === undefined || text.length === 0) { + return + } + + const node: MarkupNode = + this.marks.length > 0 + ? { type: MarkupNodeType.text, text, marks: [...this.marks] } + : { type: MarkupNodeType.text, text } + + this.push(node) + } + + openMark (mark: MarkupMarkType, attrs?: Record): void { + this.marks.push(attrs !== undefined ? { type: mark, attrs } : { type: mark }) + } + + closeMark (mark: MarkupMarkType): void { + if (this.marks[this.marks.length - 1].type === mark) { + this.marks.pop() + } + } + + openNode (node: MarkupNodeType, attrs?: Record): void { + this.stack.push(attrs !== undefined ? { type: node, attrs } : { type: node }) + } + + closeNode (node: MarkupNodeType): void { + this.marks.splice(0) + const info = this.stack.pop() + if (info !== undefined) { + this.push(info) + } + } + + addNode (node: MarkupNode): void { + this.push(node) + } + + push (node: MarkupNode): void { + const parent = this.top() + if (parent !== undefined) { + const content = parent.content ?? [] + content.push(node) + parent.content = content + } + } +} + +function nodeHandler ({ node, getAttrs, wrapContent, wrapNode }: HtmlNodeRule): HtmlTagHandler { + return { + handleOpenTag: (state: HtmlParseState, tag: string, attributes: Record) => { + const attrs = + typeof getAttrs === 'function' + ? getAttrs(attributes) + : typeof getAttrs === 'object' + ? { ...getAttrs } + : undefined + + if (wrapNode === true) { + state.openNode(MarkupNodeType.paragraph) + } + + state.openNode(node, attrs) + + if (wrapContent === true) { + state.openNode(MarkupNodeType.paragraph) + } + }, + handleCloseTag: (state: HtmlParseState, tag: string) => { + if (wrapContent === true && state.top()?.type === MarkupNodeType.paragraph) { + state.closeNode(MarkupNodeType.paragraph) + } + + state.closeNode(node) + + if (wrapNode === true) { + state.closeNode(MarkupNodeType.paragraph) + } + } + } +} + +function markHandler ({ mark, getAttrs }: HtmlMarkRule): HtmlTagHandler { + return { + handleOpenTag: (state: HtmlParseState, tag: string, attributes: Record) => { + const attrs = + typeof getAttrs === 'function' + ? getAttrs(attributes) + : typeof getAttrs === 'object' + ? { ...getAttrs } + : undefined + state.openMark(mark, attrs) + }, + handleCloseTag: (state: HtmlParseState, tag: string) => { + state.closeMark(mark) + } + } +} + +function ignoreHandler (rule: HtmlIgnoreRule): HtmlTagHandler { + return { + handleOpenTag: () => {}, + handleCloseTag: () => {} + } +} + +function specialHandler (rule: HtmlSpecialRule): HtmlTagHandler { + return { + handleOpenTag: (state: HtmlParseState, tag: string, attributes: Record) => { + rule.handleOpenTag(state, tag, attributes) + }, + handleCloseTag: (state: HtmlParseState, tag: string) => { + rule.handleCloseTag(state, tag) + } + } +} + +const styleRules: HtmlStyleRule[] = [ + { + style: 'text-align', + getAttrs: (value: string) => { + return { textAlign: value ?? null } + } + } +] + +const markRules: Record = { + a: { + mark: MarkupMarkType.link, + getAttrs: (attributes: Record) => { + return { + href: attributes.href, + title: attributes.title + } + } + }, + b: { + mark: MarkupMarkType.bold + }, + em: { + mark: MarkupMarkType.em + }, + i: { + mark: MarkupMarkType.em + }, + s: { + mark: MarkupMarkType.strike + }, + strong: { + mark: MarkupMarkType.bold + }, + u: { + mark: MarkupMarkType.underline + } +} + +const nodeRules: Record = { + h1: { + node: MarkupNodeType.heading, + getAttrs: (attributes: Record) => { + return { + level: 1, + textAlign: attributes.textAlign ?? null + } + } + }, + h2: { + node: MarkupNodeType.heading, + getAttrs: (attributes: Record) => { + return { + level: 2, + textAlign: attributes.textAlign ?? null + } + } + }, + h3: { + node: MarkupNodeType.heading, + getAttrs: (attributes: Record) => { + return { + level: 3, + textAlign: attributes.textAlign ?? null + } + } + }, + h4: { + node: MarkupNodeType.heading, + getAttrs: (attributes: Record) => { + return { + level: 4, + textAlign: attributes.textAlign ?? null + } + } + }, + h5: { + node: MarkupNodeType.heading, + getAttrs: (attributes: Record) => { + return { + level: 5, + textAlign: attributes.textAlign ?? null + } + } + }, + h6: { + node: MarkupNodeType.heading, + getAttrs: (attributes: Record) => { + return { + level: 6, + textAlign: attributes.textAlign ?? null + } + } + }, + blockquote: { + node: MarkupNodeType.blockquote + }, + pre: { + node: MarkupNodeType.code_block + }, + hr: { + node: MarkupNodeType.horizontal_rule + }, + br: { + node: MarkupNodeType.hard_break + }, + ol: { + node: MarkupNodeType.ordered_list, + getAttrs: (attributes: Record) => { + return { + start: parseInt(attributes.starts ?? '1') + } + } + }, + ul: { + node: MarkupNodeType.bullet_list + }, + li: { + node: MarkupNodeType.list_item + }, + img: { + node: MarkupNodeType.image, + wrapNode: true, + getAttrs: (attributes: Record) => { + return { + src: attributes.src, + alt: attributes.alt, + title: attributes.title ?? null, + width: attributes.width !== undefined ? parseInt(attributes.width) : undefined, + height: attributes.height !== undefined ? parseInt(attributes.height) : undefined, + 'file-id': attributes['file-id'] ?? null + } + } + }, + sub: { + node: MarkupNodeType.subLink + }, + table: { + node: MarkupNodeType.table + }, + tr: { + node: MarkupNodeType.table_row + }, + th: { + node: MarkupNodeType.table_header, + getAttrs: (attributes: Record) => { + return { + colspan: attributes.colspan !== undefined ? parseInt(attributes.colspan) : undefined, + rowspan: attributes.rowspan !== undefined ? parseInt(attributes.rowspan) : undefined, + colwidth: attributes.colwidth !== undefined ? parseInt(attributes.colwidth) : undefined + } + }, + wrapContent: true + }, + td: { + node: MarkupNodeType.table_cell, + getAttrs: (attributes: Record) => { + return { + colspan: attributes.colspan !== undefined ? parseInt(attributes.colspan) : undefined, + rowspan: attributes.rowspan !== undefined ? parseInt(attributes.rowspan) : undefined, + colwidth: attributes.colwidth !== undefined ? parseInt(attributes.colwidth) : undefined + } + }, + wrapContent: true + }, + comment: { + node: MarkupNodeType.comment + } +} + +const specialRules: Record = { + p: { + handleOpenTag: (state: HtmlParseState, tag: string, attributes: Record) => { + const top = state.top() + if (top?.type !== MarkupNodeType.paragraph) { + const attrs = { textAlign: attributes.textAlign ?? null } + state.openNode(MarkupNodeType.paragraph, attrs) + } + }, + handleCloseTag: (state: HtmlParseState, tag: string) => { + const top = state.top() + if (top?.type === MarkupNodeType.paragraph) { + state.closeNode(MarkupNodeType.paragraph) + } + } + }, + span: { + handleOpenTag: (state: HtmlParseState, tag: string, attributes: Record) => { + const dataType = attributes['data-type'] + if (dataType === 'reference') { + state.openNode(MarkupNodeType.reference, { + id: attributes['data-id'], + objectclass: attributes['data-objectclass'], + label: attributes['data-label'] + }) + } + }, + handleCloseTag: (state: HtmlParseState, tag: string) => { + const top = state.top() + if (top?.type === MarkupNodeType.reference) { + delete top.content + state.closeNode(MarkupNodeType.reference) + } + } + }, + code: { + handleOpenTag: (state: HtmlParseState, tag: string, attributes: Record) => { + const top = state.top() + if (top?.type === MarkupNodeType.code_block) { + const classes = attributes.class?.split(' ') ?? [] + const language = classes.find((c) => c.startsWith('language-')) + if (language !== undefined) { + top.attrs = { language: language.substring(9) } + } + } else { + state.openMark(MarkupMarkType.code) + } + }, + handleCloseTag: (state: HtmlParseState, tag: string) => { + const top = state.top() + if (top?.type === MarkupNodeType.code_block) { + // do nothing + } else { + state.closeMark(MarkupMarkType.code) + } + } + } +} + +const ignoreRules: Record = { + html: {}, + head: {}, + body: {}, + thead: {}, + tbody: {} +} + +export class HtmlParser { + private readonly handlers: Record = {} + + constructor (private readonly options: HtmlParserOptions = {}) { + Object.entries(nodeRules).forEach(([tag, rule]) => { + this.handlers[tag] = nodeHandler(rule) + }) + + Object.entries(markRules).forEach(([tag, rule]) => { + this.handlers[tag] = markHandler(rule) + }) + + Object.entries(specialRules).forEach(([tag, rule]) => { + this.handlers[tag] = specialHandler(rule) + }) + + Object.entries(ignoreRules).forEach(([tag, rule]) => { + this.handlers[tag] = ignoreHandler(rule) + }) + } + + parse (html: string): MarkupNode { + const root: MarkupNode = { type: MarkupNodeType.doc, content: [] } + const state = new HtmlParseState(root, this.handlers) + + let rawPos: number | undefined + let rawDepth: number | undefined + + const parser = new Parser( + { + onopentag (tag: string, attributes: Record) { + if (rawDepth !== undefined) { + rawDepth += 1 + return + } + + if (attributes.style !== undefined) { + const attrs = extractStyleAttrs(attributes, styleRules) + if (attrs !== undefined) { + attributes = { ...attributes, ...attrs } + } + } + + const handler = state.handlers[tag] + if (handler !== undefined) { + handler.handleOpenTag(state, tag, attributes) + } else { + rawPos = parser.startIndex + rawDepth = 0 + } + }, + + ontext (text: string) { + if (rawDepth !== undefined) { + return + } + + if (text.length === 0) { + return + } + + state.addText(text) + }, + + oncomment (text: string) { + if (rawDepth !== undefined) { + return + } + + state.openNode(MarkupNodeType.comment) + state.addText(text) + state.closeNode(MarkupNodeType.comment) + }, + + onclosetag (tag: string) { + if (rawDepth !== undefined) { + rawDepth -= 1 + + if (rawDepth === 0) { + const start = rawPos ?? 0 + const end = parser.endIndex + const text = html.substring(start, end) + state.addText(text) + rawPos = undefined + rawDepth = undefined + } + return + } + + const handler = state.handlers[tag] + if (handler !== undefined) { + handler.handleCloseTag(state, tag) + } else { + // do nothing + } + } + }, + { + decodeEntities: true, + lowerCaseTags: true, + recognizeSelfClosing: true + } + ) + + parser.write(html) + parser.end() + + return state.root + } +} + +function extractStyleAttrs (attrs: Record, rules: HtmlStyleRule[]): Record | undefined { + const style = attrs.style + if (style !== undefined) { + const styles: Record = {} + + // parse style attribute + style.split(';').forEach((stylePart) => { + const [key, value] = stylePart.split(':') + styles[key.trim()] = value?.trim() + }) + + const result = {} + + rules.forEach((rule) => { + if (styles[rule.style] !== undefined) { + const attrs = rule.getAttrs?.(styles[rule.style]) + if (attrs !== undefined) { + Object.assign(result, attrs) + } + } + }) + + return result + } +} diff --git a/packages/text-html/src/serializer.ts b/packages/text-html/src/serializer.ts new file mode 100644 index 0000000000..d7191b568c --- /dev/null +++ b/packages/text-html/src/serializer.ts @@ -0,0 +1,324 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type Attrs, + type AttrValue, + type MarkupMark, + type MarkupNode, + MarkupMarkType, + MarkupNodeType +} from '@hcengineering/text-core' + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface HtmlSerializerOptions {} + +export class HtmlSerializer { + constructor (private readonly options: HtmlSerializerOptions = {}) {} + + serialize (markup: MarkupNode): string { + const builder = new NodeBuilder(true) + addNode(builder, markup) + return builder.toText() + } +} + +class NodeBuilder { + textParts: string[] = [] + + constructor (private readonly addTags: boolean) {} + + addText (text: string): void { + this.textParts.push(text) + } + + openTag ( + tag: string, + attributes: Record = {}, + options?: { newLine?: boolean, selfClosing?: boolean } + ): void { + if (this.addTags) { + this.textParts.push('<') + this.textParts.push(tag) + + for (const [key, value] of Object.entries(attributes)) { + if (value == null) continue + + if (typeof value === 'boolean') { + if (value) { + this.textParts.push(` ${key}`) + } + } else { + this.textParts.push(` ${key}="${escapeHtml(String(value))}"`) + } + } + + this.textParts.push(options?.selfClosing === true ? '/>' : '>') + } else { + if (options?.newLine === true) { + this.textParts.push('\n') + } + } + } + + closeTag (tag: string, options?: { newLine: boolean }): void { + if (this.addTags) { + this.textParts.push(``) + } else if (options?.newLine === true) { + this.textParts.push('\n') + } + } + + toText (): string { + return this.textParts.join('') + } +} + +// Helper function to escape HTML special characters +function escapeHtml (text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') +} + +function addMark (builder: NodeBuilder, mark?: MarkupMark, next?: () => void): void { + if (mark != null) { + const attrs = mark.attrs ?? {} + + if (mark.type === MarkupMarkType.bold) { + builder.openTag('strong') + next?.() + builder.closeTag('strong') + } else if (mark.type === MarkupMarkType.code) { + builder.openTag('code') + next?.() + builder.closeTag('code') + } else if (mark.type === MarkupMarkType.em) { + builder.openTag('em') + next?.() + builder.closeTag('em') + } else if (mark.type === MarkupMarkType.link) { + builder.openTag('a', { href: attrs.href, target: attrs.target }) + next?.() + builder.closeTag('a') + } else if (mark.type === MarkupMarkType.strike) { + builder.openTag('s') + next?.() + builder.closeTag('s') + } else if (mark.type === MarkupMarkType.underline) { + builder.openTag('u') + next?.() + builder.closeTag('u') + } else { + // Handle unknown mark as span with data attribute + builder.openTag('span', { 'data-mark-type': mark.type as string }) + next?.() + builder.closeTag('span') + } + } +} + +function addMarks (builder: NodeBuilder, marks: MarkupMark[], next?: () => void): void { + if (marks.length > 0) { + const mark = marks[0] + const others = marks.slice(1) + + if (others.length > 0) { + addMark(builder, mark, () => { + addMarks(builder, others, next) + }) + } else { + addMark(builder, mark, next) + } + } +} + +function addNodes (builder: NodeBuilder, nodes: MarkupNode[]): void { + nodes.forEach((childNode) => { + addNode(builder, childNode) + }) +} + +function addNodeContent (builder: NodeBuilder, node?: MarkupNode): void { + if (node == null) return + + const attrs = node.attrs ?? {} + const nodes = node.content ?? [] + const style = toStyleAttr(attrs) + + if (node.type === MarkupNodeType.doc) { + addNodes(builder, nodes) + } else if (node.type === MarkupNodeType.paragraph) { + builder.openTag('p', { style }) + addNodes(builder, nodes) + builder.closeTag('p') + } else if (node.type === MarkupNodeType.blockquote) { + builder.openTag('blockquote') + addNodes(builder, nodes) + builder.closeTag('blockquote') + } else if (node.type === MarkupNodeType.horizontal_rule) { + builder.openTag('hr', {}, { selfClosing: true }) + } else if (node.type === MarkupNodeType.heading) { + const level = toNumber(attrs.level) ?? 1 + const tag = `h${level}` + builder.openTag(tag, { style }) + addNodes(builder, nodes) + builder.closeTag(tag) + } else if (node.type === MarkupNodeType.code_block) { + const attrs = node.attrs?.language !== undefined ? { class: `language-${node.attrs.language}` } : {} + builder.openTag('pre') + builder.openTag('code', attrs) + addNodes(builder, nodes) + builder.closeTag('code') + builder.closeTag('pre') + } else if (node.type === MarkupNodeType.text) { + builder.addText(node.text ?? '') + } else if (node.type === MarkupNodeType.image) { + const src = toString(attrs.src) + const alt = toString(attrs.alt) + const width = toString(attrs.width) + const height = toString(attrs.height) + builder.openTag('img', { src, alt, width, height }, { selfClosing: true }) + } else if (node.type === MarkupNodeType.reference) { + const label = toString(attrs.label) + builder.openTag('span', { + 'data-type': 'reference', + 'data-id': attrs.id, + 'data-objectclass': attrs.objectclass, + 'data-label': attrs.label + }) + builder.addText(label !== undefined ? `@${label}` : '') + builder.closeTag('span') + } else if (node.type === MarkupNodeType.hard_break) { + builder.openTag('br', {}, { selfClosing: true }) + } else if (node.type === MarkupNodeType.ordered_list) { + builder.openTag('ol') + addNodes(builder, nodes) + builder.closeTag('ol') + } else if (node.type === MarkupNodeType.bullet_list) { + builder.openTag('ul') + addNodes(builder, nodes) + builder.closeTag('ul') + } else if (node.type === MarkupNodeType.list_item) { + builder.openTag('li') + addNodes(builder, nodes) + builder.closeTag('li') + } else if (node.type === MarkupNodeType.todoList) { + builder.openTag('ul', { 'data-type': MarkupNodeType.todoList }) + addNodes(builder, nodes) + builder.closeTag('ul') + } else if (node.type === MarkupNodeType.todoItem) { + const checked = node.attrs?.checked === true || node.attrs?.checked === 'true' + const disabled = node.attrs?.disabled === true || node.attrs?.disabled === 'true' + + builder.openTag('li', { + 'data-type': MarkupNodeType.todoItem, + 'data-todoid': node.attrs?.todoid, + 'data-userid': node.attrs?.userid, + 'data-checked': checked + }) + + builder.openTag('input', { type: 'checkbox', checked, disabled }, { selfClosing: true }) + + addNodes(builder, nodes) + + builder.closeTag('li') + } else if (node.type === MarkupNodeType.taskList) { + builder.openTag('ul', { 'data-type': MarkupNodeType.taskList }) + addNodes(builder, nodes) + builder.closeTag('ul') + } else if (node.type === MarkupNodeType.taskItem) { + const checked = node.attrs?.checked === true || node.attrs?.checked === 'true' + const disabled = node.attrs?.disabled === true || node.attrs?.disabled === 'true' + + builder.openTag('li', { + 'data-type': MarkupNodeType.taskItem, + 'data-checked': checked + }) + builder.openTag('input', { type: 'checkbox', checked, disabled }, { selfClosing: true }) + + addNodes(builder, nodes) + + builder.closeTag('li') + } else if (node.type === MarkupNodeType.subLink) { + builder.openTag('sub') + addNodes(builder, nodes) + builder.closeTag('sub') + } else if (node.type === MarkupNodeType.table) { + builder.openTag('table') + builder.openTag('tbody') + addNodes(builder, nodes) + builder.closeTag('tbody') + builder.closeTag('table') + } else if (node.type === MarkupNodeType.table_row) { + builder.openTag('tr') + addNodes(builder, nodes) + builder.closeTag('tr') + } else if (node.type === MarkupNodeType.table_cell) { + builder.openTag('td') + addNodes(builder, nodes) + builder.closeTag('td') + } else if (node.type === MarkupNodeType.table_header) { + builder.openTag('th') + addNodes(builder, nodes) + builder.closeTag('th') + } else if (node.type === MarkupNodeType.comment) { + builder.addText('') + } else { + // Handle unknown node types as div with data attribute + builder.openTag('div', { 'data-node-type': node.type }) + addNodes(builder, nodes) + builder.closeTag('div') + } +} + +function addNode (builder: NodeBuilder, node: MarkupNode): void { + const marks = node.marks ?? [] + + if (marks.length > 0) { + addMarks(builder, marks, () => { + addNodeContent(builder, node) + }) + } else { + addNodeContent(builder, node) + } +} + +function toString (value: AttrValue | undefined): string | undefined { + return value !== undefined ? `${value}` : undefined +} + +function toNumber (value: AttrValue | undefined): number | undefined { + if (typeof value === 'boolean') { + return value ? 1 : 0 + } + + return value != null ? (typeof value === 'string' ? parseInt(value) : value) : undefined +} + +function toStyleAttr (attrs: Attrs): string | undefined { + const styles: string[] = [] + + if (attrs.textAlign != null) { + styles.push(`text-align: ${attrs.textAlign}`) + } + + return styles.length > 0 ? styles.join('; ') : undefined +} diff --git a/packages/text-html/tsconfig.json b/packages/text-html/tsconfig.json new file mode 100644 index 0000000000..59e4fd4297 --- /dev/null +++ b/packages/text-html/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/packages/text-markdown/.eslintrc.js b/packages/text-markdown/.eslintrc.js new file mode 100644 index 0000000000..72235dc283 --- /dev/null +++ b/packages/text-markdown/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/packages/text-markdown/config/rig.json b/packages/text-markdown/config/rig.json new file mode 100644 index 0000000000..0110930f55 --- /dev/null +++ b/packages/text-markdown/config/rig.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig" +} diff --git a/packages/text-markdown/jest.config.js b/packages/text-markdown/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/packages/text-markdown/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/packages/text-markdown/package.json b/packages/text-markdown/package.json new file mode 100644 index 0000000000..b6b31f11b4 --- /dev/null +++ b/packages/text-markdown/package.json @@ -0,0 +1,50 @@ +{ + "name": "@hcengineering/text-markdown", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "files": [ + "lib/**/*", + "types/**/*", + "tsconfig.json" + ], + "author": "Anticrm Platform Contributors", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "test": "jest --passWithNoTests --silent", + "build:watch": "compile", + "format": "format src", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "^0.6.0", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "typescript": "^5.3.3", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5", + "@types/markdown-it": "~13.0.0" + }, + "dependencies": { + "@hcengineering/text-core": "^0.6.0", + "@hcengineering/text-html": "^0.6.0", + "markdown-it": "^14.0.0", + "fast-equals": "^5.2.2" + }, + "repository": "https://github.com/hcengineering/platform", + "publishConfig": { + "registry": "https://npm.pkg.github.com" + } +} diff --git a/packages/text-markdown/src/__tests__/markdown.test.ts b/packages/text-markdown/src/__tests__/markdown.test.ts new file mode 100644 index 0000000000..4176a81a6c --- /dev/null +++ b/packages/text-markdown/src/__tests__/markdown.test.ts @@ -0,0 +1,830 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { markdownToMarkup, markupToMarkdown } from '..' +import { isMarkdownsEquals } from '../compare' + +const refUrl: string = 'ref://' +const imageUrl: string = 'http://localhost' +const options = { refUrl, imageUrl } + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace jest { + interface Matchers { + toEqualMarkdown: (expected: string) => R + } + } +} + +expect.extend({ + toEqualMarkdown (received: string, expected: string) { + const pass = isMarkdownsEquals(received, expected) + return { + message: () => + pass + ? `Expected markdown strings NOT to be equal:\n Received: ${received}\n Expected: ${expected}` + : `Expected markdown strings to be equal:\n Received: ${received}\n Expected: ${expected}`, + pass + } + } +}) + +describe('markdownToMarkup', () => { + const tests: Array<{ name: string, markdown: string, markup: object }> = [ + { + name: 'simple text', + markdown: 'Lorem ipsum dolor sit amet.', + markup: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'Lorem ipsum dolor sit amet.', + marks: [] + } + ] + } + ] + } + }, + { + name: 'text with heading', + markdown: `# Lorem ipsum + +Lorem ipsum dolor sit amet. +`, + markup: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { level: 1 }, + content: [ + { + type: 'text', + text: 'Lorem ipsum', + marks: [] + } + ] + }, + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'Lorem ipsum dolor sit amet.', + marks: [] + } + ] + } + ] + } + }, + { + name: 'bullet list', + markdown: `# bullet list +- list item 1 +- list item 2 +`, + markup: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { level: 1 }, + content: [ + { + type: 'text', + text: 'bullet list', + marks: [] + } + ] + }, + { + type: 'bulletList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'list item 1', + marks: [] + } + ] + } + ] + }, + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'list item 2', + marks: [] + } + ] + } + ] + } + ] + } + ] + } + }, + { + name: 'todos', + markdown: `# TODO +- [ ] todo 1 +- [x] todo 2 + `, + markup: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { level: 1 }, + content: [ + { + type: 'text', + text: 'TODO', + marks: [] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: false }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'todo 1', + marks: [] + } + ] + } + ] + }, + { + type: 'todoItem', + attrs: { checked: true }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'todo 2', + marks: [] + } + ] + } + ] + } + ] + } + ] + } + }, + { + name: 'todos followed by list items', + markdown: `# todo and list +- [ ] todo 1 +- [x] todo 2 +- list item 1 +- list item 2 +`, + markup: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { level: 1 }, + content: [ + { + type: 'text', + text: 'todo and list', + marks: [] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: false }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'todo 1', + marks: [] + } + ] + } + ] + }, + { + type: 'todoItem', + attrs: { checked: true }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'todo 2', + marks: [] + } + ] + } + ] + } + ] + }, + { + type: 'bulletList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'list item 1', + marks: [] + } + ] + } + ] + }, + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'list item 2', + marks: [] + } + ] + } + ] + } + ] + } + ] + } + }, + { + name: 'todos followed by list items', + markdown: `# mixed lists +- [ ] todo 1 +- list item 1 +- [x] todo 2 +- list item 2 +`, + markup: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { level: 1 }, + content: [ + { + type: 'text', + text: 'mixed lists', + marks: [] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: false }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'todo 1', + marks: [] + } + ] + } + ] + } + ] + }, + { + type: 'bulletList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'list item 1', + marks: [] + } + ] + } + ] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: true }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'todo 2', + marks: [] + } + ] + } + ] + } + ] + }, + { + type: 'bulletList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'list item 2', + marks: [] + } + ] + } + ] + } + ] + } + ] + } + }, + { + name: 'nested todos', + markdown: `# nested todos +- [ ] todo + - [x] sub todo +`, + markup: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { level: 1 }, + content: [ + { + type: 'text', + text: 'nested todos', + marks: [] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: false }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'todo', + marks: [] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: true }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'sub todo', + marks: [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + }, + { + name: 'nested lists', + markdown: `# nested lists +- [ ] todo + - sub list item + - [x] sub todo +- list item + - [x] sub todo + - sub list item +`, + markup: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { level: 1 }, + content: [ + { + type: 'text', + text: 'nested lists', + marks: [] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: false }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'todo', + marks: [] + } + ] + }, + { + type: 'bulletList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'sub list item', + marks: [] + } + ] + } + ] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: true }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'sub todo', + marks: [] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + { + type: 'bulletList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'list item', + marks: [] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: true }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'sub todo', + marks: [] + } + ] + } + ] + } + ] + }, + { + type: 'bulletList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'listItem', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'sub list item', + marks: [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + }, + { + name: 'nested todos', + markdown: `# nested todos +- [ ] todo + - [x] sub todo +`, + markup: { + type: 'doc', + content: [ + { + type: 'heading', + attrs: { level: 1 }, + content: [ + { + type: 'text', + text: 'nested todos', + marks: [] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: false }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'todo', + marks: [] + } + ] + }, + { + type: 'todoList', + attrs: { + bullet: '-' + }, + content: [ + { + type: 'todoItem', + attrs: { checked: true }, + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'sub todo', + marks: [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + } + ] + + describe('to markup', () => { + tests.forEach(({ name, markdown, markup }) => { + it(name, () => { + const parsed = markdownToMarkup(markdown, options) + expect(parsed).toEqual(markup) + }) + }) + }) + + describe('to markup and back', () => { + tests.forEach(({ name, markdown, markup }) => { + it(name, () => { + const json = markdownToMarkup(markdown, options) + const serialized = markupToMarkdown(json, options) + expect(serialized).toEqualMarkdown(markdown) + }) + }) + }) +}) + +describe('markdownToMarkup -> markupToMarkdown', () => { + const tests: Array<{ name: string, markdown: string }> = [ + { name: 'Italic', markdown: '*Asteriscs* and _Underscores_' }, + { name: 'Bold', markdown: '**Asteriscs** and __Underscores__' }, + { name: 'Bullet list with asteriscs', markdown: 'Asterisks :\r\n* Firstly\r\n* Secondly' }, + { name: 'Bullet list with dashes', markdown: 'Dashes :\r\n- Firstly\r\n- Secondly' }, + { name: 'TODO list with asteriscs', markdown: '* [ ] Take\n* [ ] Do\n\n' }, + { name: 'TODO list with dashes', markdown: '- [x] Take\n- [ ] Do\n\n' }, + { + name: 'Different markers', + markdown: + 'Asterisks bulleted list:\r\n* Asterisks: *Italic* and **Bold*** Underscores: _Italic_ and __Bold__\r\n\r\nDash bulleted list:\r\n- Asterisks: *Italic* and **Bold**\r\n- Underscores: _Italic_ and __Bold__\r\n-' + }, + { name: 'Single line comment', markdown: '' }, + { + name: 'Multsiline comment', + markdown: + '"\r\n\r\nThis PR fix issue #22424\r\n\r\n\r\n"' + }, + { + name: 'Link', + markdown: 'See [link](https://example.com)' + }, + { + name: 'Codeblock', + markdown: '```typescript\nconst x: number = 42;\n```' + }, + { + name: 'Image', + markdown: 'image' + }, + { + name: 'Table', + markdown: + '

    Header 1

    Header 2

    Cell 1

    Cell 2

    Cell 3

    Cell 4

    ' + }, + { + name: 'Sub', + markdown: 'View in Huly TSK-50' + } + // { + // name: 'Malformed', + // markdown: 'try to parse me' + // } + ] + + tests.forEach(({ name, markdown }) => { + it(name, () => { + const json = markdownToMarkup(markdown, options) + const serialized = markupToMarkdown(json, options) + expect(serialized).toEqualMarkdown(markdown) + }) + }) +}) diff --git a/packages/text/src/markdown/compare.ts b/packages/text-markdown/src/compare.ts similarity index 72% rename from packages/text/src/markdown/compare.ts rename to packages/text-markdown/src/compare.ts index bca7202cea..6dcaf87101 100644 --- a/packages/text/src/markdown/compare.ts +++ b/packages/text-markdown/src/compare.ts @@ -1,3 +1,18 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + /** * Calculate Sørensen–Dice coefficient */ diff --git a/packages/text-markdown/src/index.ts b/packages/text-markdown/src/index.ts new file mode 100644 index 0000000000..7618c5548b --- /dev/null +++ b/packages/text-markdown/src/index.ts @@ -0,0 +1,47 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { MarkupNode } from '@hcengineering/text-core' +import { MarkdownParser } from './parser' +import { MarkdownState, storeMarks, storeNodes } from './serializer' + +export * from './compare' +export * from './parser' +export * from './serializer' + +/** @public */ +export interface MarkdownOptions { + refUrl?: string + imageUrl?: string +} + +/** @public */ +export function markupToMarkdown (markup: MarkupNode, options?: MarkdownOptions): string { + const refUrl = options?.refUrl ?? 'ref://' + const imageUrl = options?.imageUrl ?? 'image://' + + const state = new MarkdownState(storeNodes, storeMarks, { tightLists: true, refUrl, imageUrl }) + state.renderContent(markup) + return state.out +} + +/** @public */ +export function markdownToMarkup (markdown: string, options?: MarkdownOptions): MarkupNode { + const refUrl = options?.refUrl ?? 'ref://' + const imageUrl = options?.imageUrl ?? 'image://' + + const parser = new MarkdownParser({ refUrl, imageUrl }) + return parser.parse(markdown ?? '') +} diff --git a/packages/text/src/markdown/marks.ts b/packages/text-markdown/src/marks.ts similarity index 54% rename from packages/text/src/markdown/marks.ts rename to packages/text-markdown/src/marks.ts index 4d7535f76b..66585ddddb 100644 --- a/packages/text/src/markdown/marks.ts +++ b/packages/text-markdown/src/marks.ts @@ -1,12 +1,23 @@ -import { deepEqual } from 'fast-equals' -import { MarkupMark, MarkupMarkType, MarkupNode } from '@hcengineering/text-core' +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// -export function traverseMarks (node: MarkupNode, f: (el: MarkupMark) => void): void { - node.marks?.forEach(f) -} +import { MarkupMark, MarkupMarkType } from '@hcengineering/text-core' +import { deepEqual } from 'fast-equals' export function markAttrs (mark: MarkupMark): Record { - return mark.attrs ?? [] + return mark.attrs ?? {} } export function isInSet (mark: MarkupMark, marks: MarkupMark[]): boolean { diff --git a/packages/text-markdown/src/node.ts b/packages/text-markdown/src/node.ts new file mode 100644 index 0000000000..45c6f286bc --- /dev/null +++ b/packages/text-markdown/src/node.ts @@ -0,0 +1,24 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Attrs, MarkupNode } from '@hcengineering/text-core' + +export function nodeContent (node: MarkupNode): MarkupNode[] { + return node?.content ?? [] +} + +export function nodeAttrs (node: MarkupNode): Attrs { + return node.attrs ?? {} +} diff --git a/packages/text/src/markdown/parser.ts b/packages/text-markdown/src/parser.ts similarity index 93% rename from packages/text/src/markdown/parser.ts rename to packages/text-markdown/src/parser.ts index e517a82d7d..cf6ba219df 100644 --- a/packages/text/src/markdown/parser.ts +++ b/packages/text-markdown/src/parser.ts @@ -1,12 +1,26 @@ -import { Extensions } from '@tiptap/core' +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Attrs, MarkupMark, MarkupMarkType, MarkupNode, MarkupNodeType } from '@hcengineering/text-core' +import { htmlToMarkup } from '@hcengineering/text-html' import MarkdownIt, { type Token } from 'markdown-it' import type { RuleCore } from 'markdown-it/lib/parser_core' import type StateCore from 'markdown-it/lib/rules_core/state_core' -import { Attrs, AttrValue, MarkupMark, MarkupMarkType, MarkupNode, MarkupNodeType } from '@hcengineering/text-core' -import { htmlToJSON } from '../markup/utils' import { addToSet, removeFromSet, sameSet } from './marks' -import { messageContent } from './node' +import { nodeContent } from './node' interface ParsingBlockRule { block: MarkupNodeType @@ -40,7 +54,7 @@ type HandlerRecord = (state: MarkdownParseState, tok: Token) => void type HandlersRecord = Record // **************************************************************** -// Mark down parser +// Markdown parser // **************************************************************** function isText (a: MarkupNode, b: MarkupNode): boolean { return (a.type === MarkupNodeType.text || a.type === MarkupNodeType.reference) && b.type === MarkupNodeType.text @@ -98,7 +112,6 @@ class MarkdownParseState { return false } - // : (string) // Adds the given text to the current position in the document, // using the current marks as styling. addText (text?: string): void { @@ -122,22 +135,11 @@ class MarkdownParseState { } } - addAttr (key: string, value: AttrValue): void { - const top = this.top() - if (top === undefined) { - return - } - - top.attrs[key] = value - } - - // : (Mark) // Adds the given mark to the set of active marks. openMark (mark: MarkupMark): void { this.marks = addToSet(mark, this.marks) } - // : (Mark) // Removes the given mark from the set of active marks. closeMark (mark: MarkupMarkType): void { this.marks = removeFromSet(mark, this.marks) @@ -172,7 +174,6 @@ class MarkdownParseState { } } - // : (NodeType, ?Object, ?[Node]) → ?Node // Add a node at the current position. addNode (type: MarkupNodeType, attrs: Attrs, content: MarkupNode[] = []): MarkupNode { const node: MarkupNode = { type, content } @@ -187,13 +188,11 @@ class MarkdownParseState { return node } - // : (NodeType, ?Object) // Wrap subsequent content in a node of the given type. openNode (type: MarkupNodeType, attrs: Attrs): void { this.stack.push({ type, attrs, content: [] }) } - // : () → ?Node // Close and return the node that is currently on top of the stack. closeNode (): MarkupNode { if (this.marks.length > 0) this.marks = [] @@ -297,7 +296,7 @@ function tokenHandlers ( tokensMark: Record, specialRules: Record, ignoreRules: Record, - extensions: Extensions + htmlParser: HtmlParser ): HandlersRecord { const handlers: HandlersRecord = {} @@ -319,15 +318,15 @@ function tokenHandlers ( handlers.html_inline = (state: MarkdownParseState, tok: Token) => { try { - const model = htmlToJSON(tok.content, extensions) - if (model.content !== undefined) { + const markup = htmlParser(tok.content) + if (markup.content !== undefined) { // unwrap content from wrapping paragraph const shouldUnwrap = - model.content.length === 1 && - model.content[0].type === MarkupNodeType.paragraph && + markup.content.length === 1 && + markup.content[0].type === MarkupNodeType.paragraph && state.top()?.type === MarkupNodeType.paragraph - const content = messageContent(shouldUnwrap ? model.content[0] : model) + const content = nodeContent(shouldUnwrap ? markup.content[0] : markup) for (const c of content) { state.push(c) } @@ -339,8 +338,8 @@ function tokenHandlers ( } handlers.html_block = (state: MarkdownParseState, tok: Token) => { try { - const model = htmlToJSON(tok.content, extensions) - const content = messageContent(model) + const model = htmlParser(tok.content) + const content = nodeContent(model) for (const c of content) { state.push(c) } @@ -463,7 +462,7 @@ const tokensBlock: Record = { noCloseToken: true }, sub: { - block: MarkupNodeType.code_block, + block: MarkupNodeType.subLink, getAttrs: (tok: Token) => { return { language: tok.info ?? '' } }, @@ -614,26 +613,32 @@ interface TaskListStateCore extends StateCore { const startsWithTodoMarkdown = (token: Token): boolean => /^\[[xX \u00A0]\][ \u00A0]/.test(token.content) const isCheckedTodoItem = (token: Token): boolean => /^\[[xX]\][ \u00A0]/.test(token.content) +export type HtmlParser = (html: string) => MarkupNode + +export interface MarkdownParserOptions { + refUrl: string + imageUrl: string + htmlParser?: HtmlParser +} + export class MarkdownParser { tokenizer: MarkdownIt tokenHandlers: Record void> + htmlParser: HtmlParser - constructor ( - readonly extensions: Extensions, - readonly refUrl: string, - readonly imageUrl: string - ) { + constructor (private readonly options: MarkdownParserOptions) { this.tokenizer = MarkdownIt('default', { html: true }) this.tokenizer.core.ruler.after('inline', 'task_list', this.listRule) this.tokenizer.core.ruler.after('inline', 'html_comment', this.htmlCommentRule) - this.tokenHandlers = tokenHandlers(tokensBlock, tokensNode, tokensMark, specialRule, ignoreRule, extensions) + this.htmlParser = options.htmlParser ?? htmlToMarkup + this.tokenHandlers = tokenHandlers(tokensBlock, tokensNode, tokensMark, specialRule, ignoreRule, this.htmlParser) } parse (text: string): MarkupNode { - const state = new MarkdownParseState(this.tokenHandlers, this.refUrl, this.imageUrl) + const state = new MarkdownParseState(this.tokenHandlers, this.options.refUrl, this.options.imageUrl) let doc: MarkupNode const tokens = this.tokenizer.parse(text, {}) diff --git a/packages/text/src/markdown/serializer.ts b/packages/text-markdown/src/serializer.ts similarity index 92% rename from packages/text/src/markdown/serializer.ts rename to packages/text-markdown/src/serializer.ts index a78d8572db..69410f9f2d 100644 --- a/packages/text/src/markdown/serializer.ts +++ b/packages/text-markdown/src/serializer.ts @@ -1,8 +1,23 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + import { MarkupMark, MarkupNode, MarkupNodeType } from '@hcengineering/text-core' -import { generateHTML } from '@tiptap/html' -import { defaultExtensions } from '../extensions' +import { markupToHtml } from '@hcengineering/text-html' + import { isInSet, markEq } from './marks' -import { messageContent, nodeAttrs } from './node' +import { nodeContent, nodeAttrs } from './node' type FirstDelim = (i: number, attrs?: Record, parentAttrs?: Record) => string interface IState { @@ -21,6 +36,7 @@ interface IState { quote: (str: string) => string repeat: (str: string, n: number) => string markString: (mark: MarkupMark, open: boolean, parent: MarkupNode, index: number) => string + renderHtml: (node: MarkupNode) => string refUrl: string imageUrl: string inAutolink?: boolean @@ -44,14 +60,14 @@ function backticksFor (side: boolean): string { } function isPlainURL (link: MarkupMark, parent: MarkupNode, index: number): boolean { - if (link.attrs.title !== undefined || !/^\w+:/.test(link.attrs.href)) return false + if (link.attrs?.title !== undefined || !/^\w+:/.test(link.attrs?.href)) return false const content = parent.content?.[index] if (content === undefined) { return false } if ( content.type !== MarkupNodeType.text || - content.text !== link.attrs.href || + content.text !== link.attrs?.href || content.marks?.[content.marks.length - 1] !== link ) { return false @@ -110,7 +126,7 @@ export const storeNodes: Record = { if (nodeAttrs(node).order !== undefined) { start = Number(nodeAttrs(node).order) } - const maxW = String(start + messageContent(node).length - 1).length + const maxW = String(start + nodeContent(node).length - 1).length const space = state.repeat(' ', maxW + 2) state.renderList(node, space, (i: number) => { const nStr = String(start + i) @@ -218,7 +234,7 @@ export const storeNodes: Record = { state.write('-->') }, hardBreak: (state, node, parent, index) => { - const content = messageContent(parent) + const content = nodeContent(parent) for (let i = index + 1; i < content.length; i++) { if (content[i].type !== node.type) { state.write('\\\n') @@ -231,8 +247,7 @@ export const storeNodes: Record = { state.text(node.text ?? '') }, table: (state, node) => { - const html = generateHTML(node, defaultExtensions) - state.write('' + html + '
    ') + state.write(state.renderHtml(node)) state.closeBlock(node) } } @@ -291,7 +306,7 @@ export const storeMarks: Record = { link: { open: (state, mark, parent, index) => { if (state.renderAHref === true) { - return `` + return `` } else { state.inAutolink = isPlainURL(mark, parent, index) return state.inAutolink ? '<' : '[' @@ -307,8 +322,8 @@ export const storeMarks: Record = { ? '>' : '](' + // eslint-disable-next-line - (mark.attrs.href as string).replace(/[\(\)"]/g, '\\$&') + - (mark.attrs.title !== undefined ? ` "${(mark.attrs.title as string).replace(/"/g, '\\"')}"` : '') + + (mark.attrs?.href as string).replace(/[\(\)"]/g, '\\$&') + + (mark.attrs?.title !== undefined ? ` "${(mark.attrs?.title as string).replace(/"/g, '\\"')}"` : '') + ')' } }, @@ -329,10 +344,13 @@ export const storeMarks: Record = { } } +export type HtmlWriter = (markup: MarkupNode) => string + export interface StateOptions { tightLists: boolean refUrl: string imageUrl: string + htmlWriter?: HtmlWriter } export class MarkdownState implements IState { nodes: Record @@ -345,6 +363,7 @@ export class MarkdownState implements IState { options: StateOptions refUrl: string imageUrl: string + htmlWriter: HtmlWriter constructor ( nodes = storeNodes, @@ -358,6 +377,7 @@ export class MarkdownState implements IState { this.inTightList = false this.refUrl = options.refUrl this.imageUrl = options.imageUrl + this.htmlWriter = options.htmlWriter ?? markupToHtml this.options = options } @@ -383,6 +403,10 @@ export class MarkdownState implements IState { } } + renderHtml (node: MarkupNode): string { + return this.htmlWriter(node) + } + wrapBlock (delim: string, firstDelim: string | null, node: MarkupNode, f: () => void): void { const old = this.delim this.write(firstDelim ?? delim) @@ -444,7 +468,7 @@ export class MarkdownState implements IState { // :: (Node) // Render the contents of `parent` as block nodes. renderContent (parent: MarkupNode): void { - messageContent(parent).forEach((node: MarkupNode, i: number) => { + nodeContent(parent).forEach((node: MarkupNode, i: number) => { this.render(node, parent, i) }) } @@ -656,7 +680,7 @@ export class MarkdownState implements IState { // Render the contents of `parent` as inline content. renderInline (parent: MarkupNode): void { const state: InlineState = { active: [], trailing: '', parent, marks: [] } - messageContent(parent).forEach((nde, index) => { + nodeContent(parent).forEach((nde, index) => { state.node = nde this.renderNodeInline(state, index) }) @@ -677,7 +701,7 @@ export class MarkdownState implements IState { const prevTight = this.inTightList this.inTightList = isTight - messageContent(node).forEach((child, i) => { + nodeContent(node).forEach((child, i) => { this.renderListItem(node, child, i, isTight, delim, firstDelim) }) this.inTightList = prevTight @@ -748,7 +772,7 @@ export class MarkdownState implements IState { } } -function makeQuery (obj: Record): string { +function makeQuery (obj: Record): string { return Object.keys(obj) .filter((it) => it[1] != null) .map(function (k) { diff --git a/packages/text-markdown/tsconfig.json b/packages/text-markdown/tsconfig.json new file mode 100644 index 0000000000..59e4fd4297 --- /dev/null +++ b/packages/text-markdown/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/packages/text-ydoc/package.json b/packages/text-ydoc/package.json index e05461b318..19ce811f1f 100644 --- a/packages/text-ydoc/package.json +++ b/packages/text-ydoc/package.json @@ -35,17 +35,14 @@ "jest": "^29.7.0", "ts-jest": "^29.1.1", "@types/jest": "^29.5.5", - "@types/markdown-it": "~13.0.0", - "jest-environment-jsdom": "^29.7.0" + "jest-environment-jsdom": "^29.7.0", + "fast-equals": "^5.2.2", + "y-prosemirror": "^1.2.15" }, "dependencies": { "@hcengineering/core": "^0.6.32", "@hcengineering/text": "^0.6.5", - "@tiptap/core": "^2.11.3", - "@tiptap/pm": "^2.11.3", - "fast-equals": "^5.2.2", - "yjs": "^13.6.23", - "y-prosemirror": "^1.2.15" + "yjs": "^13.6.23" }, "repository": "https://github.com/hcengineering/platform", "publishConfig": { diff --git a/packages/text-ydoc/src/__tests__/ydoc.test.ts b/packages/text-ydoc/src/__tests__/ydoc.test.ts index 132ccafb1d..3b6a99bd23 100644 --- a/packages/text-ydoc/src/__tests__/ydoc.test.ts +++ b/packages/text-ydoc/src/__tests__/ydoc.test.ts @@ -13,62 +13,125 @@ // limitations under the License. // -import { Markup } from '@hcengineering/core' -import { markupToJSON } from '@hcengineering/text' -import { jsonToYDocNoSchema, markupToYDoc, markupToYDocNoSchema, yDocToMarkup } from '../ydoc' +import { type Markup, generateId } from '@hcengineering/core' +import { type MarkupNode, jsonToMarkup, jsonToPmNode, markupToJSON } from '@hcengineering/text' +import { prosemirrorToYXmlFragment, yDocToProsemirrorJSON } from 'y-prosemirror' +import { deepEqual } from 'fast-equals' +import { Doc as YDoc } from 'yjs' +import { markupToYDoc, yDocToMarkup } from '../ydoc' -describe('ydoc', () => { - const markups: Markup[] = [ - // just text - '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"hello world"}]}]}', - // just text with bold mark - '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","marks":[{"type":"bold","attrs":{}}],"text":"hello world"}]}]}', - // separate paragraphs with bold mark - '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","marks":[{"type":"bold","attrs":{}}],"text":"hello"}]},{"type":"paragraph","content":[{"type":"text","marks":[{"type":"bold","attrs":{}}],"text":"world"}]}]}', - // mixed text and text with bold mark - '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"hello "},{"type":"text","marks":[{"type":"bold","attrs":{}}],"text":"world"}]}]}', - // mixed text with italic and text with bold and italic marks - '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"hello "},{"type":"text","marks":[{"type":"bold","attrs":{}},{"type":"italic","attrs":{}}],"text":"world"}]}]}', - // text with link and italic marks - '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"hello "},{"type":"text","text":"hello world","marks":[{"type":"link","attrs":{"href":"http://example.com","target":"_blank","rel":"noopener noreferrer","class":"cursor-pointer"}},{"type":"italic","attrs":{}}]}]}]}', - // an image - '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"image","attrs":{"src":"http://example.com/image.jpg","alt":"image"}}]}]}', - // a table with formatting inside +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace jest { + interface Matchers { + toEqualMarkup: (expected: string) => R + } + } +} + +expect.extend({ + toEqualMarkup (received: string, expected: string) { + const pass = received === expected || deepEqual(JSON.parse(received), JSON.parse(expected)) + return { + message: () => + pass + ? `Expected markup strings NOT to be equal:\n Received: ${received}\n Expected: ${expected}` + : `Expected markup strings to be equal:\n Received: ${received}\n Expected: ${expected}`, + pass + } + } +}) + +function referenceMarkupToYDoc (markup: Markup, field: string): YDoc { + const ydoc = new YDoc({ guid: generateId() }) + const fragment = ydoc.getXmlFragment(field) + prosemirrorToYXmlFragment(jsonToPmNode(markupToJSON(markup)), fragment) + return ydoc +} + +function referenceYDocToMarkup (ydoc: YDoc, field: string): Markup { + const json = yDocToProsemirrorJSON(ydoc, field) + return jsonToMarkup(json as MarkupNode) +} + +const markups: Array<[string, Markup]> = [ + ['text', '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"hello world"}]}]}'], + [ + 'text with bold mark', + '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","marks":[{"type":"bold","attrs":{}}],"text":"hello world"}]}]}' + ], + [ + 'separate paragraphs with bold mark', + '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","marks":[{"type":"bold","attrs":{}}],"text":"hello"}]},{"type":"paragraph","content":[{"type":"text","marks":[{"type":"bold","attrs":{}}],"text":"world"}]}]}' + ], + [ + 'mixed text and text with bold mark', + '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"hello "},{"type":"text","marks":[{"type":"bold","attrs":{}}],"text":"world"}]}]}' + ], + [ + 'mixed text with italic and text with bold and italic marks', + '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"hello "},{"type":"text","marks":[{"type":"bold","attrs":{}},{"type":"italic","attrs":{}}],"text":"world"}]}]}' + ], + [ + 'text with link and italic marks', + '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"hello "},{"type":"text","text":"hello world","marks":[{"type":"link","attrs":{"href":"http://example.com","target":"_blank","rel":"noopener noreferrer","class":"cursor-pointer"}},{"type":"italic","attrs":{}}]}]}]}' + ], + [ + 'image', + '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"image","attrs":{"src":"http://example.com/image.jpg","alt":"image"}}]}]}' + ], + [ + 'table with formatting inside', '{"type":"doc","content":[{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"type":"text","marks":[{"type":"bold","attrs":{}}],"text":"1"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"type":"text","marks":[{"type":"italic","attrs":{}}],"text":"2"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"codeBlock","content":[{"type":"text","text":"3"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"type":"text","text":"4"}]}]}]}]}]}' ] +] - describe.each(markups)('markupToYDoc', (markup) => { - it('converts markup to ydoc and back', () => { +describe('markupToYDoc', () => { + describe.each(markups)('compare with reference', (name, markup) => { + it(name, () => { + const expected = referenceMarkupToYDoc(markup, 'test') + const actual = markupToYDoc(markup, 'test') + expect(actual.toJSON()).toEqual(expected.toJSON()) + }) + }) + + describe.each(markups)('converts markup to ydoc and back', (name, markup) => { + it(name, () => { + console.log(markup, name) const ydoc = markupToYDoc(markup, 'test') - const back = yDocToMarkup(ydoc, 'test') + const actual = referenceYDocToMarkup(ydoc, 'test') - expect(JSON.parse(back)).toEqual(JSON.parse(markup)) - }) - }) - - describe.each(markups)('markupToYDocNoSchema', (markup) => { - it('converts markup to ydoc', () => { - const ydoc1 = markupToYDoc(markup, 'test') - const ydoc2 = markupToYDocNoSchema(markup, 'test') - - expect(ydoc2.getXmlFragment('test').toJSON()).toEqual(ydoc1.getXmlFragment('test').toJSON()) - }) - - it('converts markup to ydoc and back', () => { - const ydoc = markupToYDocNoSchema(markup, 'test') - const back = yDocToMarkup(ydoc, 'test') - - expect(JSON.parse(back)).toEqual(JSON.parse(markup)) - }) - }) - - describe.each(markups)('jsonToYDocNoSchema', (markup) => { - it('converts json to ydoc', () => { - const json = markupToJSON(markup) - const ydoc1 = jsonToYDocNoSchema(json, 'test') - const ydoc2 = markupToYDoc(markup, 'test') - - expect(ydoc1.getXmlFragment('test').toJSON()).toEqual(ydoc2.getXmlFragment('test').toJSON()) + expect(actual).toEqualMarkup(markup) + }) + }) +}) + +describe('yDocToMarkup', () => { + describe.each(markups)('compare with original', (name, markup) => { + it(name, () => { + const ydoc = referenceMarkupToYDoc(markup, 'test') + const actual = yDocToMarkup(ydoc, 'test') + + expect(actual).toEqualMarkup(markup) + }) + }) + + describe.each(markups)('compare with reference', (name, markup) => { + it(name, () => { + const ydoc = referenceMarkupToYDoc(markup, 'test') + const expected = referenceYDocToMarkup(ydoc, 'test') + const actual = yDocToMarkup(ydoc, 'test') + + expect(actual).toEqualMarkup(expected) + }) + }) + + describe.each(markups)('converts ydoc to markup and back', (name, markup) => { + it(name, () => { + const expected = referenceMarkupToYDoc(markup, 'test') + const actual = referenceMarkupToYDoc(yDocToMarkup(expected, 'test'), 'test') + + expect(actual.toJSON()).toEqual(expected.toJSON()) }) }) }) diff --git a/packages/text-ydoc/src/ydoc.ts b/packages/text-ydoc/src/ydoc.ts index a517662099..937ac86fc8 100644 --- a/packages/text-ydoc/src/ydoc.ts +++ b/packages/text-ydoc/src/ydoc.ts @@ -1,5 +1,5 @@ // -// Copyright © 2023 Hardcore Engineering Inc. +// Copyright © 2025 Hardcore Engineering Inc. // // Licensed under the Eclipse Public License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. You may @@ -14,63 +14,40 @@ // import { generateId, Markup } from '@hcengineering/core' -import { Extensions, getSchema } from '@tiptap/core' -import { Node, Schema } from '@tiptap/pm/model' -import { prosemirrorJSONToYDoc, prosemirrorToYDoc, yDocToProsemirrorJSON } from 'y-prosemirror' -import { - Doc as YDoc, - XmlElement as YXmlElement, - XmlFragment as YXmlFragment, - XmlText as YXmlText, - applyUpdate, - encodeStateAsUpdate -} from 'yjs' -import { defaultExtensions, jsonToMarkup, markupToJSON, markupToPmNode, type MarkupNode } from '@hcengineering/text' - -const defaultSchema = getSchema(defaultExtensions) +import { jsonToMarkup, MarkupMarkType, MarkupNodeType, markupToJSON, type MarkupNode } from '@hcengineering/text' +import { Doc as YDoc, XmlElement as YXmlElement, XmlFragment as YXmlFragment, XmlText as YXmlText } from 'yjs' /** - * @public - */ -export function markupToYDoc (markup: Markup, field: string, schema?: Schema, extensions?: Extensions): YDoc { - const node = markupToPmNode(markup, schema, extensions) - return prosemirrorToYDoc(node, field) -} - -/** - * Convert markup to Y.Doc without using ProseMirror schema + * Convert Markup to Y.Doc * * @public */ -export function markupToYDocNoSchema (markup: Markup, field: string): YDoc { - return jsonToYDocNoSchema(markupToJSON(markup), field) +export function markupToYDoc (markup: Markup, field: string): YDoc { + return jsonToYDoc(markupToJSON(markup), field) } /** - * Convert ProseMirror JSON to Y.Doc without using ProseMirror schema + * Convert Markup JSON to Y.Doc * * @public */ -export function jsonToYDocNoSchema (json: MarkupNode, field: string): YDoc { +export function jsonToYDoc (json: MarkupNode, field: string): YDoc { const ydoc = new YDoc({ guid: generateId() }) const fragment = ydoc.getXmlFragment(field) const nodes = json.type === 'doc' ? json.content ?? [] : [json] - nodes.map((p) => nodeToYXmlElement(fragment, p)) + nodes.map((p) => nodeToXmlElement(fragment, p)) return ydoc } -/** - * Convert ProseMirror JSON Node representation to YXmlElement - * */ -function nodeToYXmlElement (parent: YXmlFragment, node: MarkupNode): YXmlElement | YXmlText { +function nodeToXmlElement (parent: YXmlFragment, node: MarkupNode): YXmlElement | YXmlText { const elem = node.type === 'text' ? new YXmlText() : new YXmlElement(node.type) parent.push([elem]) if (elem instanceof YXmlElement) { if (node.content !== undefined && node.content.length > 0) { - node.content.map((p) => nodeToYXmlElement(elem, p)) + node.content.map((p) => nodeToXmlElement(elem, p)) } } else { // https://github.com/yjs/y-prosemirror/blob/master/src/plugins/sync-plugin.js#L777 @@ -98,73 +75,59 @@ function nodeToYXmlElement (parent: YXmlFragment, node: MarkupNode): YXmlElement } /** + * Convert Y.Doc to Markup + * * @public */ export function yDocToMarkup (ydoc: YDoc, field: string): Markup { - const json = yDocToProsemirrorJSON(ydoc, field) - return jsonToMarkup(json as MarkupNode) + const fragment = ydoc.getXmlFragment(field) + const json = xmlFragmentToNode(fragment) + return jsonToMarkup({ type: MarkupNodeType.doc, content: json }) } -/** - * Get ProseMirror nodes from Y.Doc content - * - * @public - */ -export function yDocContentToNodes (content: ArrayBuffer, schema?: Schema, extensions?: Extensions): Node[] { - schema ??= extensions === undefined ? defaultSchema : getSchema(extensions ?? defaultExtensions) +function xmlFragmentToNode (fragment: YXmlFragment): MarkupNode[] { + const result: MarkupNode[] = [] - const nodes: Node[] = [] + for (let i = 0; i < fragment.length; i++) { + const item = fragment.get(i) + if (item instanceof YXmlElement) { + const node: MarkupNode = { + type: item.nodeName as MarkupNodeType + } - try { - const ydoc = new YDoc({ - gc: false, - guid: generateId() - }) - const uint8arr = new Uint8Array(content) - applyUpdate(ydoc, uint8arr) + // Handle attributes + const attrs = item.getAttributes() + if (Object.keys(attrs).length > 0) { + node.attrs = attrs + } - for (const field of ydoc.share.keys()) { - try { - const body = yDocToProsemirrorJSON(ydoc, field) - nodes.push(schema.nodeFromJSON(body)) - } catch {} + // Handle content + if (item.length > 0) { + node.content = xmlFragmentToNode(item) + } + + result.push(node) + } else if (item instanceof YXmlText) { + // Handle text with marks + const delta = item.toDelta() + for (const op of delta) { + const textNode: MarkupNode = { + type: MarkupNodeType.text, + text: op.insert + } + + // Convert attributes to marks + if (op.attributes != null) { + textNode.marks = Object.entries(op.attributes).map(([type, attrs]) => ({ + type: type as MarkupMarkType, + attrs: attrs as Record + })) + } + + result.push(textNode) + } } - } catch (err: any) { - console.error(err) } - return nodes -} - -/** - * Update Y.Doc content - * - * @public - */ -export function updateYDocContent ( - content: ArrayBuffer, - updateFn: (body: Record) => Record, - schema?: Schema, - extensions?: Extensions -): YDoc | undefined { - schema ??= extensions === undefined ? defaultSchema : getSchema(extensions ?? defaultExtensions) - - try { - const ydoc = new YDoc({ guid: generateId(), gc: false }) - const res = new YDoc({ guid: generateId(), gc: false }) - const uint8arr = new Uint8Array(content) - applyUpdate(ydoc, uint8arr) - - for (const field of ydoc.share.keys()) { - const body = yDocToProsemirrorJSON(ydoc, field) - const updated = updateFn(body) - const yDoc = prosemirrorJSONToYDoc(schema, updated, field) - const update = encodeStateAsUpdate(yDoc) - applyUpdate(res, update) - } - - return res - } catch (err: any) { - console.error(err) - } + return result } diff --git a/packages/text/package.json b/packages/text/package.json index e99c899bae..186c6fc864 100644 --- a/packages/text/package.json +++ b/packages/text/package.json @@ -35,8 +35,7 @@ "jest": "^29.7.0", "ts-jest": "^29.1.1", "@types/jest": "^29.5.5", - "@types/markdown-it": "~13.0.0", - "jest-environment-jsdom": "^29.7.0" + "@types/markdown-it": "~13.0.0" }, "dependencies": { "@hcengineering/core": "^0.6.32", @@ -63,7 +62,6 @@ "@tiptap/extension-underline": "^2.11.3", "@tiptap/suggestion": "^2.11.3", "prosemirror-codemark": "^0.4.2", - "markdown-it": "^14.0.0", "fast-equals": "^5.2.2", "@tiptap/extension-text-align": "~2.11.0", "@tiptap/extension-text-style": "~2.11.0" diff --git a/packages/text/src/index.ts b/packages/text/src/index.ts index 00756c4f12..2a69c881d7 100644 --- a/packages/text/src/index.ts +++ b/packages/text/src/index.ts @@ -21,11 +21,6 @@ export * from './marks/code' export * from './marks/colors' export * from './marks/noteBase' export * from './marks/inlineComment' -export * from './markdown' -export * from './markdown/serializer' -export * from './markdown/parser' -export * from './markdown/compare' -export * from './markdown/node' export * from './kits/default-kit' export * from './kits/server-kit' diff --git a/packages/text/src/markdown/__tests__/compare.test.ts b/packages/text/src/markdown/__tests__/compare.test.ts deleted file mode 100644 index 80e8452e8f..0000000000 --- a/packages/text/src/markdown/__tests__/compare.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { parseMessageMarkdown, serializeMessage } from '..' -import { ServerKit } from '../../kits/server-kit' -import { isMarkdownsEquals } from '../compare' - -const refUrl: string = 'ref://' -const imageUrl: string = 'http://localhost' - -const extensions = [ServerKit] - -const tests: Array<{ name: string, body: string }> = [ - { name: 'Italic', body: '*Asteriscs* and _Underscores_' }, - { name: 'Bold', body: '**Asteriscs** and __Underscores__' }, - { name: 'Bullet list with asteriscs', body: 'Asterisks :\r\n* Firstly\r\n* Secondly' }, - { name: 'Bullet list with dashes', body: 'Dashes :\r\n- Firstly\r\n- Secondly' }, - { name: 'TODO list with asteriscs', body: '* [ ] Take\n* [ ] Do\n\n' }, - { name: 'TODO list with dashes', body: '- [x] Take\n- [ ] Do\n\n' }, - { - name: 'Different markers', - body: 'Asterisks bulleted list:\r\n* Asterisks: *Italic* and **Bold*** Underscores: _Italic_ and __Bold__\r\n\r\nDash bulleted list:\r\n- Asterisks: *Italic* and **Bold**\r\n- Underscores: _Italic_ and __Bold__\r\n-' - }, - { name: 'Single line comment', body: '' }, - { - name: 'Real with multiline comment', - body: '"\r\n\r\nThis PR fix issue #22424\r\n\r\n\r\n"' - } -] - -describe('md compatibility', () => { - tests.forEach(({ name, body: description }) => { - it(name, () => { - const json = parseMessageMarkdown(description, refUrl, imageUrl, extensions) - const serialized = serializeMessage(json, refUrl, imageUrl) - expect(isMarkdownsEquals(description, serialized)).toBe(true) - }) - }) -}) diff --git a/packages/text/src/markdown/__tests__/markdown.test.ts b/packages/text/src/markdown/__tests__/markdown.test.ts deleted file mode 100644 index 247cdd51f2..0000000000 --- a/packages/text/src/markdown/__tests__/markdown.test.ts +++ /dev/null @@ -1,742 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { ServerKit } from '../../kits/server-kit' -import { parseMessageMarkdown } from '..' - -const refUrl: string = 'ref://' -const imageUrl: string = 'http://localhost' - -const extensions = [ServerKit] - -const tests: Array<{ name: string, markdown: string, markup: object }> = [ - { - name: 'simple text', - markdown: 'Lorem ipsum dolor sit amet.', - markup: { - type: 'doc', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'Lorem ipsum dolor sit amet.', - marks: [] - } - ] - } - ] - } - }, - { - name: 'text with heading', - markdown: `# Lorem ipsum - -Lorem ipsum dolor sit amet. -`, - markup: { - type: 'doc', - content: [ - { - type: 'heading', - attrs: { level: 1 }, - content: [ - { - type: 'text', - text: 'Lorem ipsum', - marks: [] - } - ] - }, - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'Lorem ipsum dolor sit amet.', - marks: [] - } - ] - } - ] - } - }, - { - name: 'bullet list', - markdown: `# bullet list -- list item 1 -- list item 2 -`, - markup: { - type: 'doc', - content: [ - { - type: 'heading', - attrs: { level: 1 }, - content: [ - { - type: 'text', - text: 'bullet list', - marks: [] - } - ] - }, - { - type: 'bulletList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'listItem', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'list item 1', - marks: [] - } - ] - } - ] - }, - { - type: 'listItem', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'list item 2', - marks: [] - } - ] - } - ] - } - ] - } - ] - } - }, - { - name: 'todos', - markdown: `# TODO -- [ ] todo 1 -- [x] todo 2 -`, - markup: { - type: 'doc', - content: [ - { - type: 'heading', - attrs: { level: 1 }, - content: [ - { - type: 'text', - text: 'TODO', - marks: [] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: false }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'todo 1', - marks: [] - } - ] - } - ] - }, - { - type: 'todoItem', - attrs: { checked: true }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'todo 2', - marks: [] - } - ] - } - ] - } - ] - } - ] - } - }, - { - name: 'todos followed by list items', - markdown: `# todo and list -- [ ] todo 1 -- [x] todo 2 -- list item 1 -- list item 2 -`, - markup: { - type: 'doc', - content: [ - { - type: 'heading', - attrs: { level: 1 }, - content: [ - { - type: 'text', - text: 'todo and list', - marks: [] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: false }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'todo 1', - marks: [] - } - ] - } - ] - }, - { - type: 'todoItem', - attrs: { checked: true }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'todo 2', - marks: [] - } - ] - } - ] - } - ] - }, - { - type: 'bulletList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'listItem', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'list item 1', - marks: [] - } - ] - } - ] - }, - { - type: 'listItem', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'list item 2', - marks: [] - } - ] - } - ] - } - ] - } - ] - } - }, - { - name: 'todos followed by list items', - markdown: `# mixed lists -- [ ] todo 1 -- list item 1 -- [x] todo 2 -- list item 2 -`, - markup: { - type: 'doc', - content: [ - { - type: 'heading', - attrs: { level: 1 }, - content: [ - { - type: 'text', - text: 'mixed lists', - marks: [] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: false }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'todo 1', - marks: [] - } - ] - } - ] - } - ] - }, - { - type: 'bulletList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'listItem', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'list item 1', - marks: [] - } - ] - } - ] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: true }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'todo 2', - marks: [] - } - ] - } - ] - } - ] - }, - { - type: 'bulletList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'listItem', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'list item 2', - marks: [] - } - ] - } - ] - } - ] - } - ] - } - }, - { - name: 'nested todos', - markdown: `# nested todos -- [ ] todo - - [x] sub todo -`, - markup: { - type: 'doc', - content: [ - { - type: 'heading', - attrs: { level: 1 }, - content: [ - { - type: 'text', - text: 'nested todos', - marks: [] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: false }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'todo', - marks: [] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: true }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'sub todo', - marks: [] - } - ] - } - ] - } - ] - } - ] - } - ] - } - ] - } - }, - { - name: 'nested lists', - markdown: `# nested lists -- [ ] todo - - sub list item - - [x] sub todo -- list item - - [x] sub todo - - sub list item -`, - markup: { - type: 'doc', - content: [ - { - type: 'heading', - attrs: { level: 1 }, - content: [ - { - type: 'text', - text: 'nested lists', - marks: [] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: false }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'todo', - marks: [] - } - ] - }, - { - type: 'bulletList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'listItem', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'sub list item', - marks: [] - } - ] - } - ] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: true }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'sub todo', - marks: [] - } - ] - } - ] - } - ] - } - ] - } - ] - }, - { - type: 'bulletList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'listItem', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'list item', - marks: [] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: true }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'sub todo', - marks: [] - } - ] - } - ] - } - ] - }, - { - type: 'bulletList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'listItem', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'sub list item', - marks: [] - } - ] - } - ] - } - ] - } - ] - } - ] - } - ] - } - }, - { - name: 'nested todos', - markdown: `# nested todos -- [ ] todo - - [x] sub todo -`, - markup: { - type: 'doc', - content: [ - { - type: 'heading', - attrs: { level: 1 }, - content: [ - { - type: 'text', - text: 'nested todos', - marks: [] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: false }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'todo', - marks: [] - } - ] - }, - { - type: 'todoList', - attrs: { - bullet: '-' - }, - content: [ - { - type: 'todoItem', - attrs: { checked: true }, - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: 'sub todo', - marks: [] - } - ] - } - ] - } - ] - } - ] - } - ] - } - ] - } - } -] - -describe('markdown', () => { - tests.forEach(({ name, markdown, markup }) => { - it(name, () => { - const parsed = parseMessageMarkdown(markdown, refUrl, imageUrl, extensions) - expect(parsed).toEqual(markup) - }) - }) -}) diff --git a/packages/text/src/markdown/index.ts b/packages/text/src/markdown/index.ts deleted file mode 100644 index 37fcca8bbd..0000000000 --- a/packages/text/src/markdown/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { MarkupNode, markupToJSON } from '@hcengineering/text-core' -import { Extensions } from '@tiptap/core' -import { defaultExtensions } from '../extensions' -import { MarkdownParser } from './parser' -import { MarkdownState, storeMarks, storeNodes } from './serializer' - -/** - * @public - */ -export function parseMessageMarkdown ( - message: string | undefined, - imageUrl: string, - refUrl: string = 'ref://', - extensions: Extensions = defaultExtensions -): MarkupNode { - const parser = new MarkdownParser(extensions, refUrl, imageUrl) - return parser.parse(message ?? '') -} - -/** - * @public - */ -export function serializeMessage (node: MarkupNode, refUrl: string, imageUrl: string): string { - const state = new MarkdownState(storeNodes, storeMarks, { tightLists: true, refUrl, imageUrl }) - state.renderContent(node) - return state.out -} - -/** - * @public - */ -export async function markupToMarkdown ( - markup: string, - refUrl: string = 'ref://', - imageUrl: string = 'http://localhost', - preprocessor?: (nodes: MarkupNode) => Promise -): Promise { - const json = markupToJSON(markup) - await preprocessor?.(json) - return serializeMessage(json, refUrl, imageUrl) -} diff --git a/packages/text/src/markdown/node.ts b/packages/text/src/markdown/node.ts deleted file mode 100644 index 3efe06c967..0000000000 --- a/packages/text/src/markdown/node.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Attrs, MarkupNode } from '@hcengineering/text-core' - -export function traverseMarkupNode (node: MarkupNode, f: (el: MarkupNode) => void): void { - f(node) - node.content?.forEach((c) => { - traverseMarkupNode(c, f) - }) -} - -export function messageContent (node: MarkupNode): MarkupNode[] { - return node?.content ?? [] -} - -export function nodeAttrs (node: MarkupNode): Attrs { - return node.attrs ?? {} -} diff --git a/packages/text/src/markup/__tests__/utils.test.ts b/packages/text/src/markup/__tests__/utils.test.ts index e115620b39..2bff4dc275 100644 --- a/packages/text/src/markup/__tests__/utils.test.ts +++ b/packages/text/src/markup/__tests__/utils.test.ts @@ -28,23 +28,11 @@ import { markupToJSON, nodeDoc, nodeParagraph, - nodeText, - markupToHTML + nodeText } from '@hcengineering/text-core' import { Editor, getSchema } from '@tiptap/core' import { ServerKit } from '../../kits/server-kit' -import { - getMarkup, - htmlToJSON, - htmlToMarkup, - htmlToPmNode, - jsonToHTML, - jsonToText, - markupToPmNode, - pmNodeToHTML, - pmNodeToJSON, - pmNodeToMarkup -} from '../utils' +import { getMarkup, htmlToJSON, htmlToMarkup, jsonToHTML, jsonToPmNode, jsonToText, pmNodeToJSON } from '../utils' // mock tiptap functions jest.mock('@tiptap/html', () => ({ @@ -253,29 +241,6 @@ describe('isEmptyNode', () => { }) }) -describe('pmNodeToMarkup', () => { - it('converts ProseMirrorNode to Markup', () => { - const schema = getSchema(extensions) - const node = schema.node('paragraph', {}, [schema.text('Hello, world!')]) - - expect(pmNodeToMarkup(node)).toEqual( - '{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Hello, world!"}]}' - ) - }) -}) - -describe('markupToPmNode', () => { - it('converts markup to ProseMirrorNode', () => { - const markup = '{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Hello, world!"}]}' - const node = markupToPmNode(markup) - - expect(node.type.name).toEqual('paragraph') - expect(node.content.childCount).toEqual(1) - expect(node.content.child(0).type.name).toEqual('text') - expect(node.content.child(0).text).toEqual('Hello, world!') - }) -}) - describe('markupToJSON', () => { it('with empty content', async () => { expect(markupToJSON('')).toEqual({ type: 'doc', content: [{ type: 'paragraph', content: [] }] }) @@ -328,7 +293,7 @@ describe('pmNodeToJSON', () => { describe('jsonToPmNode', () => { it('converts json to ProseMirrorNode', () => { const markup = '{"type":"paragraph","content":[{"type":"text","text":"Hello, world!"}]}' - const node = markupToPmNode(markup) + const node = jsonToPmNode(markupToJSON(markup)) expect(node.type.name).toEqual('paragraph') expect(node.content.childCount).toEqual(1) @@ -345,14 +310,6 @@ describe('htmlToMarkup', () => { }) }) -describe('markupToHTML', () => { - it('converts markup to HTML', () => { - const markup = '{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"hello"}]}]}' - const expectedHtml = '

    hello

    ' - expect(markupToHTML(markup)).toEqual(expectedHtml) - }) -}) - describe('htmlToJSON', () => { it('converts HTML to JSON', () => { const html = '

    hello

    ' @@ -369,28 +326,6 @@ describe('jsonToHTML', () => { }) }) -describe('pmNodeToHTML', () => { - it('converts ProseMirrorNode to HTML', () => { - const schema = getSchema(extensions) - const node = schema.node('paragraph', {}, [schema.text('hello')]) - - expect(pmNodeToHTML(node)).toEqual('

    hello

    ') - }) -}) - -describe('htmlToPmNode', () => { - it('converts html to ProseMirrorNode', () => { - const node = htmlToPmNode('

    hello

    ') - - expect(node.type.name).toEqual('doc') - expect(node.content.childCount).toEqual(1) - expect(node.content.child(0).type.name).toEqual('paragraph') - expect(node.content.child(0).childCount).toEqual(1) - expect(node.content.child(0).child(0).type.name).toEqual('text') - expect(node.content.child(0).child(0).text).toEqual('hello') - }) -}) - describe('jsonToText', () => { it('returns text for text node', () => { const node: MarkupNode = { diff --git a/packages/text/src/markup/utils.ts b/packages/text/src/markup/utils.ts index e8e93cb773..0746171454 100644 --- a/packages/text/src/markup/utils.ts +++ b/packages/text/src/markup/utils.ts @@ -18,15 +18,7 @@ import { Editor, Extensions, getSchema } from '@tiptap/core' import { generateHTML, generateJSON } from '@tiptap/html' import { Node as ProseMirrorNode, Schema } from '@tiptap/pm/model' -import { - MarkupNode, - emptyMarkupNode, - jsonToMarkup, - markupToJSON, - nodeDoc, - nodeParagraph, - nodeText -} from '@hcengineering/text-core' +import { MarkupNode, jsonToMarkup } from '@hcengineering/text-core' import { defaultExtensions } from '../extensions' /** @public */ @@ -39,39 +31,6 @@ export function getMarkup (editor?: Editor): Markup { // Markup -/** @public */ -export function pmNodeToMarkup (node: ProseMirrorNode): Markup { - return jsonToMarkup(pmNodeToJSON(node)) -} - -/** @public */ -export function markupToPmNode (markup: Markup, schema?: Schema, extensions?: Extensions): ProseMirrorNode { - const json = markupToJSON(markup) - return jsonToPmNode(json, schema, extensions) -} - -/** @public */ -export function markupHtmlToJSON (markup: Markup): MarkupNode { - if (markup == null || markup === '') { - return emptyMarkupNode() - } - - try { - // Ideally Markup should contain only serialized JSON - // But there seem to be some cases when it contains HTML or plain text - // So we need to handle those cases and produce valid MarkupNode - if (markup.startsWith('{')) { - return JSON.parse(markup) as MarkupNode - } else if (markup.startsWith('<')) { - return htmlToJSON(markup, defaultExtensions) - } else { - return nodeDoc(nodeParagraph(nodeText(markup))) - } - } catch (error) { - return emptyMarkupNode() - } -} - /** @public */ export function jsonToPmNode (json: MarkupNode, schema?: Schema, extensions?: Extensions): ProseMirrorNode { schema ??= extensions == null ? defaultSchema : getSchema(extensions ?? defaultExtensions) @@ -89,11 +48,6 @@ export function jsonToText (node: MarkupNode, schema?: Schema, extensions?: Exte return pmNode.textBetween(0, pmNode.content.size, '\n', '') } -/** @public */ -export function pmNodeToText (node: ProseMirrorNode): string { - return jsonToText(node.toJSON()) -} - // export function markupToText (markup: Markup, schema?: Schema, extensions?: Extensions): string { // const pmNode = markupToPmNode(markup, schema, extensions) // return pmNode.textBetween(0, pmNode.content.size, '\n', '') @@ -124,16 +78,3 @@ export function jsonToHTML (json: MarkupNode, extensions?: Extensions): string { extensions = extensions ?? defaultExtensions return generateHTML(json, extensions) } - -/** @public */ -export function htmlToPmNode (html: string, schema?: Schema, extensions?: Extensions): ProseMirrorNode { - schema ??= extensions == null ? defaultSchema : getSchema(extensions ?? defaultExtensions) - const json = htmlToJSON(html, extensions) - return ProseMirrorNode.fromJSON(schema, json) -} - -/** @public */ -export function pmNodeToHTML (node: ProseMirrorNode, extensions?: Extensions): string { - extensions ??= defaultExtensions - return generateHTML(node.toJSON(), extensions) -} diff --git a/plugins/gmail-resources/package.json b/plugins/gmail-resources/package.json index fc0360f89e..71e647fb1c 100644 --- a/plugins/gmail-resources/package.json +++ b/plugins/gmail-resources/package.json @@ -44,6 +44,7 @@ "@hcengineering/ui": "^0.6.15", "@hcengineering/presentation": "^0.6.3", "@hcengineering/text": "^0.6.5", + "@hcengineering/text-html": "^0.6.0", "@hcengineering/text-editor": "^0.6.0", "@hcengineering/text-editor-resources": "^0.6.0", "@hcengineering/contact": "^0.6.24", diff --git a/plugins/gmail-resources/src/components/NewMessage.svelte b/plugins/gmail-resources/src/components/NewMessage.svelte index c5eb969546..3ca0d6d7f2 100644 --- a/plugins/gmail-resources/src/components/NewMessage.svelte +++ b/plugins/gmail-resources/src/components/NewMessage.svelte @@ -24,7 +24,8 @@ import { createQuery, getClient } from '@hcengineering/presentation' import { Integration } from '@hcengineering/setting' import templates, { TemplateDataProvider } from '@hcengineering/templates' - import { EmptyMarkup, isEmptyMarkup, markupToHTML } from '@hcengineering/text' + import { EmptyMarkup, isEmptyMarkup, markupToJSON } from '@hcengineering/text' + import { markupToHtml } from '@hcengineering/text-html' import { StyledTextEditor } from '@hcengineering/text-editor-resources' import { Button, EditBox, IconArrowLeft, IconAttachment, Label, Scroller } from '@hcengineering/ui' import { createEventDispatcher, onDestroy } from 'svelte' @@ -71,7 +72,7 @@ core.space.Workspace, { ...obj, - content: markupToHTML(content), + content: markupToHtml(markupToJSON(content)), attachments: attachments.length, from: selectedIntegration.createdBy, copy: copy diff --git a/plugins/gmail-resources/src/components/NewMessages.svelte b/plugins/gmail-resources/src/components/NewMessages.svelte index df6eff3a92..07bcd034a0 100644 --- a/plugins/gmail-resources/src/components/NewMessages.svelte +++ b/plugins/gmail-resources/src/components/NewMessages.svelte @@ -24,7 +24,8 @@ import setting, { Integration } from '@hcengineering/setting' import templates, { TemplateDataProvider } from '@hcengineering/templates' import { StyledTextEditor } from '@hcengineering/text-editor-resources' - import { EmptyMarkup, isEmptyMarkup, markupToHTML } from '@hcengineering/text' + import { EmptyMarkup, isEmptyMarkup, markupToJSON } from '@hcengineering/text' + import { markupToHtml } from '@hcengineering/text-html' import { Button, EditBox, @@ -83,7 +84,7 @@ const target = contacts.find((p) => p._id === channel.attachedTo) if (target === undefined) continue templateProvider.set(contact.class.Contact, target) - const htmlContent = markupToHTML(content) + const htmlContent = markupToHtml(markupToJSON(content)) const message = await templateProvider.fillTemplate(htmlContent) const id = await client.createDoc(plugin.class.NewMessage, core.space.Workspace, { subject, diff --git a/plugins/telegram-resources/src/components/Chat.svelte b/plugins/telegram-resources/src/components/Chat.svelte index 37c12481e5..03e9d5305d 100644 --- a/plugins/telegram-resources/src/components/Chat.svelte +++ b/plugins/telegram-resources/src/components/Chat.svelte @@ -26,7 +26,7 @@ import setting, { Integration } from '@hcengineering/setting' import type { NewTelegramMessage, SharedTelegramMessage, TelegramMessage } from '@hcengineering/telegram' import templates, { TemplateDataProvider } from '@hcengineering/templates' - import { markupToHTML } from '@hcengineering/text' + import { jsonToHTML, markupToJSON } from '@hcengineering/text' import { Button, Icon, @@ -122,7 +122,7 @@ channel._class, 'newMessages', { - content: markupToHTML(message), + content: jsonToHTML(markupToJSON(message)), status: 'new', attachments }, diff --git a/rush.json b/rush.json index f366776de0..3264937b74 100644 --- a/rush.json +++ b/rush.json @@ -485,6 +485,16 @@ "projectFolder": "packages/text-core", "shouldPublish": true }, + { + "packageName": "@hcengineering/text-markdown", + "projectFolder": "packages/text-markdown", + "shouldPublish": true + }, + { + "packageName": "@hcengineering/text-html", + "projectFolder": "packages/text-html", + "shouldPublish": true + }, { "packageName": "@hcengineering/text", "projectFolder": "packages/text", diff --git a/server-plugins/chunter-resources/package.json b/server-plugins/chunter-resources/package.json index a46cc8aba4..079dd0819e 100644 --- a/server-plugins/chunter-resources/package.json +++ b/server-plugins/chunter-resources/package.json @@ -51,6 +51,7 @@ "@hcengineering/server-notification": "^0.6.1", "@hcengineering/server-notification-resources": "^0.6.0", "@hcengineering/server": "^0.6.4", - "@hcengineering/text-core": "^0.6.0" + "@hcengineering/text-core": "^0.6.0", + "@hcengineering/text": "^0.6.5" } } diff --git a/server-plugins/chunter-resources/src/index.ts b/server-plugins/chunter-resources/src/index.ts index 7045711166..d7b7e96c02 100644 --- a/server-plugins/chunter-resources/src/index.ts +++ b/server-plugins/chunter-resources/src/index.ts @@ -45,7 +45,8 @@ import { getDocCollaborators, getMixinTx } from '@hcengineering/server-notification-resources' -import { markupToHTML, markupToText, stripTags } from '@hcengineering/text-core' +import { markupToText, stripTags } from '@hcengineering/text-core' +import { jsonToHTML, markupToJSON } from '@hcengineering/text' import { workbenchId } from '@hcengineering/workbench' import { NOTIFICATION_BODY_SIZE } from '@hcengineering/server-notification' @@ -84,7 +85,7 @@ export async function ChatMessageTextPresenter (doc: ChatMessage): Promise { - return markupToHTML(doc.message) + return jsonToHTML(markupToJSON(doc.message)) } /** diff --git a/server-plugins/telegram-resources/package.json b/server-plugins/telegram-resources/package.json index bae15acf6c..8bbf4d734d 100644 --- a/server-plugins/telegram-resources/package.json +++ b/server-plugins/telegram-resources/package.json @@ -50,6 +50,6 @@ "@hcengineering/server-token": "^0.6.11", "@hcengineering/setting": "^0.6.17", "@hcengineering/telegram": "^0.6.21", - "@hcengineering/text-core": "^0.6.0" + "@hcengineering/text": "^0.6.5" } } diff --git a/server-plugins/telegram-resources/src/index.ts b/server-plugins/telegram-resources/src/index.ts index 2d0bcd1450..be05508301 100644 --- a/server-plugins/telegram-resources/src/index.ts +++ b/server-plugins/telegram-resources/src/index.ts @@ -48,7 +48,7 @@ import serverTelegram from '@hcengineering/server-telegram' import { generateToken } from '@hcengineering/server-token' import setting, { Integration } from '@hcengineering/setting' import telegram, { TelegramMessage, TelegramNotificationRequest } from '@hcengineering/telegram' -import { markupToHTML } from '@hcengineering/text-core' +import { jsonToHTML, markupToJSON } from '@hcengineering/text' /** * @public @@ -168,7 +168,7 @@ async function activityMessageToHtml (control: TriggerControl, message: Activity const { hierarchy } = control if (hierarchy.isDerived(message._class, chunter.class.ChatMessage)) { const chatMessage = message as ChatMessage - return markupToHTML(chatMessage.message) + return jsonToHTML(markupToJSON(chatMessage.message)) } else { const resource = getTextPresenter(message._class, control.hierarchy) @@ -176,7 +176,7 @@ async function activityMessageToHtml (control: TriggerControl, message: Activity const fn = await getResource(resource.presenter) const textData = await fn(message, control) if (textData !== undefined && textData !== '') { - return markupToHTML(textData) + return jsonToHTML(markupToJSON(textData)) } } } @@ -210,9 +210,9 @@ async function getTranslatedData ( if (hierarchy.isDerived(data._class, notification.class.MentionInboxNotification)) { const text = (data as MentionInboxNotification).messageHtml - body = text !== undefined ? markupToHTML(text) : body + body = text !== undefined ? jsonToHTML(markupToJSON(text)) : body } else if (data.data !== undefined) { - body = markupToHTML(data.data) + body = jsonToHTML(markupToJSON(data.data)) } else if (message !== undefined) { const html = await activityMessageToHtml(control, message) if (html !== undefined) { diff --git a/server/collaborator/package.json b/server/collaborator/package.json index 354978b400..fe70d150ac 100644 --- a/server/collaborator/package.json +++ b/server/collaborator/package.json @@ -70,7 +70,6 @@ "@tiptap/html": "^2.11.3", "mongodb": "^6.12.0", "yjs": "^13.6.23", - "y-prosemirror": "^1.2.15", "express": "^4.21.2", "body-parser": "^1.20.2", "cors": "^2.8.5", diff --git a/server/collaborator/src/storage/platform.ts b/server/collaborator/src/storage/platform.ts index e6ab07da05..2b90d59bd9 100644 --- a/server/collaborator/src/storage/platform.ts +++ b/server/collaborator/src/storage/platform.ts @@ -20,7 +20,7 @@ import { decodeDocumentId } from '@hcengineering/collaborator-client' import core, { AttachedData, MeasureContext, Ref, Space, TxOperations } from '@hcengineering/core' import { StorageAdapter } from '@hcengineering/server-core' import { areEqualMarkups } from '@hcengineering/text' -import { markupToYDocNoSchema } from '@hcengineering/text-ydoc' +import { markupToYDoc } from '@hcengineering/text-ydoc' import { Doc as YDoc } from 'yjs' import { Context } from '../context' @@ -63,7 +63,7 @@ export class PlatformStorageAdapter implements CollabStorageAdapter { }) }) if (markup !== undefined) { - const ydoc = markupToYDocNoSchema(markup, documentId.objectAttr) + const ydoc = markupToYDoc(markup, documentId.objectAttr) // if document was loaded from the initial content or storage we need to save // it to ensure the next time we load it from the ydoc document diff --git a/server/collaborator/src/transformers/markup.ts b/server/collaborator/src/transformers/markup.ts index 3f311858dc..262e9c5ff5 100644 --- a/server/collaborator/src/transformers/markup.ts +++ b/server/collaborator/src/transformers/markup.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { markupToYDocNoSchema, yDocToMarkup } from '@hcengineering/text-ydoc' +import { markupToYDoc, yDocToMarkup } from '@hcengineering/text-ydoc' import { Transformer } from '@hocuspocus/transformer' import { Doc } from 'yjs' @@ -37,7 +37,7 @@ export class MarkupTransformer implements Transformer { toYdoc (document: any, fieldName: string): Doc { if (typeof document === 'string' && document !== '') { - return markupToYDocNoSchema(document, fieldName) + return markupToYDoc(document, fieldName) } return new Doc() diff --git a/server/tool/package.json b/server/tool/package.json index fb34871673..633e35cd0f 100644 --- a/server/tool/package.json +++ b/server/tool/package.json @@ -61,6 +61,7 @@ "@hcengineering/minio": "^0.6.0", "fast-equals": "^5.2.2", "@hcengineering/text": "^0.6.5", + "@hcengineering/text-markdown": "^0.6.0", "js-yaml": "^4.1.0" } } diff --git a/server/tool/src/initializer.ts b/server/tool/src/initializer.ts index e4e2308b8d..5766a9d56b 100644 --- a/server/tool/src/initializer.ts +++ b/server/tool/src/initializer.ts @@ -21,7 +21,8 @@ import { ModelLogger } from '@hcengineering/model' import { makeRank } from '@hcengineering/rank' import { StorageFileUploader, UnifiedFormatImporter } from '@hcengineering/importer' import type { StorageAdapter } from '@hcengineering/server-core' -import { jsonToMarkup, parseMessageMarkdown } from '@hcengineering/text' +import { jsonToMarkup } from '@hcengineering/text' +import { markdownToMarkup } from '@hcengineering/text-markdown' import { pickPrimarySocialId } from '@hcengineering/contact' import { v4 as uuid } from 'uuid' import path from 'path' @@ -256,7 +257,7 @@ export class WorkspaceInitializer { } private parseMarkdown (text: string): string { - const json = parseMessageMarkdown(text ?? '', this.imageUrl) + const json = markdownToMarkup(text ?? '', { imageUrl: this.imageUrl }) return JSON.stringify(json) } @@ -320,7 +321,7 @@ export class WorkspaceInitializer { ): Promise { const doc = makeCollabId(objectClass, objectId, objectAttr) - const json = parseMessageMarkdown(data ?? '', this.imageUrl) + const json = markdownToMarkup(data ?? '', { imageUrl: this.imageUrl }) const markup = jsonToMarkup(json) return await saveCollabJson(this.ctx, this.storageAdapter, this.wsIds, doc, markup) diff --git a/services/ai-bot/pod-ai-bot/package.json b/services/ai-bot/pod-ai-bot/package.json index 9582019ded..82ff8389ec 100644 --- a/services/ai-bot/pod-ai-bot/package.json +++ b/services/ai-bot/pod-ai-bot/package.json @@ -77,6 +77,8 @@ "@hcengineering/server-token": "^0.6.11", "@hcengineering/setting": "^0.6.17", "@hcengineering/text": "^0.6.5", + "@hcengineering/text-html": "^0.6.0", + "@hcengineering/text-markdown": "^0.6.0", "@hcengineering/workbench": "^0.6.16", "cors": "^2.8.5", "dotenv": "~16.0.0", diff --git a/services/ai-bot/pod-ai-bot/src/controller.ts b/services/ai-bot/pod-ai-bot/src/controller.ts index 85af08b5e1..fe080f2be8 100644 --- a/services/ai-bot/pod-ai-bot/src/controller.ts +++ b/services/ai-bot/pod-ai-bot/src/controller.ts @@ -27,7 +27,7 @@ import { Room } from '@hcengineering/love' import { WorkspaceInfoRecord } from '@hcengineering/server-ai-bot' import { getAccountClient } from '@hcengineering/server-client' import { generateToken } from '@hcengineering/server-token' -import { htmlToMarkup, markupToHTML } from '@hcengineering/text' +import { htmlToMarkup, markupToJSON, jsonToHTML } from '@hcengineering/text' import { isWorkspaceLoginInfo } from '@hcengineering/account-client' import { encodingForModel } from 'js-tiktoken' import OpenAI from 'openai' @@ -195,7 +195,7 @@ export class AIControl { if (this.openai === undefined) { return undefined } - const html = markupToHTML(req.text) + const html = jsonToHTML(markupToJSON(req.text)) const result = await translateHtml(this.openai, html, req.lang) const text = result !== undefined ? htmlToMarkup(result) : req.text return { diff --git a/services/ai-bot/pod-ai-bot/src/utils/tools.ts b/services/ai-bot/pod-ai-bot/src/utils/tools.ts index 9dd889b50e..e003108a84 100644 --- a/services/ai-bot/pod-ai-bot/src/utils/tools.ts +++ b/services/ai-bot/pod-ai-bot/src/utils/tools.ts @@ -1,7 +1,7 @@ import { MarkupBlobRef, PersonId, Ref } from '@hcengineering/core' import document, { Document, getFirstRank, Teamspace } from '@hcengineering/document' import { makeRank } from '@hcengineering/rank' -import { parseMessageMarkdown } from '@hcengineering/text' +import { markdownToMarkup } from '@hcengineering/text-markdown' import { BaseFunctionsArgs, RunnableFunctionWithoutParse, @@ -91,7 +91,7 @@ async function saveFile ( if (content === undefined) { return 'Error while converting pdf to markdown' } - const converted = JSON.stringify(parseMessageMarkdown(content, 'image://')) + const converted = JSON.stringify(markdownToMarkup(content)) const client = await workspaceClient.opClient const fileId = uuid() diff --git a/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts b/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts index 5d42e2f9a5..cd4e59764a 100644 --- a/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts +++ b/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts @@ -63,7 +63,8 @@ import { createChatCompletionWithTools, requestSummary } from '../utils/openai' import { connectPlatform } from '../utils/platform' import { LoveController } from './love' import { DbStorage } from '../storage' -import { jsonToMarkup, MarkdownParser, markupToText } from '@hcengineering/text' +import { jsonToMarkup, markupToText } from '@hcengineering/text' +import { markdownToMarkup } from '@hcengineering/text-markdown' import { countTokens } from '@hcengineering/openai' import { getAccountClient } from '@hcengineering/server-client' import { getGlobalPerson } from '../utils/account' @@ -347,8 +348,7 @@ export class WorkspaceClient { void this.pushHistory(response, 'assistant', responseTokens, personUuid, objectId, objectClass) - const parser = new MarkdownParser([], '', '') - const parseResponse = jsonToMarkup(parser.parse(response)) + const parseResponse = jsonToMarkup(markdownToMarkup(response, { refUrl: '', imageUrl: '' })) if (messageClass === chunter.class.ChatMessage) { await op.addCollection( diff --git a/services/calendar/pod-calendar/src/calendar.ts b/services/calendar/pod-calendar/src/calendar.ts index 7152edc31e..702b89df5f 100644 --- a/services/calendar/pod-calendar/src/calendar.ts +++ b/services/calendar/pod-calendar/src/calendar.ts @@ -35,7 +35,7 @@ import core, { parseSocialIdString } from '@hcengineering/core' import setting from '@hcengineering/setting' -import { htmlToMarkup, markupToHTML } from '@hcengineering/text' +import { htmlToMarkup, jsonToHTML, markupToJSON } from '@hcengineering/text' import { deepEqual } from 'fast-equals' import { calendar_v3 } from 'googleapis' import type { Collection, Db } from 'mongodb' @@ -972,7 +972,7 @@ export class CalendarClient { start: convertDate(event.date, event.allDay, getTimezone(event)), end: convertDate(event.dueDate, event.allDay, getTimezone(event)), id: event.eventId, - description: markupToHTML(event.description), + description: jsonToHTML(markupToJSON(event.description)), summary: event.title } if (event.location != null) { diff --git a/services/github/pod-github/package.json b/services/github/pod-github/package.json index 521fa395e0..3f65d24b2b 100644 --- a/services/github/pod-github/package.json +++ b/services/github/pod-github/package.json @@ -76,6 +76,7 @@ "@hcengineering/tracker": "^0.6.24", "@hcengineering/query": "^0.6.12", "@hcengineering/text": "^0.6.5", + "@hcengineering/text-markdown": "^0.6.0", "@hcengineering/github": "^0.6.0", "@hcengineering/attachment": "^0.6.14", "dotenv": "~16.0.0", diff --git a/services/github/pod-github/src/markdown/__tests__/textmodel.test.ts b/services/github/pod-github/src/markdown/__tests__/textmodel.test.ts index 4f9b2e2107..183aeeaddc 100644 --- a/services/github/pod-github/src/markdown/__tests__/textmodel.test.ts +++ b/services/github/pod-github/src/markdown/__tests__/textmodel.test.ts @@ -19,10 +19,10 @@ import { MarkupNode, MarkupNodeType, jsonToMarkup, - MarkdownState, traverseAllMarks, - traverseMarkupNode + traverseNode } from '@hcengineering/text' +import { MarkdownState } from '@hcengineering/text-markdown' import { markdownToMarkup, markupToMarkdown, parseMessageMarkdown, serializeMessage } from '..' describe('server', () => { @@ -788,8 +788,9 @@ A list of closed updated issues` const msg = parseMessageMarkdown(t1, 'ref://', 'http://', 'http://') const nodes = [] - traverseMarkupNode(msg, (node) => { + traverseNode(msg, (node) => { nodes.push(node) + return true }) expect(nodes.length).toEqual(5) const marks = [] diff --git a/services/github/pod-github/src/markdown/index.ts b/services/github/pod-github/src/markdown/index.ts index 16a1efd153..bcbf4881d2 100644 --- a/services/github/pod-github/src/markdown/index.ts +++ b/services/github/pod-github/src/markdown/index.ts @@ -13,15 +13,8 @@ // limitations under the License. // -import { - MarkupNode, - jsonToMarkup, - markupToJSON, - MarkdownParser, - storeNodes, - storeMarks, - MarkdownState -} from '@hcengineering/text' +import { MarkupNode, jsonToMarkup, markupToJSON, htmlToJSON } from '@hcengineering/text' +import { MarkdownParser, storeNodes, storeMarks, MarkdownState } from '@hcengineering/text-markdown' import { GithubKit } from './extensions' import { hasHulyLink, hasHulyLinkText } from '../sync/guest' @@ -42,8 +35,12 @@ export function parseMessageMarkdown ( } }) ] - const parser = new MarkdownParser(extensions, refUrl, imageUrl) - return parser.parse(message ?? '') + const htmlParser = (html: string): MarkupNode => { + return htmlToJSON(html, extensions) + } + const parser = new MarkdownParser({ refUrl, imageUrl, htmlParser }) + const json = parser.parse(message ?? '') + return json } /** diff --git a/services/github/pod-github/src/sync/guest.ts b/services/github/pod-github/src/sync/guest.ts index 0e7a48ce39..a573c44dcd 100644 --- a/services/github/pod-github/src/sync/guest.ts +++ b/services/github/pod-github/src/sync/guest.ts @@ -3,7 +3,7 @@ // import { Branding, generateUuid, PersonUuid, TxOperations, WorkspaceIds, WorkspaceUuid } from '@hcengineering/core' -import { MarkupMarkType, MarkupNode, MarkupNodeType, traverseMarkupNode } from '@hcengineering/text' +import { MarkupMarkType, MarkupNode, MarkupNodeType, traverseNode } from '@hcengineering/text' import { getPublicLink } from '@hcengineering/server-guest-resources' import { Task } from '@hcengineering/task' import { generateToken } from '@hcengineering/server-token' @@ -23,9 +23,9 @@ export function hasHulyLink (href: string, guestLink: string): boolean { export async function stripGuestLink (markdown: MarkupNode): Promise { const toRemove: MarkupNode[] = [] - traverseMarkupNode(markdown, (node) => { + traverseNode(markdown, (node) => { if (node.content === undefined) { - return + return false } const oldLength = node.content.length node.content = node.content.filter((it) => it.type !== MarkupNodeType.subLink) @@ -35,14 +35,18 @@ export async function stripGuestLink (markdown: MarkupNode): Promise { if (node.content.length !== oldLength && node.type === MarkupNodeType.paragraph) { toRemove.push(node) } + + return true }) // traverse nodes once again and remove empty parent node - traverseMarkupNode(markdown, (node) => { + traverseNode(markdown, (node) => { if (node.content === undefined) { - return + return false } node.content = node.content.filter((it) => !toRemove.includes(it)) + + return true }) } export async function appendGuestLink ( diff --git a/services/github/pod-github/src/worker.ts b/services/github/pod-github/src/worker.ts index b459a15230..bb9f065f40 100644 --- a/services/github/pod-github/src/worker.ts +++ b/services/github/pod-github/src/worker.ts @@ -55,7 +55,8 @@ import { LiveQuery } from '@hcengineering/query' import { StorageAdapter } from '@hcengineering/server-core' import { getPublicLinkUrl } from '@hcengineering/server-guest-resources' import task, { ProjectType, TaskType } from '@hcengineering/task' -import { MarkupNode, isMarkdownsEquals, jsonToMarkup } from '@hcengineering/text' +import { MarkupNode, jsonToMarkup } from '@hcengineering/text' +import { isMarkdownsEquals } from '@hcengineering/text-markdown' import tracker from '@hcengineering/tracker' import { User } from '@octokit/webhooks-types' import { App, Octokit } from 'octokit'