diff --git a/packages/presentation/src/components/markup/NodeContent.svelte b/packages/presentation/src/components/markup/NodeContent.svelte index 381b6c1155..80eb244986 100644 --- a/packages/presentation/src/components/markup/NodeContent.svelte +++ b/packages/presentation/src/components/markup/NodeContent.svelte @@ -185,6 +185,8 @@ {/each} {/if} + {:else if node.type === MarkupNodeType.comment} + {:else} unknown node: "{node.type}" {#if nodes.length > 0} diff --git a/packages/text/src/kits/server-kit.ts b/packages/text/src/kits/server-kit.ts index 0a29d71ef8..1b01d876f6 100644 --- a/packages/text/src/kits/server-kit.ts +++ b/packages/text/src/kits/server-kit.ts @@ -32,6 +32,7 @@ import { CodeBlockExtension, codeBlockOptions } from '../nodes' import { DefaultKit, DefaultKitOptions } from './default-kit' import { CodeExtension, codeOptions } from '../marks/code' import { NoteBaseExtension } from '../marks/noteBase' +import { CommentNode } from '../nodes/comment' const headingLevels: Level[] = [1, 2, 3, 4, 5, 6] @@ -88,6 +89,7 @@ export const ServerKit = Extension.create({ TodoItemNode, TodoListNode, ReferenceNode, + CommentNode, NodeUuid, NoteBaseExtension ] diff --git a/packages/text/src/markdown/__tests__/compare.test.ts b/packages/text/src/markdown/__tests__/compare.test.ts new file mode 100644 index 0000000000..80e8452e8f --- /dev/null +++ b/packages/text/src/markdown/__tests__/compare.test.ts @@ -0,0 +1,51 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { parseMessageMarkdown, serializeMessage } from '..' +import { ServerKit } from '../../kits/server-kit' +import { isMarkdownsEquals } from '../compare' + +const refUrl: string = 'ref://' +const imageUrl: string = 'http://localhost' + +const extensions = [ServerKit] + +const tests: Array<{ name: string, body: string }> = [ + { name: 'Italic', body: '*Asteriscs* and _Underscores_' }, + { name: 'Bold', body: '**Asteriscs** and __Underscores__' }, + { name: 'Bullet list with asteriscs', body: 'Asterisks :\r\n* Firstly\r\n* Secondly' }, + { name: 'Bullet list with dashes', body: 'Dashes :\r\n- Firstly\r\n- Secondly' }, + { name: 'TODO list with asteriscs', body: '* [ ] Take\n* [ ] Do\n\n' }, + { name: 'TODO list with dashes', body: '- [x] Take\n- [ ] Do\n\n' }, + { + name: 'Different markers', + body: 'Asterisks bulleted list:\r\n* Asterisks: *Italic* and **Bold*** Underscores: _Italic_ and __Bold__\r\n\r\nDash bulleted list:\r\n- Asterisks: *Italic* and **Bold**\r\n- Underscores: _Italic_ and __Bold__\r\n-' + }, + { name: 'Single line comment', body: '' }, + { + name: 'Real with multiline comment', + body: '"\r\n\r\nThis PR fix issue #22424\r\n\r\n\r\n"' + } +] + +describe('md compatibility', () => { + tests.forEach(({ name, body: description }) => { + it(name, () => { + const json = parseMessageMarkdown(description, refUrl, imageUrl, extensions) + const serialized = serializeMessage(json, refUrl, imageUrl) + expect(isMarkdownsEquals(description, serialized)).toBe(true) + }) + }) +}) diff --git a/packages/text/src/markdown/__tests__/markdown.test.ts b/packages/text/src/markdown/__tests__/markdown.test.ts index f7b25fd36e..247cdd51f2 100644 --- a/packages/text/src/markdown/__tests__/markdown.test.ts +++ b/packages/text/src/markdown/__tests__/markdown.test.ts @@ -96,6 +96,9 @@ Lorem ipsum dolor sit amet. }, { type: 'bulletList', + attrs: { + bullet: '-' + }, content: [ { type: 'listItem', @@ -154,6 +157,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', @@ -216,6 +222,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', @@ -253,6 +262,9 @@ Lorem ipsum dolor sit amet. }, { type: 'bulletList', + attrs: { + bullet: '-' + }, content: [ { type: 'listItem', @@ -313,6 +325,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', @@ -334,6 +349,9 @@ Lorem ipsum dolor sit amet. }, { type: 'bulletList', + attrs: { + bullet: '-' + }, content: [ { type: 'listItem', @@ -354,6 +372,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', @@ -375,6 +396,9 @@ Lorem ipsum dolor sit amet. }, { type: 'bulletList', + attrs: { + bullet: '-' + }, content: [ { type: 'listItem', @@ -418,6 +442,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', @@ -435,6 +462,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', @@ -487,6 +517,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', @@ -504,6 +537,9 @@ Lorem ipsum dolor sit amet. }, { type: 'bulletList', + attrs: { + bullet: '-' + }, content: [ { type: 'listItem', @@ -524,6 +560,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', @@ -549,6 +588,9 @@ Lorem ipsum dolor sit amet. }, { type: 'bulletList', + attrs: { + bullet: '-' + }, content: [ { type: 'listItem', @@ -565,6 +607,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', @@ -586,6 +631,9 @@ Lorem ipsum dolor sit amet. }, { type: 'bulletList', + attrs: { + bullet: '-' + }, content: [ { type: 'listItem', @@ -633,6 +681,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', @@ -650,6 +701,9 @@ Lorem ipsum dolor sit amet. }, { type: 'todoList', + attrs: { + bullet: '-' + }, content: [ { type: 'todoItem', diff --git a/packages/text/src/markdown/compare.ts b/packages/text/src/markdown/compare.ts index ea269499fd..bca7202cea 100644 --- a/packages/text/src/markdown/compare.ts +++ b/packages/text/src/markdown/compare.ts @@ -34,13 +34,20 @@ export function calcSørensenDiceCoefficient (a: string, b: string): number { * Perform markdown diff/comparison to understand do we have a major differences. */ export function isMarkdownsEquals (source1: string, source2: string): boolean { - const lines1 = source1 - .split('\n') - .map((it) => it.trimEnd()) - .join('\n') - const lines2 = source2 - .split('\n') - .map((it) => it.trimEnd()) - .join('\n') + const normalizeLineEndings = (str: string): string => str.replace(/\r?\n/g, '\n') + + const excludeBlankLines = (str: string): string => + str + .split('\n') + .map((it) => it.trimEnd()) + .filter((it) => it.length > 0) + .join('\n') + + const norm1 = normalizeLineEndings(source1) + const lines1 = excludeBlankLines(norm1) + + const norm2 = normalizeLineEndings(source2) + const lines2 = excludeBlankLines(norm2) + return lines1 === lines2 } diff --git a/packages/text/src/markdown/parser.ts b/packages/text/src/markdown/parser.ts index 647211cecb..2e187e61e1 100644 --- a/packages/text/src/markdown/parser.ts +++ b/packages/text/src/markdown/parser.ts @@ -417,8 +417,18 @@ const tokensBlock: Record = { paragraph: { block: MarkupNodeType.paragraph }, list_item: { block: MarkupNodeType.list_item }, task_item: { block: MarkupNodeType.taskItem, getAttrs: (tok) => ({ 'data-type': 'taskItem' }) }, - bullet_list: { block: MarkupNodeType.bullet_list }, - todo_list: { block: MarkupNodeType.todoList }, + bullet_list: { + block: MarkupNodeType.bullet_list, + getAttrs: (tok) => ({ + bullet: tok.markup + }) + }, + todo_list: { + block: MarkupNodeType.todoList, + getAttrs: (tok) => ({ + bullet: tok.markup + }) + }, todo_item: { block: MarkupNodeType.todoItem, getAttrs: (tok) => ({ @@ -523,9 +533,24 @@ const tokensNode: Record = { hardbreak: { node: MarkupNodeType.hard_break } } const tokensMark: Record = { - em: { mark: MarkupMarkType.em }, - bold: { mark: MarkupMarkType.bold }, - strong: { mark: MarkupMarkType.bold }, + em: { + mark: MarkupMarkType.em, + getAttrs: (tok: Token, state: MarkdownParseState) => { + return { marker: tok.markup } + } + }, + bold: { + mark: MarkupMarkType.bold, + getAttrs: (tok: Token, state: MarkdownParseState) => { + return { marker: tok.markup } + } + }, + strong: { + mark: MarkupMarkType.bold, + getAttrs: (tok: Token, state: MarkdownParseState) => { + return { marker: tok.markup } + } + }, s: { mark: MarkupMarkType.strike }, u: { mark: MarkupMarkType.underline }, code_inline: { @@ -602,6 +627,7 @@ export class MarkdownParser { html: true }) this.tokenizer.core.ruler.after('inline', 'task_list', this.listRule) + this.tokenizer.core.ruler.after('inline', 'html_comment', this.htmlCommentRule) this.tokenHandlers = tokenHandlers(tokensBlock, tokensNode, tokensMark, specialRule, ignoreRule, extensions) } @@ -619,6 +645,19 @@ export class MarkdownParser { return doc } + htmlCommentRule: RuleCore = (state: StateCore): boolean => { + const tokens = state.tokens + for (let i = 0; i < tokens.length; i++) { + // Prosemirror entirely ignores comments when parsing, so + // here we replaces html comment tag with a custom tag so the comments got parsed as a node + if (tokens[i].type === 'html_block' || tokens[i].type === 'html_inline') { + const content = tokens[i].content.replaceAll('', '') + tokens[i].content = content + } + } + return true + } + listRule: RuleCore = (state: TaskListStateCore): boolean => { const tokens = state.tokens const states: Array<{ closeIdx: number, lastItemIdx: number }> = [] @@ -647,7 +686,9 @@ export class MarkdownParser { if (itemCloseIdx === -1) { itemCloseIdx = tokens.length - i } else if (tokens[i].type !== tokens[tokens.length - itemCloseIdx].type) { - tokens.splice(i + 1, 0, new state.Token('bullet_list_open', 'ul', 1)) + const bulletListOpen = new state.Token('bullet_list_open', 'ul', 1) + bulletListOpen.markup = tokens[i + 1].markup + tokens.splice(i + 1, 0, bulletListOpen) tokens.splice(i + 1, 0, new state.Token('bullet_list_close', 'ul', -1)) convertTodoList(tokens, i + 2, tokens.length - listCloseIdx, tokens.length - itemCloseIdx) listCloseIdx = tokens.length - i - 1 diff --git a/packages/text/src/markdown/serializer.ts b/packages/text/src/markdown/serializer.ts index d0386c5f11..6a11631e2d 100644 --- a/packages/text/src/markdown/serializer.ts +++ b/packages/text/src/markdown/serializer.ts @@ -4,7 +4,7 @@ import { messageContent, nodeAttrs } from './node' import { MarkupMark, MarkupNode, MarkupNodeType } from '../markup/model' import { defaultExtensions } from '../extensions' -type FirstDelim = (i: number, attrs?: Record) => string +type FirstDelim = (i: number, attrs?: Record, parentAttrs?: Record) => string interface IState { wrapBlock: (delim: string, firstDelim: string | null, node: MarkupNode, f: () => void) => void flushClose: (size: number) => void @@ -59,13 +59,14 @@ function isPlainURL (link: MarkupMark, parent: MarkupNode, index: number): boole return index === (parent.content?.length ?? 0) - 1 || !isInSet(link, parent.content?.[index + 1]?.marks ?? []) } -const formatTodoItem: FirstDelim = (i, attrs) => { +const formatTodoItem: FirstDelim = (i, attrs, parentAttrs?: Record) => { const meta = attrs?.todoid !== undefined && attrs?.userid !== undefined ? `` : '' - return `* [${attrs?.checked === true ? 'x' : ' '}] ${meta}` + const bullet = parentAttrs?.bullet ?? '*' + return `${bullet} [${attrs?.checked === true ? 'x' : ' '}] ${meta}` } // ************************************************************* @@ -196,6 +197,11 @@ export const storeNodes: Record = { ')' ) }, + comment: (state, node) => { + state.write('') + }, hardBreak: (state, node, parent, index) => { const content = messageContent(parent) for (let i = index + 1; i < content.length; i++) { @@ -667,7 +673,7 @@ export class MarkdownState implements IState { firstDelim: FirstDelim ): void { if (i > 0 && isTight) this.flushClose(1) - this.wrapBlock(delim, firstDelim(i, node.content?.[i].attrs), node, () => { + this.wrapBlock(delim, firstDelim(i, node.content?.[i].attrs, node.attrs), node, () => { this.render(child, node, i) }) } @@ -711,8 +717,11 @@ export class MarkdownState implements IState { // : (Mark, bool, string?) → string // Get the markdown string for a given opening or closing mark. markString (mark: MarkupMark, open: boolean, parent: MarkupNode, index: number): string { - const info = this.marks[mark.type] - const value = open ? info.open : info.close + let value = mark.attrs?.marker + if (value === undefined) { + const info = this.marks[mark.type] + value = open ? info.open : info.close + } return typeof value === 'string' ? value : value(this, mark, parent, index) ?? '' } } diff --git a/packages/text/src/markup/model.ts b/packages/text/src/markup/model.ts index 78fe89d876..453f729ea4 100644 --- a/packages/text/src/markup/model.ts +++ b/packages/text/src/markup/model.ts @@ -37,7 +37,8 @@ export enum MarkupNodeType { table = 'table', table_row = 'tableRow', table_cell = 'tableCell', - table_header = 'tableHeader' + table_header = 'tableHeader', + comment = 'comment' } /** @public */ diff --git a/packages/text/src/markup/utils.ts b/packages/text/src/markup/utils.ts index 59a3eb0cdb..016a4c64ee 100644 --- a/packages/text/src/markup/utils.ts +++ b/packages/text/src/markup/utils.ts @@ -200,7 +200,7 @@ export function markupToHTML (markup: Markup, extensions?: Extensions): string { /** @public */ export function htmlToJSON (html: string, extensions?: Extensions): MarkupNode { extensions = extensions ?? defaultExtensions - return generateJSON(html, extensions) as MarkupNode + return generateJSON(html, extensions, { preserveWhitespace: 'full' }) as MarkupNode } /** @public */ diff --git a/packages/text/src/nodes/comment.ts b/packages/text/src/nodes/comment.ts new file mode 100644 index 0000000000..1b09c0358b --- /dev/null +++ b/packages/text/src/nodes/comment.ts @@ -0,0 +1,47 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Node } from '@tiptap/core' +import { Node as ProseMirrorNode } from '@tiptap/pm/model' + +/** + * @public + */ +export interface Comment { + parentNode: ProseMirrorNode | null +} + +/** + * @public + */ +export const CommentNode = Node.create({ + name: 'comment', + group: 'inline', + inline: true, + content: 'text*', + marks: '_', + + parseHTML () { + return [ + { + tag: 'comment' + } + ] + }, + + renderText () { + return '' + } +}) diff --git a/packages/text/src/nodes/index.ts b/packages/text/src/nodes/index.ts index a854411b8e..6833d0e78b 100644 --- a/packages/text/src/nodes/index.ts +++ b/packages/text/src/nodes/index.ts @@ -18,4 +18,5 @@ export * from './reference' export * from './todo' export * from './file' export * from './codeblock' +export * from './comment' export { getDataAttribute } from './utils' diff --git a/plugins/text-editor-resources/src/components/MarkupDiffViewer.svelte b/plugins/text-editor-resources/src/components/MarkupDiffViewer.svelte index 79672f26e8..52b3233f8d 100644 --- a/plugins/text-editor-resources/src/components/MarkupDiffViewer.svelte +++ b/plugins/text-editor-resources/src/components/MarkupDiffViewer.svelte @@ -16,7 +16,7 @@ -->