UBERF-9634 Handle unsupported markdown in github integration (port to develop) (#8260)

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>

services/github/pod-github/src/config.ts

@@ -99,7 +99,7 @@ const config: Config = (() => {
     WebhookSecret: process.env[envMap.WebhookSecret] ?? 'secret',
     EnterpriseHostname: process.env[envMap.EnterpriseHostname],
     Port: parseInt(process.env[envMap.Port] ?? '3500'),
-    BotName: process.env[envMap.BotName] ?? 'dev[bot]',
+    BotName: process.env[envMap.BotName] ?? 'ao-huly-dev[bot]',
 
     MongoURL: process.env[envMap.MongoURL],
     ConfigurationDB: process.env[envMap.ConfigurationDB] ?? '%github',

services/github/pod-github/src/markdown/__tests__/textmodel.test.ts

@@ -545,13 +545,13 @@ A list of closed updated issues`
   })
 
   it('Check underline heading rule', () => {
-    const t1 = 'Hello\n---\nSome text'
+    const t1 = 'Hello\n---\n\nSome text'
     const msg = parseMessageMarkdown(t1, 'ref://', 'http://', 'http://')
     expect(msg.type).toEqual(MarkupNodeType.doc)
 
     const md = serializeMessage(msg, 'ref://', 'http://')
 
-    expect(md).toEqual('## Hello\n\nSome text')
+    expect(md).toEqual('Hello\n---\n\nSome text')
   })
 
   it('Check horizontal line', () => {

services/github/pod-github/src/sync/comments.ts

@@ -188,7 +188,7 @@ export class CommentSyncManager implements DocSyncManager {
         })
 
         const messageData: MessageData = {
-          message: await this.provider.getMarkup(integration, event.comment.body)
+          message: await this.provider.getMarkupSafe(integration, event.comment.body)
         }
 
         if (commentData !== undefined) {
@@ -280,7 +280,7 @@ export class CommentSyncManager implements DocSyncManager {
     const account = existing?.modifiedBy ?? (await this.provider.getAccountU(comment.user))?._id ?? core.account.System
 
     const messageData: MessageData = {
-      message: await this.provider.getMarkup(container.container, comment.body)
+      message: await this.provider.getMarkupSafe(container.container, comment.body)
     }
     if (existing === undefined) {
       try {

services/github/pod-github/src/sync/issueBase.ts

@@ -796,7 +796,7 @@ export abstract class IssueSyncManagerBase {
 
     //   if (k === 'description' && pv != null) {
     //     const mdown = await this.provider.getMarkdown(pv)
-    //     pv = await this.provider.getMarkup(container.container, mdown, this.stripGuestLink)
+    //     pv = await this.provider.getMarkupSafe(container.container, mdown, this.stripGuestLink)
     //   }
     //   if (pv != null && pv !== v) {
     //     // We have conflict of values, assume platform is more proper one.
@@ -1014,7 +1014,7 @@ export abstract class IssueSyncManagerBase {
     // if (platformUpdate.description != null) {
     //   // Need to convert to markdown
     //   issueUpdate.body = await this.provider.getMarkdown(platformUpdate.description ?? '')
-    //   issueData.description = await this.provider.getMarkup(
+    //   issueData.description = await this.provider.getMarkupSafe(
     //     container.container,
     //     issueUpdate.body ?? '',
     //     this.stripGuestLink

services/github/pod-github/src/sync/issues.ts

@@ -220,7 +220,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
         const update: IssueUpdate = {}
         const du: DocumentUpdate<DocSyncInfo> = {}
         if (event.changes.body !== undefined) {
-          update.description = await this.provider.getMarkup(integration, event.issue.body, this.stripGuestLink)
+          update.description = await this.provider.getMarkupSafe(integration, event.issue.body, this.stripGuestLink)
           du.markdown = await this.provider.getMarkdown(update.description)
         }
         if (event.changes.title !== undefined) {
@@ -413,7 +413,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
         addHulyLink: false, // Do not need, since we create comment on Github about issue is connected.
         current: {
           title: issueExternal.title,
-          description: await this.provider.getMarkup(container.container, issueExternal.body, this.stripGuestLink)
+          description: await this.provider.getMarkupSafe(container.container, issueExternal.body, this.stripGuestLink)
         }
       }
       needCreateConnectedAtHuly = true
@@ -501,7 +501,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
 
     const issueData = {
       title: issueExternal.title,
-      description: await this.provider.getMarkup(container.container, issueExternal.body, this.stripGuestLink),
+      description: await this.provider.getMarkupSafe(container.container, issueExternal.body, this.stripGuestLink),
       assignee: assignees[0]?.person,
       repository: info.repository,
       remainingTime: 0
@@ -767,7 +767,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
           },
           { url: issueExternal.url, id: existing._id }
         )
-        issueData.description = await this.provider.getMarkup(container.container, body, this.stripGuestLink)
+        issueData.description = await this.provider.getMarkupSafe(container.container, body, this.stripGuestLink)
       } else if (hasFieldStateChanges) {
         await this.ctx.withLog(
           '==> updateIssue',

services/github/pod-github/src/sync/projects.ts

@@ -174,7 +174,7 @@ export class ProjectsSyncManager implements DocSyncManager {
 
       const messageData: MilestoneData = {
         label: milestoneExternal.label,
-        description: await this.provider.getMarkup(container.container, milestoneExternal.description)
+        description: await this.provider.getMarkupSafe(container.container, milestoneExternal.description)
       }
 
       await this.handleDiffUpdateMilestone(existing, info, messageData, container, milestoneExternal)

services/github/pod-github/src/sync/pullrequests.ts

@@ -211,7 +211,11 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
           update.title = event.pull_request.title
         }
         if (event.changes.body !== undefined) {
-          update.description = await this.provider.getMarkup(integration, event.pull_request.body, this.stripGuestLink)
+          update.description = await this.provider.getMarkupSafe(
+            integration,
+            event.pull_request.body,
+            this.stripGuestLink
+          )
           du.markdown = await this.provider.getMarkdown(update.description)
         }
         if (event.changes.base !== undefined) {
@@ -464,7 +468,11 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
     }
     const pullRequestData: GithubPullRequestData = {
       title: pullRequestExternal.title,
-      description: await this.provider.getMarkup(container.container, pullRequestExternal.body, this.stripGuestLink),
+      description: await this.provider.getMarkupSafe(
+        container.container,
+        pullRequestExternal.body,
+        this.stripGuestLink
+      ),
       assignee: assignees[0]?.person ?? null,
       reviewers: reviewers.map((it: any) => it.person),
       draft: pullRequestExternal.isDraft,
@@ -1067,7 +1075,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
           },
           { url: issueExternal.url }
         )
-        issueData.description = await this.provider.getMarkup(container.container, body, this.stripGuestLink)
+        issueData.description = await this.provider.getMarkupSafe(container.container, body, this.stripGuestLink)
       } else if (hasFieldsUpdate) {
         await this.ctx.withLog('==> updatePullRequest:', {}, async () => {
           this.ctx.info('update-fields', {

services/github/pod-github/src/sync/reviewComments.ts

@@ -245,7 +245,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
           )
           if (reviewObj !== undefined) {
             const lastModified = Date.now()
-            const body = await this.provider.getMarkup(integration, event.comment.body)
+            const body = await this.provider.getMarkupSafe(integration, event.comment.body)
             await derivedClient.diffUpdate(
               reviewData,
               {
@@ -343,7 +343,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
     }
 
     const messageData: ReviewCommentData = {
-      body: await this.provider.getMarkup(container.container, reviewComment.body),
+      body: await this.provider.getMarkupSafe(container.container, reviewComment.body),
       diffHunk: reviewComment.diffHunk,
       isMinimized: reviewComment.isMinimized,
       reviewUrl: reviewComment.pullRequestReview.url,
@@ -414,7 +414,7 @@ export class ReviewCommentSyncManager implements DocSyncManager {
 
     if (Object.keys(platformUpdate).length > 0) {
       if (platformUpdate.body !== undefined) {
-        const body = await this.provider.getMarkup(container.container, platformUpdate.body)
+        const body = await this.provider.getMarkupSafe(container.container, platformUpdate.body)
         const okit = (await this.provider.getOctokit(account)) ?? container.container.octokit
         const q = `mutation updateReviewComment($commentID: ID!, $body: String!) {
           updatePullRequestReviewComment(input: {

services/github/pod-github/src/sync/reviews.ts

@@ -305,7 +305,7 @@ export class ReviewSyncManager implements DocSyncManager {
     const account = existing?.modifiedBy ?? (await this.provider.getAccount(review.author))?._id ?? core.account.System
 
     const messageData: ReviewData = {
-      body: await this.provider.getMarkup(container.container, review.body),
+      body: await this.provider.getMarkupSafe(container.container, review.body),
       state: toReviewState(review.state),
       comments: (review.comments?.nodes ?? []).map((it) => it.url)
     }

services/github/pod-github/src/types.ts

@@ -87,6 +87,11 @@ export interface IntegrationManager {
   getAccount: (user?: UserInfo | null) => Promise<any | undefined>
   getAccountU: (user: User) => Promise<any | undefined>
   getOctokit: (account: PersonId) => Promise<Octokit | undefined>
+  getMarkupSafe: (
+    container: IntegrationContainer,
+    text?: string | null,
+    preprocessor?: (nodes: MarkupNode) => Promise<void>
+  ) => Promise<string>
   getMarkup: (
     container: IntegrationContainer,
     text?: string | null,

services/github/pod-github/src/worker.ts

@@ -55,7 +55,7 @@ import { LiveQuery } from '@hcengineering/query'
 import { StorageAdapter } from '@hcengineering/server-core'
 import { getPublicLinkUrl } from '@hcengineering/server-guest-resources'
 import task, { ProjectType, TaskType } from '@hcengineering/task'
-import { MarkupNode, jsonToMarkup } from '@hcengineering/text'
+import { MarkupNode, MarkupNodeType, jsonToMarkup } from '@hcengineering/text'
 import { isMarkdownsEquals } from '@hcengineering/text-markdown'
 import tracker from '@hcengineering/tracker'
 import { User } from '@octokit/webhooks-types'
@@ -174,8 +174,40 @@ export class GithubWorker implements IntegrationManager {
     body: string
   ): Promise<{ markdownCompatible: boolean, markdown: string }> {
     const markupText = await this.getMarkup(container, body)
-    const markDown = await this.getMarkdown(markupText)
-    return { markdownCompatible: isMarkdownsEquals(body, markDown), markdown: markDown }
+    const markdown = await this.getMarkdown(markupText)
+    const markdownCompatible = isMarkdownsEquals(body, markdown)
+    return { markdownCompatible, markdown }
+  }
+
+  async getMarkupSafe (
+    container: IntegrationContainer,
+    text?: string | null,
+    preprocessor?: (nodes: MarkupNode) => Promise<void>
+  ): Promise<string> {
+    if (text == null) {
+      return ''
+    }
+
+    const markup = await this.getMarkup(container, text, preprocessor)
+    const markdown = await this.getMarkdown(markup)
+    const compatible = isMarkdownsEquals(text, markdown)
+
+    return compatible
+      ? markup
+      : jsonToMarkup({
+        type: MarkupNodeType.doc,
+        content: [
+          {
+            type: MarkupNodeType.markdown,
+            content: [
+              {
+                type: MarkupNodeType.text,
+                text
+              }
+            ]
+          }
+        ]
+      })
   }
 
   async getMarkup (