UBERF-4725 Migrate collaborative content (#5717)

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
2026-08-23 21:02:24 +02:00 · 2024-08-22 16:03:42 +05:00
parent 0e72b85978
commit df2a9b2708
97 changed files with 702 additions and 820 deletions
@@ -4,14 +4,14 @@
 //

 import { CollaboratorClient, getClient as getCollaboratorClient } from '@hcengineering/collaborator-client'
-import { Hierarchy, WorkspaceId } from '@hcengineering/core'
+import { WorkspaceId } from '@hcengineering/core'
 import { generateToken } from '@hcengineering/server-token'
 import config from './config'

 /**
 * @public
 */
-export function createCollaboratorClient (hierarchy: Hierarchy, workspaceId: WorkspaceId): CollaboratorClient {
+export function createCollaboratorClient (workspaceId: WorkspaceId): CollaboratorClient {
  const token = generateToken(config.SystemEmail, workspaceId, { mode: 'github' })
-  return getCollaboratorClient(hierarchy, workspaceId, token, config.CollaboratorURL)
+  return getCollaboratorClient(workspaceId, token, config.CollaboratorURL)
 }
@@ -54,7 +54,7 @@ const envMap: { [key in keyof Config]: string } = {
  MongoURL: 'MONGO_URL',
  ConfigurationDB: 'MONGO_DB',

-  CollaboratorURL: 'COLLABORATOR_API_URL',
+  CollaboratorURL: 'COLLABORATOR_URL',

  ProductID: 'PRODUCT_ID',

@@ -5,7 +5,7 @@
 import { Branding, TxOperations, WorkspaceIdWithUrl } from '@hcengineering/core'
 import { MarkupMarkType, MarkupNode, MarkupNodeType, traverseMarkupNode } from '@hcengineering/text'
 import { getPublicLink } from '@hcengineering/server-guest-resources'
-import { Issue } from '@hcengineering/tracker'
+import { Task } from '@hcengineering/task'

 const githubLinkText = process.env.LINK_TEXT ?? 'Huly&reg;:'

@@ -46,7 +46,7 @@ export async function stripGuestLink (markdown: MarkupNode): Promise<void> {
 }
 export async function appendGuestLink (
  client: TxOperations,
-  doc: Issue,
+  doc: Task,
  markdown: MarkupNode,
  workspace: WorkspaceIdWithUrl,
  branding: Branding | null
@@ -14,16 +14,16 @@ import core, {
  Account,
  AttachedDoc,
  Class,
+  CollaborativeDoc,
  Doc,
  DocumentUpdate,
+  Markup,
  MeasureContext,
  Ref,
  Space,
  Status,
  TxOperations,
-  generateId,
-  getCollaborativeDoc,
-  getCollaborativeDocId
+  generateId
 } from '@hcengineering/core'
 import { IntlString } from '@hcengineering/platform'
 import { LiveQuery } from '@hcengineering/query'
@@ -72,11 +72,18 @@ import {
  isGHWriteAllowed
 } from './utils'

+/**
+ * @public
+ */
+export type WithMarkup<T> = {
+  [P in keyof T]: T[P] extends CollaborativeDoc ? Markup : T[P]
+}
+
 /**
 * @public
 */
 export type GithubIssueData = Omit<
-Issue,
+WithMarkup<Issue>,
 | 'commits'
 | 'attachments'
 | 'commits'
@@ -103,6 +110,11 @@ Issue,
 > &
 Record<string, any>

+/**
+ * @public
+ */
+export type IssueUpdate = DocumentUpdate<WithMarkup<Issue>>
+
 /**
 * @public
 */
@@ -325,7 +337,7 @@ export abstract class IssueSyncManagerBase {
  async handleUpdate (
    external: IssueExternalData,
    derivedClient: TxOperations,
-    update: DocumentUpdate<Issue>,
+    update: IssueUpdate,
    account: Ref<Account>,
    prj: GithubProject,
    needSync: boolean,
@@ -334,7 +346,7 @@ export abstract class IssueSyncManagerBase {
      state: DocSyncInfo,
      existing: Issue,
      external: IssueExternalData,
-      update: DocumentUpdate<Issue>
+      update: IssueUpdate
    ) => Promise<boolean>,
    extraSyncUpdate?: DocumentUpdate<DocSyncInfo>
  ): Promise<void> {
@@ -354,13 +366,22 @@ export abstract class IssueSyncManagerBase {
      const lastModified = new Date().getTime()

      if (doc !== undefined && ((await verifyUpdate?.(syncData, doc, external, update)) ?? true)) {
+        const issueData: DocumentUpdate<Issue> = { ...update, description: doc.description }
        if (
          update.description !== undefined &&
          !areEqualMarkups(update.description, syncData.current?.description ?? '')
        ) {
          try {
-            const collaborativeDoc = getCollaborativeDoc(getCollaborativeDocId(doc._id, 'description'))
-            await this.collaborator.updateContent(collaborativeDoc, 'description', update.description)
+            const versionId = `${Date.now()}`
+            issueData.description = await this.collaborator.updateContent(
+              doc.description,
+              { description: update.description },
+              {
+                versionId,
+                versionName: versionId,
+                createdBy: account
+              }
+            )
          } catch (err: any) {
            Analytics.handleError(err)
            this.ctx.error(err)
@@ -405,7 +426,12 @@ export abstract class IssueSyncManagerBase {
          },
          lastModified
        )
-        await this.client.diffUpdate(this.client.getHierarchy().as(doc, prj.mixinClass), update, lastModified, account)
+        await this.client.diffUpdate(
+          this.client.getHierarchy().as(doc, prj.mixinClass),
+          issueData,
+          lastModified,
+          account
+        )
        this.provider.sync()
      }
    }
@@ -649,7 +675,7 @@ export abstract class IssueSyncManagerBase {

  abstract performIssueFieldsUpdate (
    info: DocSyncInfo,
-    existing: Issue,
+    existing: WithMarkup<Issue>,
    platformUpdate: DocumentUpdate<Issue>,
    issueData: GithubIssueData,
    container: ContainerFocus,
@@ -662,7 +688,7 @@ export abstract class IssueSyncManagerBase {

  async handleDiffUpdate (
    target: IssueSyncTarget,
-    existing: Issue,
+    existing: WithMarkup<Issue>,
    info: DocSyncInfo,
    issueData: GithubIssueData,
    container: ContainerFocus,
@@ -894,21 +920,30 @@ export abstract class IssueSyncManagerBase {
      }
    }

-    if (Object.keys(update).length > 0) {
-      // Update collaborative description
+    // Update collaborative description
+    if (update.description !== undefined) {
      this.ctx.info(`<= perform ${issueExternal.url} update to collaborator`, {
        workspace: this.provider.getWorkspaceId().name
      })
-      if (update.description !== undefined) {
-        try {
-          issueData.description = update.description
-          const collaborativeDoc = getCollaborativeDoc(getCollaborativeDocId(existingIssue._id, 'description'))
-          await this.collaborator.updateContent(collaborativeDoc, 'description', update.description)
-        } catch (err: any) {
-          Analytics.handleError(err)
-          this.ctx.error('error during description update', err)
-        }
+      try {
+        const versionId = `${Date.now()}`
+        issueData.description = update.description
+        update.description = await this.collaborator.updateContent(
+          existingIssue.description,
+          { description: update.description },
+          {
+            versionId,
+            versionName: versionId,
+            createdBy: account
+          }
+        )
+      } catch (err: any) {
+        Analytics.handleError(err)
+        this.ctx.error('error during description update', err)
      }
+    }
+
+    if (Object.keys(update).length > 0) {
      // We have some fields to update of existing from external
      this.ctx.info(`<= perform ${issueExternal.url} update to platform`, {
        ...update,
@@ -930,7 +965,7 @@ export abstract class IssueSyncManagerBase {
  private async notifyConnected (
    container: ContainerFocus,
    info: DocSyncInfo,
-    existing: Issue,
+    existing: WithMarkup<Issue>,
    issueExternal: IssueExternalData
  ): Promise<void> {
    const repo = container.repository.find((it) => it._id === info.repository) as GithubIntegrationRepository
@@ -948,9 +983,9 @@ export abstract class IssueSyncManagerBase {

  async collectIssueUpdate (
    info: DocSyncInfo,
-    doc: Issue,
+    doc: WithMarkup<Issue>,
    platformUpdate: DocumentUpdate<Issue>,
-    issueData: Pick<Issue, 'title' | 'description' | 'assignee' | 'status'>,
+    issueData: Pick<WithMarkup<Issue>, 'title' | 'description' | 'assignee' | 'status'>,
    container: ContainerFocus,
    issueExternal: IssueExternalData,
    _class: Ref<Class<Issue>>
@@ -19,11 +19,9 @@ import core, {
  TxOperations,
  cutObjectArray,
  generateId,
-  getCollaborativeDoc,
-  getCollaborativeDocId
+  makeCollaborativeDoc
 } from '@hcengineering/core'
 import task, { TaskType, calcRank } from '@hcengineering/task'
-import { isEmptyMarkup } from '@hcengineering/text'
 import tracker, { Issue, IssuePriority } from '@hcengineering/tracker'
 import { Issue as GithubIssue, IssuesEvent, ProjectsV2ItemEvent } from '@octokit/webhooks-types'
 import github, {
@@ -47,7 +45,7 @@ import {
 } from '../types'
 import { IssueExternalData, issueDetails } from './githubTypes'
 import { appendGuestLink } from './guest'
-import { GithubIssueData, IssueSyncManagerBase, IssueSyncTarget } from './issueBase'
+import { GithubIssueData, IssueSyncManagerBase, IssueSyncTarget, IssueUpdate, WithMarkup } from './issueBase'
 import { syncConfig } from './syncConfig'
 import { getSince, gqlp, guessStatus, isGHWriteAllowed, syncRunner } from './utils'

@@ -214,7 +212,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
        break
      }
      case 'edited': {
-        const update: DocumentUpdate<Issue> = {}
+        const update: IssueUpdate = {}
        const du: DocumentUpdate<DocSyncInfo> = {}
        if (event.changes.body !== undefined) {
          update.description = await this.provider.getMarkup(integration, event.issue.body, this.stripGuestLink)
@@ -373,6 +371,11 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
        return { needSync: githubSyncVersion }
      }

+      const description = await this.ctx.withLog('query collaborative description', {}, async () => {
+        const content = await this.collaborator.getContent((existing as Issue).description)
+        return content.description ?? ''
+      })
+
      this.ctx.info('create github issue', {
        title: (existing as Issue).title,
        number: (existing as Issue).number,
@@ -382,7 +385,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
        'create github issue',
        {},
        async () => {
-          this.createPromise = this.createGithubIssue(container, existing as Issue, repository)
+          this.createPromise = this.createGithubIssue(container, { ...(existing as Issue), description }, repository)
          return await this.createPromise
        },
        { id: (existing as Issue).identifier, workspace: this.provider.getWorkspaceId().name }
@@ -612,13 +615,12 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
      }
    } else {
      try {
-        const collaborativeDoc = getCollaborativeDoc(getCollaborativeDocId(existing._id, 'description'))
        const description = await this.ctx.withLog(
          'query collaborative description',
          {},
          async () => {
-            const content = await this.collaborator.getContent(collaborativeDoc, 'description')
-            return isEmptyMarkup(content) ? (existing as Issue).description : content
+            const content = await this.collaborator.getContent((existing as Issue).description)
+            return content.description ?? ''
          },
          { url: issueExternal.url }
        )
@@ -657,9 +659,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan

  async performIssueFieldsUpdate (
    info: DocSyncInfo,
-    existing: Issue,
+    existing: WithMarkup<Issue>,
    platformUpdate: DocumentUpdate<Issue>,
-    issueData: Pick<Issue, 'title' | 'description' | 'assignee' | 'status' | 'remainingTime' | 'component'>,
+    issueData: Pick<WithMarkup<Issue>, 'title' | 'description' | 'assignee' | 'status' | 'remainingTime' | 'component'>,
    container: ContainerFocus,
    issueExternal: IssueExternalData,
    okit: Octokit,
@@ -751,7 +753,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan

  async createGithubIssue (
    container: ContainerFocus,
-    existing: Issue,
+    existing: WithMarkup<Issue>,
    repository: GithubIntegrationRepository
  ): Promise<IssueExternalData | undefined> {
    const existingIssue = existing
@@ -847,8 +849,13 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan

    const number = (incResult as any).object.sequence

+    const issueId = info._id as unknown as Ref<Issue>
+
+    const { description, ...update } = issueData
+
    const value: AttachedData<Issue> = {
-      ...issueData,
+      ...update,
+      description: makeCollaborativeDoc(issueId, 'description'),
      kind: taskType,
      component: null,
      milestone: null,
@@ -867,7 +874,8 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
      childInfo: [],
      identifier: `${prj.identifier}-${number}`
    }
-    const issueId = info._id as unknown as Ref<Issue>
+
+    await this.collaborator.updateContent(value.description, { description })

    await this.client.addCollection(
      tracker.class.Issue,
@@ -16,11 +16,9 @@ import core, {
  WithLookup,
  cutObjectArray,
  generateId,
-  getCollaborativeDoc,
-  getCollaborativeDocId
+  makeCollaborativeDoc
 } from '@hcengineering/core'
 import task, { TaskType, calcRank, makeRank } from '@hcengineering/task'
-import { isEmptyMarkup } from '@hcengineering/text'
 import time, { ToDo, ToDoPriority } from '@hcengineering/time'
 import tracker, { Issue, IssuePriority, IssueStatus, Project } from '@hcengineering/tracker'
 import { OctokitResponse } from '@octokit/types'
@@ -59,13 +57,15 @@ import {
  toReviewDecision,
  toReviewState
 } from './githubTypes'
-import { GithubIssueData, IssueSyncManagerBase, IssueSyncTarget } from './issueBase'
+import { GithubIssueData, IssueSyncManagerBase, IssueSyncTarget, WithMarkup } from './issueBase'
 import { syncConfig } from './syncConfig'
 import { errorToObj, getSinceRaw, gqlp, guessStatus, isGHWriteAllowed, syncDerivedDocuments, syncRunner } from './utils'

 type GithubPullRequestData = GithubIssueData &
 Omit<GithubPullRequest, keyof Issue | 'commits' | 'reviews' | 'reviewComments'>

+type GithubPullRequestUpdate = DocumentUpdate<WithMarkup<GithubPullRequest>>
+
 export class PullRequestSyncManager extends IssueSyncManagerBase implements DocSyncManager {
  externalDerivedSync = true
  async handleEvent<T>(integration: IntegrationContainer, derivedClient: TxOperations, evt: T): Promise<void> {
@@ -194,7 +194,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
        break
      }
      case 'edited': {
-        const update: DocumentUpdate<GithubPullRequest> = {}
+        const update: GithubPullRequestUpdate = {}
        const du: DocumentUpdate<DocSyncInfo> = {}
        if (event.changes.title !== undefined) {
          update.title = event.pull_request.title
@@ -584,9 +584,8 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
          'query collaborative pull request description',
          {},
          async () => {
-            const collaborativeDoc = getCollaborativeDoc(getCollaborativeDocId(existing._id, 'description'))
-            const content = await this.collaborator.getContent(collaborativeDoc, 'description')
-            return isEmptyMarkup(content) ? (existing as Issue).description : content
+            const content = await this.collaborator.getContent((existing as any).description)
+            return content.description
          },
          { url: pullRequestExternal.url }
        )
@@ -987,7 +986,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
    info: DocSyncInfo,
    existing: Issue,
    platformUpdate: DocumentUpdate<Issue>,
-    issueData: Pick<Issue, 'title' | 'description' | 'assignee' | 'status' | 'remainingTime' | 'component'>,
+    issueData: Pick<WithMarkup<Issue>, 'title' | 'description' | 'assignee' | 'status' | 'remainingTime' | 'component'>,
    container: ContainerFocus,
    issueExternal: IssueExternalData,
    okit: Octokit,
@@ -1162,10 +1161,14 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
      account
    )

+    const prId = info._id as unknown as Ref<GithubPullRequest>
+
+    const { description, ...data } = pullRequestData
    const project = (incResult as any).object as Project
    const number = project.sequence
    const value: AttachedData<GithubPullRequest> = {
-      ...pullRequestData,
+      ...data,
+      description: makeCollaborativeDoc(prId, 'description'),
      kind: taskType,
      component: null,
      milestone: null,
@@ -1188,7 +1191,8 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
      reviews: 0
    }

-    const prId = info._id as unknown as Ref<GithubPullRequest>
+    await this.collaborator.updateContent(value.description, { description })
+
    await client.addCollection(
      github.class.GithubPullRequest,
      info.space,
@@ -320,7 +320,7 @@ export class GithubWorker implements IntegrationManager {

    this.repositoryManager = new RepositorySyncMapper(this.ctx.newChild('repository', {}), this._client, this.app)

-    this.collaborator = createCollaboratorClient(this._client.getHierarchy(), this.workspace)
+    this.collaborator = createCollaboratorClient(this.workspace)

    this.personMapper = new UsersSyncManager(this.ctx.newChild('users', {}), this._client, this.liveQuery)