UBERF-10471: Fix Github miss status updates and allow to re-integrate existing repos (#8842)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

services/github/pod-github/src/sync/comments.ts

@@ -528,7 +528,7 @@ export class CommentSyncManager implements DocSyncManager {
     }
     const syncInfo = await this.client.findAll<DocSyncInfo>(github.class.DocSyncInfo, {
       space: repo.githubProject,
-      repository: repo._id,
+      // repository: repo._id, // If we skip repository, we will find orphaned comments, so we could connect them on.
       objectClass: chunter.class.ChatMessage,
       url: { $in: comments.map((it) => (it.url ?? '').toLowerCase()) }
     })
@@ -550,14 +550,19 @@ export class CommentSyncManager implements DocSyncManager {
             lastModified
           })
         } else {
-          if (!deepEqual(existing.external, comment) || existing.externalVersion !== githubExternalSyncVersion) {
+          if (
+            !deepEqual(existing.external, comment) ||
+            existing.externalVersion !== githubExternalSyncVersion ||
+            existing.repository !== repo._id
+          ) {
             await derivedClient.diffUpdate(
               existing,
               {
                 needSync: '',
                 external: comment,
                 externalVersion: githubExternalSyncVersion,
-                lastModified
+                lastModified,
+                repository: repo._id
               },
               lastModified
             )

services/github/pod-github/src/sync/issueBase.ts

@@ -69,6 +69,7 @@ import {
   errorToObj,
   getCreateStatus,
   getType,
+  guessStatus,
   isGHWriteAllowed
 } from './utils'
 
@@ -1075,6 +1076,51 @@ export abstract class IssueSyncManagerBase {
     return issueUpdate
   }
 
+  async performDocumentExternalSync (
+    ctx: MeasureContext,
+    info: DocSyncInfo,
+    previousExternal: IssueExternalData,
+    issueExternal: IssueExternalData,
+    derivedClient: TxOperations
+  ): Promise<void> {
+    // TODO: Since Github integeration need to be re-written to use cards, so this is quick fix to not loose data in case of external sync while service was offline.
+
+    const update: IssueUpdate = {}
+    const du: DocumentUpdate<DocSyncInfo> = {}
+    const account = (await this.provider.getAccount(issueExternal.author)) ?? core.account.System
+
+    const container = await this.provider.getContainer(info.space)
+    if (container == null) {
+      return
+    }
+    const type = await this.provider.getTaskTypeOf(container.project.type, tracker.class.Issue)
+    const statuses = await this.provider.getStatuses(type?._id)
+
+    if (previousExternal.state !== issueExternal.state) {
+      update.status = (
+        await guessStatus({ state: issueExternal.state, stateReason: issueExternal.stateReason }, statuses)
+      )._id
+    }
+    if (previousExternal.title !== issueExternal.title) {
+      update.title = issueExternal.title
+    }
+    if (previousExternal.body !== issueExternal.body) {
+      update.description = await this.provider.getMarkupSafe(
+        container.container,
+        issueExternal.body,
+        this.stripGuestLink
+      )
+      du.markdown = await this.provider.getMarkdown(update.description)
+    }
+    if (!deepEqual(previousExternal.assignees, issueExternal.assignees)) {
+      const assignees = await this.getAssignees(issueExternal)
+      update.assignee = assignees?.[0] ?? null
+    }
+    if (Object.keys(update).length > 0) {
+      await this.handleUpdate(issueExternal, derivedClient, update, account, container.project, false)
+    }
+  }
+
   async syncIssues (
     _class: Ref<Class<Doc>>,
     repo: GithubIntegrationRepository,
@@ -1089,7 +1135,7 @@ export abstract class IssueSyncManagerBase {
       syncDocs ??
       (await this.client.findAll<DocSyncInfo>(github.class.DocSyncInfo, {
         space: repo.githubProject,
-        repository: repo._id,
+        // repository: repo._id, // If we skip repository, we will find orphaned issues, so we could connect them on.
         objectClass: _class,
         url: { $in: issues.map((it) => (it.url ?? '').toLowerCase()) }
       }))
@@ -1123,9 +1169,19 @@ export abstract class IssueSyncManagerBase {
           if (syncDocs !== undefined) {
             syncDocs = syncDocs.filter((it) => it._id !== existing._id)
           }
-          const externalEqual = deepEqual(existing.external, issue)
+          const externalEqual = deepEqual(existing.external, issue) && existing.repository === repo._id
           if (!externalEqual || existing.externalVersion !== githubExternalSyncVersion) {
-            this.ctx.info('Update sync doc', { url: issue.url, workspace: this.provider.getWorkspaceId() })
+            this.ctx.info('Update sync doc(extarnal changes)', {
+              url: issue.url,
+              workspace: this.provider.getWorkspaceId()
+            })
+
+            if (existing.needSync === githubSyncVersion || existing.repository !== repo._id) {
+              // Sync external if and only if no changes from platform or we do resync from github.
+              // We need to apply changes from Github, while service was offline.
+              await this.performDocumentExternalSync(this.ctx, existing, existing.external, issue, derivedClient)
+            }
+
             await ops.diffUpdate(
               existing,
               {
@@ -1134,6 +1190,7 @@ export abstract class IssueSyncManagerBase {
                 externalVersion: githubExternalSyncVersion,
                 derivedVersion: '', // Clear derived state to recalculate it.
                 externalVersionSince: '',
+                repository: repo._id,
                 lastModified: new Date(issue.updatedAt).getTime()
               },
               Date.now()

services/github/pod-github/src/sync/utils.ts

@@ -370,12 +370,13 @@ export async function syncDerivedDocuments<T extends { url: string }> (
       })
     } else {
       processed.add(existing._id)
-      if (!deepEqual(existing.external, r)) {
+      if (!deepEqual(existing.external, r) || existing.repository !== repo._id) {
         // Only update if had changes.
         await derivedClient.update(existing, {
           external: r,
           needSync: '', // We need to check if we had any changes.
           derivedVersion: '',
+          repository: repo._id,
           externalVersion: githubExternalSyncVersion,
           lastModified: new Date(r.updatedAt ?? r.createdAt).getTime(),
           ...extra