Add script to restore Github integrations (#9700)

Signed-off-by: Artem Savchenko <armisav@gmail.com>
2026-09-28 04:25:03 +02:00 · 2025-08-20 12:13:29 +07:00
parent 47402753ff
commit b438971926
3 changed files with 134 additions and 188 deletions
@@ -125,7 +125,7 @@ import { existsSync } from 'fs'
 import { mkdir, writeFile } from 'fs/promises'
 import { dirname } from 'path'
 import { restoreMarkupRefs } from './markup'
-import { performIntegrationMigrations } from './integrations'
+import { restoreGithubIntegrations } from './restoreGithub'

 const colorConstants = {
  colorRed: '\u001b[31m',
@@ -2702,13 +2702,12 @@ export function devTool (
    })

  program
-    .command('migrate-integrations')
-    .option('--db <db>', 'DB name', 'gmail-service')
-    .option('--region <region>', 'DB region')
-    .action(async (cmd: { db: string, region?: string }) => {
-      const { dbUrl, txes } = prepareTools()
+    .command('restore-github-integrations')
+    .option('-d, --dryrun', 'Dry run', false)
+    .action(async (cmd: { dryrun: boolean }) => {
+      const { dbUrl } = prepareTools()

-      await performIntegrationMigrations(dbUrl, cmd.region ?? null, txes)
+      await restoreGithubIntegrations(dbUrl, cmd.dryrun)
    })

  program
@@ -1,181 +0,0 @@
-import {
-  type Client,
-  type IntegrationKind,
-  MeasureMetricsContext,
-  type Ref,
-  type Tx,
-  type WorkspaceInfoWithStatus,
-  type WorkspaceUuid,
-  systemAccountUuid
-} from '@hcengineering/core'
-import { getAccountClient } from '@hcengineering/server-client'
-import { generateToken } from '@hcengineering/server-token'
-import { type Integration } from '@hcengineering/account-client'
-import setting, { type IntegrationType, type Integration as IntegrationSetting } from '@hcengineering/setting'
-
-import { type PipelineFactory, createDummyStorageAdapter, wrapPipeline } from '@hcengineering/server-core'
-import { createBackupPipeline, createEmptyBroadcastOps } from '@hcengineering/server-pipeline'
-
-interface WorkspaceInfoProvider {
-  getWorkspaceInfo: (workspaceUuid: WorkspaceUuid) => Promise<WorkspaceInfoWithStatus | undefined>
-}
-
-const GMAIL_INTEGRATION: IntegrationKind = 'gmail' as any
-const CALENDAR_INTEGRATION: IntegrationKind = 'google-calendar' as any
-const GITHUB_INTEGRATION: IntegrationKind = 'github' as any
-
-export async function performIntegrationMigrations (dbUrl: string, region: string | null, txes: Tx[]): Promise<void> {
-  console.log('Start integration migrations')
-  const token = generateToken(systemAccountUuid, undefined, { service: 'admin', admin: 'true' })
-  const accountClient = getAccountClient(token)
-
-  const allWorkspaces = await accountClient.listWorkspaces(region)
-  const byId = new Map(allWorkspaces.map((it) => [it.uuid, it]))
-  const workspaceProvider: WorkspaceInfoProvider = {
-    getWorkspaceInfo: async (workspaceUuid: WorkspaceUuid) => {
-      const ws = byId.get(workspaceUuid as any)
-      if (ws == null) {
-        console.error('No workspace found for token', workspaceUuid)
-        return undefined
-      }
-      return ws
-    }
-  }
-  const metricsContext = new MeasureMetricsContext('integrations-migrate', {})
-
-  const factory: PipelineFactory = createBackupPipeline(metricsContext, dbUrl, txes, {
-    externalStorage: createDummyStorageAdapter(),
-    usePassedCtx: true
-  })
-
-  await migrateIntegrations(token, workspaceProvider, factory, metricsContext)
-
-  console.log('Finished integration migrations')
-}
-
-async function migrateIntegrations (
-  token: string,
-  workspaceProvider: WorkspaceInfoProvider,
-  factory: PipelineFactory,
-  metricsContext: MeasureMetricsContext
-): Promise<void> {
-  try {
-    console.log('Start integrations data migrations')
-    const accountClient = getAccountClient(token)
-    const integrations = await accountClient.listIntegrations({})
-    console.log('Integrations count', integrations.length)
-
-    const workspaceUuids = new Set<WorkspaceUuid>()
-    const integrationsByWorkspaces = integrations.reduce<Record<string, typeof integrations>>((acc, integration) => {
-      const workspaceId = integration.workspaceUuid
-      if (workspaceId == null) {
-        return acc
-      }
-      if (acc[workspaceId] === undefined) {
-        workspaceUuids.add(workspaceId)
-        acc[workspaceId] = []
-      }
-      acc[workspaceId].push(integration)
-      return acc
-    }, {})
-
-    for (const workspace of workspaceUuids) {
-      const wsInfo = await workspaceProvider.getWorkspaceInfo(workspace)
-      if (wsInfo == null) {
-        console.error('No workspace found for token', workspace)
-        continue
-      }
-      const pipeline = await factory(metricsContext, wsInfo, createEmptyBroadcastOps(), null)
-      const client = wrapPipeline(metricsContext, pipeline, wsInfo, false)
-      const integrations = integrationsByWorkspaces[workspace]
-      for (const integration of integrations) {
-        try {
-          const integrationSetting = await getIntegrationSetting(client, integration)
-          if (integrationSetting === undefined) {
-            console.warn('No integration setting found for integration', integration)
-            continue
-          }
-          const integrationData = getIntegrationData(integrationSetting, integration)
-          const updatedData = {
-            ...(integration.data ?? {}),
-            ...integrationData
-          }
-          const updatedIntegration: Integration = {
-            ...integration,
-            data: updatedData
-          }
-          await accountClient.updateIntegration(updatedIntegration)
-          console.log('Updated integration', updatedIntegration.socialId, updatedIntegration.workspaceUuid)
-        } catch (e: any) {
-          console.error('Error updating integration', integration.socialId, integration.workspaceUuid, e)
-        }
-      }
-    }
-    console.log('Integrations migrations done')
-  } catch (e) {
-    console.error('Error migrating integrations', e)
-  }
-}
-
-async function getIntegrationSetting (
-  client: Client,
-  integration: Integration
-): Promise<IntegrationSetting | undefined> {
-  const integrationType = getIntegrationType(integration)
-  if (integrationType === undefined) {
-    return undefined
-  }
-  let integrations = await client.findAll(setting.class.Integration, {
-    createdBy: integration.socialId,
-    type: integrationType
-  })
-  if (integrations.length === 0) {
-    integrations = await client.findAll(setting.class.Integration, {
-      modifiedBy: integration.socialId,
-      type: integrationType
-    })
-  }
-  if (integrations.length === 0) {
-    return undefined
-  }
-  const enabledIntegration = integrations.find((i) => !i.disabled && i.value !== undefined && i.value !== '')
-  if (enabledIntegration !== undefined) {
-    return enabledIntegration
-  }
-  return integrations.find((i) => i.value !== undefined && i.value !== '')
-}
-
-function getIntegrationData (setting: IntegrationSetting, integration: Integration): Record<string, any> | undefined {
-  if (setting.value == null || setting.value === '') {
-    return {}
-  }
-  if (integration.kind === GMAIL_INTEGRATION) {
-    return {
-      email: setting.value
-    }
-  }
-  if (integration.kind === CALENDAR_INTEGRATION) {
-    return {
-      email: setting.value
-    }
-  }
-  if (integration.kind === GITHUB_INTEGRATION) {
-    return {
-      installationId: setting.value
-    }
-  }
-}
-
-function getIntegrationType (integration: Integration): Ref<IntegrationType> | undefined {
-  if (integration.kind === GMAIL_INTEGRATION) {
-    return 'gmail:integrationType:Gmail' as any
-  }
-  if (integration.kind === CALENDAR_INTEGRATION) {
-    return 'calendar:integrationType:Calendar' as any
-  }
-  if (integration.kind === GITHUB_INTEGRATION) {
-    return 'github:integrationType:Github' as any
-  }
-
-  return undefined
-}
@@ -0,0 +1,128 @@
+import { type IntegrationKind, type PersonId, type WorkspaceUuid, systemAccountUuid } from '@hcengineering/core'
+import { getAccountClient } from '@hcengineering/server-client'
+import { generateToken } from '@hcengineering/server-token'
+import { type Integration } from '@hcengineering/account-client'
+import { getDBClient } from '@hcengineering/postgres'
+
+const GITHUB_INTEGRATION: IntegrationKind = 'github' as any
+
+interface IntegrationSetting {
+  workspaceId: WorkspaceUuid
+  createdBy: PersonId
+  installationIds: number[]
+}
+
+export async function restoreGithubIntegrations (dbUrl: string, dryrun: boolean): Promise<void> {
+  try {
+    const pg = getDBClient(dbUrl)
+    const pgClient = await pg.getClient()
+    const token = generateToken(systemAccountUuid, undefined, { service: 'admin', admin: 'true' })
+    const accountClient = getAccountClient(token)
+
+    const integrationSettings = await pgClient<
+    { workspaceId: WorkspaceUuid, createdBy: PersonId, installationId: number }[]
+    >`
+      SELECT "workspaceId", "createdBy", data -> 'installationId' "installationId"
+      FROM github
+      WHERE _class='github:class:GithubIntegration'
+    `
+
+    // Group by workspace and createdBy, collecting installationIds
+    const uniqueSettings = groupIntegrationSettings(integrationSettings)
+    console.info('Start restoring GitHub installation IDs, count: ', uniqueSettings.length)
+
+    let createdCount = 0
+    let updatedCount = 0
+    for (const setting of uniqueSettings) {
+      try {
+        const existingIntegration = await accountClient.getIntegration({
+          workspaceUuid: setting.workspaceId,
+          socialId: setting.createdBy,
+          kind: GITHUB_INTEGRATION
+        })
+
+        // Determine the installationId to use
+        const installationId =
+          setting.installationIds.length === 1 ? setting.installationIds[0] : setting.installationIds // Use array if multiple values
+
+        if (existingIntegration != null) {
+          const existingInstallationId = existingIntegration?.data?.installationId
+          const isSame = Array.isArray(installationId)
+            ? Array.isArray(existingInstallationId) &&
+              installationId.length === existingInstallationId.length &&
+              installationId.every((id) => existingInstallationId.includes(id))
+            : existingInstallationId === installationId
+
+          if (isSame) {
+            if (dryrun) {
+              console.info('Dry run: skip existing integration', existingIntegration)
+            }
+            continue
+          }
+
+          const updatedData = {
+            ...(existingIntegration.data ?? {}),
+            installationId
+          }
+          const updatedIntegration: Integration = {
+            ...existingIntegration,
+            data: updatedData
+          }
+          if (dryrun) {
+            console.info('Dry run: would update integration', existingIntegration, updatedIntegration)
+            continue
+          }
+          await accountClient.updateIntegration(updatedIntegration)
+          updatedCount++
+        } else {
+          const integration: Integration = {
+            workspaceUuid: setting.workspaceId,
+            socialId: setting.createdBy,
+            kind: GITHUB_INTEGRATION,
+            data: {
+              installationId
+            }
+          }
+          if (dryrun) {
+            console.info('Dry run: would create integration', integration)
+            continue
+          }
+          await accountClient.createIntegration(integration)
+          createdCount++
+        }
+      } catch (e: any) {
+        console.error('Error restoring GitHub integration', setting.createdBy, setting.workspaceId, e)
+      }
+    }
+    console.info(`Finished restoring GitHub integrations, updated: ${updatedCount}, created: ${createdCount}`)
+  } catch (e) {
+    console.error('Failed to restore GitHub integrations', e)
+  }
+}
+
+function groupIntegrationSettings (
+  settings: Array<{ workspaceId: WorkspaceUuid, createdBy: PersonId, installationId: number }>
+): IntegrationSetting[] {
+  const groupedSettings = new Map<string, IntegrationSetting>()
+
+  for (const setting of settings) {
+    if (setting.installationId == null) continue
+
+    const key = `${setting.workspaceId}-${setting.createdBy}`
+    const existing = groupedSettings.get(key)
+
+    if (existing != null) {
+      if (!existing.installationIds.includes(setting.installationId)) {
+        existing.installationIds.push(setting.installationId)
+      }
+    } else {
+      groupedSettings.set(key, {
+        workspaceId: setting.workspaceId,
+        createdBy: setting.createdBy,
+        installationIds: [setting.installationId]
+      })
+    }
+  }
+
+  return Array.from(groupedSettings.values())
+}