diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index a4a62692a8..9292b4e01e 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -125,7 +125,7 @@ import { existsSync } from 'fs' import { mkdir, writeFile } from 'fs/promises' import { dirname } from 'path' import { restoreMarkupRefs } from './markup' -import { performIntegrationMigrations } from './integrations' +import { restoreGithubIntegrations } from './restoreGithub' const colorConstants = { colorRed: '\u001b[31m', @@ -2702,13 +2702,12 @@ export function devTool ( }) program - .command('migrate-integrations') - .option('--db ', 'DB name', 'gmail-service') - .option('--region ', 'DB region') - .action(async (cmd: { db: string, region?: string }) => { - const { dbUrl, txes } = prepareTools() + .command('restore-github-integrations') + .option('-d, --dryrun', 'Dry run', false) + .action(async (cmd: { dryrun: boolean }) => { + const { dbUrl } = prepareTools() - await performIntegrationMigrations(dbUrl, cmd.region ?? null, txes) + await restoreGithubIntegrations(dbUrl, cmd.dryrun) }) program diff --git a/dev/tool/src/integrations.ts b/dev/tool/src/integrations.ts deleted file mode 100644 index 557b91e299..0000000000 --- a/dev/tool/src/integrations.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { - type Client, - type IntegrationKind, - MeasureMetricsContext, - type Ref, - type Tx, - type WorkspaceInfoWithStatus, - type WorkspaceUuid, - systemAccountUuid -} from '@hcengineering/core' -import { getAccountClient } from '@hcengineering/server-client' -import { generateToken } from '@hcengineering/server-token' -import { type Integration } from '@hcengineering/account-client' -import setting, { type IntegrationType, type Integration as IntegrationSetting } from '@hcengineering/setting' - -import { type PipelineFactory, createDummyStorageAdapter, wrapPipeline } from '@hcengineering/server-core' -import { createBackupPipeline, createEmptyBroadcastOps } from '@hcengineering/server-pipeline' - -interface WorkspaceInfoProvider { - getWorkspaceInfo: (workspaceUuid: WorkspaceUuid) => Promise -} - -const GMAIL_INTEGRATION: IntegrationKind = 'gmail' as any -const CALENDAR_INTEGRATION: IntegrationKind = 'google-calendar' as any -const GITHUB_INTEGRATION: IntegrationKind = 'github' as any - -export async function performIntegrationMigrations (dbUrl: string, region: string | null, txes: Tx[]): Promise { - console.log('Start integration migrations') - const token = generateToken(systemAccountUuid, undefined, { service: 'admin', admin: 'true' }) - const accountClient = getAccountClient(token) - - const allWorkspaces = await accountClient.listWorkspaces(region) - const byId = new Map(allWorkspaces.map((it) => [it.uuid, it])) - const workspaceProvider: WorkspaceInfoProvider = { - getWorkspaceInfo: async (workspaceUuid: WorkspaceUuid) => { - const ws = byId.get(workspaceUuid as any) - if (ws == null) { - console.error('No workspace found for token', workspaceUuid) - return undefined - } - return ws - } - } - const metricsContext = new MeasureMetricsContext('integrations-migrate', {}) - - const factory: PipelineFactory = createBackupPipeline(metricsContext, dbUrl, txes, { - externalStorage: createDummyStorageAdapter(), - usePassedCtx: true - }) - - await migrateIntegrations(token, workspaceProvider, factory, metricsContext) - - console.log('Finished integration migrations') -} - -async function migrateIntegrations ( - token: string, - workspaceProvider: WorkspaceInfoProvider, - factory: PipelineFactory, - metricsContext: MeasureMetricsContext -): Promise { - try { - console.log('Start integrations data migrations') - const accountClient = getAccountClient(token) - const integrations = await accountClient.listIntegrations({}) - console.log('Integrations count', integrations.length) - - const workspaceUuids = new Set() - const integrationsByWorkspaces = integrations.reduce>((acc, integration) => { - const workspaceId = integration.workspaceUuid - if (workspaceId == null) { - return acc - } - if (acc[workspaceId] === undefined) { - workspaceUuids.add(workspaceId) - acc[workspaceId] = [] - } - acc[workspaceId].push(integration) - return acc - }, {}) - - for (const workspace of workspaceUuids) { - const wsInfo = await workspaceProvider.getWorkspaceInfo(workspace) - if (wsInfo == null) { - console.error('No workspace found for token', workspace) - continue - } - const pipeline = await factory(metricsContext, wsInfo, createEmptyBroadcastOps(), null) - const client = wrapPipeline(metricsContext, pipeline, wsInfo, false) - const integrations = integrationsByWorkspaces[workspace] - for (const integration of integrations) { - try { - const integrationSetting = await getIntegrationSetting(client, integration) - if (integrationSetting === undefined) { - console.warn('No integration setting found for integration', integration) - continue - } - const integrationData = getIntegrationData(integrationSetting, integration) - const updatedData = { - ...(integration.data ?? {}), - ...integrationData - } - const updatedIntegration: Integration = { - ...integration, - data: updatedData - } - await accountClient.updateIntegration(updatedIntegration) - console.log('Updated integration', updatedIntegration.socialId, updatedIntegration.workspaceUuid) - } catch (e: any) { - console.error('Error updating integration', integration.socialId, integration.workspaceUuid, e) - } - } - } - console.log('Integrations migrations done') - } catch (e) { - console.error('Error migrating integrations', e) - } -} - -async function getIntegrationSetting ( - client: Client, - integration: Integration -): Promise { - const integrationType = getIntegrationType(integration) - if (integrationType === undefined) { - return undefined - } - let integrations = await client.findAll(setting.class.Integration, { - createdBy: integration.socialId, - type: integrationType - }) - if (integrations.length === 0) { - integrations = await client.findAll(setting.class.Integration, { - modifiedBy: integration.socialId, - type: integrationType - }) - } - if (integrations.length === 0) { - return undefined - } - const enabledIntegration = integrations.find((i) => !i.disabled && i.value !== undefined && i.value !== '') - if (enabledIntegration !== undefined) { - return enabledIntegration - } - return integrations.find((i) => i.value !== undefined && i.value !== '') -} - -function getIntegrationData (setting: IntegrationSetting, integration: Integration): Record | undefined { - if (setting.value == null || setting.value === '') { - return {} - } - if (integration.kind === GMAIL_INTEGRATION) { - return { - email: setting.value - } - } - if (integration.kind === CALENDAR_INTEGRATION) { - return { - email: setting.value - } - } - if (integration.kind === GITHUB_INTEGRATION) { - return { - installationId: setting.value - } - } -} - -function getIntegrationType (integration: Integration): Ref | undefined { - if (integration.kind === GMAIL_INTEGRATION) { - return 'gmail:integrationType:Gmail' as any - } - if (integration.kind === CALENDAR_INTEGRATION) { - return 'calendar:integrationType:Calendar' as any - } - if (integration.kind === GITHUB_INTEGRATION) { - return 'github:integrationType:Github' as any - } - - return undefined -} diff --git a/dev/tool/src/restoreGithub.ts b/dev/tool/src/restoreGithub.ts new file mode 100644 index 0000000000..eba4af38bd --- /dev/null +++ b/dev/tool/src/restoreGithub.ts @@ -0,0 +1,128 @@ +import { type IntegrationKind, type PersonId, type WorkspaceUuid, systemAccountUuid } from '@hcengineering/core' +import { getAccountClient } from '@hcengineering/server-client' +import { generateToken } from '@hcengineering/server-token' +import { type Integration } from '@hcengineering/account-client' +import { getDBClient } from '@hcengineering/postgres' + +const GITHUB_INTEGRATION: IntegrationKind = 'github' as any + +interface IntegrationSetting { + workspaceId: WorkspaceUuid + createdBy: PersonId + installationIds: number[] +} + +export async function restoreGithubIntegrations (dbUrl: string, dryrun: boolean): Promise { + try { + const pg = getDBClient(dbUrl) + const pgClient = await pg.getClient() + const token = generateToken(systemAccountUuid, undefined, { service: 'admin', admin: 'true' }) + const accountClient = getAccountClient(token) + + const integrationSettings = await pgClient< + { workspaceId: WorkspaceUuid, createdBy: PersonId, installationId: number }[] + >` + SELECT "workspaceId", "createdBy", data -> 'installationId' "installationId" + FROM github + WHERE _class='github:class:GithubIntegration' + ` + + // Group by workspace and createdBy, collecting installationIds + const uniqueSettings = groupIntegrationSettings(integrationSettings) + console.info('Start restoring GitHub installation IDs, count: ', uniqueSettings.length) + + let createdCount = 0 + let updatedCount = 0 + for (const setting of uniqueSettings) { + try { + const existingIntegration = await accountClient.getIntegration({ + workspaceUuid: setting.workspaceId, + socialId: setting.createdBy, + kind: GITHUB_INTEGRATION + }) + + // Determine the installationId to use + const installationId = + setting.installationIds.length === 1 ? setting.installationIds[0] : setting.installationIds // Use array if multiple values + + if (existingIntegration != null) { + const existingInstallationId = existingIntegration?.data?.installationId + const isSame = Array.isArray(installationId) + ? Array.isArray(existingInstallationId) && + installationId.length === existingInstallationId.length && + installationId.every((id) => existingInstallationId.includes(id)) + : existingInstallationId === installationId + + if (isSame) { + if (dryrun) { + console.info('Dry run: skip existing integration', existingIntegration) + } + continue + } + + const updatedData = { + ...(existingIntegration.data ?? {}), + installationId + } + const updatedIntegration: Integration = { + ...existingIntegration, + data: updatedData + } + if (dryrun) { + console.info('Dry run: would update integration', existingIntegration, updatedIntegration) + continue + } + await accountClient.updateIntegration(updatedIntegration) + updatedCount++ + } else { + const integration: Integration = { + workspaceUuid: setting.workspaceId, + socialId: setting.createdBy, + kind: GITHUB_INTEGRATION, + data: { + installationId + } + } + if (dryrun) { + console.info('Dry run: would create integration', integration) + continue + } + await accountClient.createIntegration(integration) + createdCount++ + } + } catch (e: any) { + console.error('Error restoring GitHub integration', setting.createdBy, setting.workspaceId, e) + } + } + console.info(`Finished restoring GitHub integrations, updated: ${updatedCount}, created: ${createdCount}`) + } catch (e) { + console.error('Failed to restore GitHub integrations', e) + } +} + +function groupIntegrationSettings ( + settings: Array<{ workspaceId: WorkspaceUuid, createdBy: PersonId, installationId: number }> +): IntegrationSetting[] { + const groupedSettings = new Map() + + for (const setting of settings) { + if (setting.installationId == null) continue + + const key = `${setting.workspaceId}-${setting.createdBy}` + const existing = groupedSettings.get(key) + + if (existing != null) { + if (!existing.installationIds.includes(setting.installationId)) { + existing.installationIds.push(setting.installationId) + } + } else { + groupedSettings.set(key, { + workspaceId: setting.workspaceId, + createdBy: setting.createdBy, + installationIds: [setting.installationId] + }) + } + } + + return Array.from(groupedSettings.values()) +}