From c3902bb9aa021e37ffd7c0331da29ce0662a6238 Mon Sep 17 00:00:00 2001 From: Alexey Zinoviev Date: Thu, 4 Sep 2025 16:16:18 +0700 Subject: [PATCH] UBERF-13485: Restore v6 from storage tool (#9777) Signed-off-by: Alexey Zinoviev --- dev/tool/src/db.ts | 233 +++++++++++++++++++++++++++++++++++++----- dev/tool/src/index.ts | 99 +++++++++++++++++- 2 files changed, 306 insertions(+), 26 deletions(-) diff --git a/dev/tool/src/db.ts b/dev/tool/src/db.ts index 6f78a575c1..a32c875672 100644 --- a/dev/tool/src/db.ts +++ b/dev/tool/src/db.ts @@ -55,7 +55,7 @@ import { type MongoClient } from 'mongodb' import type postgres from 'postgres' import { type Row } from 'postgres' import { getToolToken } from './utils' -import { createFileBackupStorage, restore } from '@hcengineering/server-backup' +import { type BackupStorage, createFileBackupStorage, restore } from '@hcengineering/server-backup' import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { getPlatformQueue } from '@hcengineering/kafka' import { @@ -1167,7 +1167,8 @@ export async function migrateTrustedV6Accounts ( accountDB: AccountDB, mongoDb: v6MongoAccountDB, dryRun: boolean, - skipWorkspaces: Set + skipWorkspaces: Set, + conflictSuffix?: string ): Promise { // Mapping between const accountsIdToUuid: Record = {} @@ -1229,13 +1230,16 @@ export async function migrateTrustedV6Accounts ( } try { - const workspaceUuid = await migrateWorkspace( - workspace, - accountDB, - accountsIdToUuid, - accountsEmailToUuid, - dryRun - ) + const [workspaceUuid] = + (await migrateWorkspace( + workspace, + accountDB, + accountsIdToUuid, + accountsEmailToUuid, + dryRun, + 'manual-creation', + conflictSuffix + )) ?? [] if (workspaceUuid !== undefined) { workspacesIdToUuid[workspace.workspace] = workspaceUuid @@ -1394,8 +1398,12 @@ async function migrateWorkspace ( accountsIdToUuid: Record, accountsEmailToUuid: Record, dryRun = true, - forcedMode?: WorkspaceMode -): Promise { + forcedMode?: WorkspaceMode, + conflictSuffix?: string, + throwExisting?: boolean, + region?: string, + branding?: string +): Promise<[WorkspaceUuid, string] | undefined> { if (workspace.workspaceUrl == null) { console.log('No workspace url, skipping', workspace.workspace) return @@ -1406,14 +1414,19 @@ async function migrateWorkspace ( console.log('No account found for workspace', workspace.workspace, 'created by', workspace.createdBy) } - const existingByUrl = await accountDB.workspace.findOne({ url: workspace.workspaceUrl }) + let existingByUrl = await accountDB.workspace.findOne({ url: workspace.workspaceUrl }) const existingByUuid = await accountDB.workspace.findOne({ uuid: workspace.uuid }) let workspaceUuid: WorkspaceUuid + let url = workspace.workspaceUrl if (existingByUuid == null) { - let url = workspace.workspaceUrl + if (existingByUrl != null && (conflictSuffix ?? '') !== '') { + url = `${url}-${conflictSuffix}` + existingByUrl = await accountDB.workspace.findOne({ url }) + } if (existingByUrl != null) { + console.log('Conflicting workspace url', url) // generate new url url = `${url}-${generateId('-')}` console.log('Generating new url', url) @@ -1424,8 +1437,8 @@ async function migrateWorkspace ( name: workspace.workspaceName, url, dataId: workspace.workspace, - branding: workspace.branding, - region: workspace.region, + branding: branding ?? workspace.branding, + region: region ?? workspace.region, createdBy, billingAccount: createdBy, createdOn: workspace.createdOn ?? Date.now() @@ -1438,6 +1451,10 @@ async function migrateWorkspace ( workspaceUuid = generateUuid() as WorkspaceUuid } } else { + if (throwExisting === true) { + throw new Error(`Workspace with the same uuid ${workspace.uuid} already exists`) + } + workspaceUuid = existingByUuid.uuid } @@ -1488,7 +1505,7 @@ async function migrateWorkspace ( } } - return workspaceUuid + return [workspaceUuid, url] } export async function restoreFromv6All ( @@ -1581,14 +1598,15 @@ export async function restoreFromv6All ( try { // Create active workspaces as archived until they are actually restored - const workspaceUuid = await migrateWorkspace( - workspace, - accountDB, - accountsIdToUuid, - accountsEmailToUuid, - false, - isActive ? 'archived' : undefined - ) + const [workspaceUuid] = + (await migrateWorkspace( + workspace, + accountDB, + accountsIdToUuid, + accountsEmailToUuid, + false, + isActive ? 'archived' : undefined + )) ?? [] if (workspaceUuid !== undefined) { workspacesIdToUuid[workspace.workspace] = workspaceUuid @@ -1720,3 +1738,170 @@ export async function restoreFromv6All ( ctx.error('Failed to restore v6 dump', { err }) } } + +export async function restoreTrustedV6Workspace ( + ctx: MeasureMetricsContext, + accountDB: AccountDB, + workspace: OldWorkspace, + accounts: OldAccount[], + invites: any[], + backupWsStorage: BackupStorage, + workspaceStorage: StorageAdapter, + txes: Tx[], + dbUrl: string, + opts?: { + conflictSuffix?: string + region?: string + branding?: string + force?: boolean + } +): Promise { + const { conflictSuffix, region, branding, force } = opts ?? {} + // Mapping between + const accountsIdToUuid: Record = {} + // Mapping between + const accountsEmailToUuid: Record = {} + let workspaceUuid: WorkspaceUuid | undefined + let newWorkspaceUrl: string | undefined + + ctx.info('Restoring workspace accounts...') + + let accountsProcessed = 0 + for (const account of accounts) { + try { + const accountUuid = await migrateAccount(account, accountDB, false) + if (accountUuid == null) { + ctx.warn('Account not restored', account) + continue + } + + accountsIdToUuid[account._id.toString()] = accountUuid + accountsEmailToUuid[account.email] = accountUuid + + accountsProcessed++ + if (accountsProcessed % 100 === 0) { + ctx.info('Processed accounts:', { accountsProcessed }) + } + } catch (err: any) { + ctx.error('Failed to restore account', { _id: account._id, email: account.email, err }) + } + } + + ctx.info('Total accounts processed:', { accountsProcessed }) + + const oldMode = workspace.mode + + try { + // Create workspace with manual-creation mode until it is restored + ;[workspaceUuid, newWorkspaceUrl] = + (await migrateWorkspace( + workspace, + accountDB, + accountsIdToUuid, + accountsEmailToUuid, + false, + 'manual-creation', + conflictSuffix, + force !== true, + region, + branding + )) ?? [] + + if (workspaceUuid === undefined) { + ctx.error('Workspace uuid not set', { workspace: workspace.workspace }) + throw new Error(`Workspace uuid not set ${workspace.workspace}`) + } + + if (newWorkspaceUrl == null) { + ctx.error('Workspace url not set', { workspace: workspace.workspace }) + throw new Error(`Workspace url not set ${workspace.workspace}`) + } + + let invitesProcessed = 0 + for (const invite of invites) { + try { + if (workspace.workspace !== invite.workspace.name) { + ctx.error( + `Invite workspace ${invite.workspace.name} doesn't match workspace being restored ${workspace.workspace}` + ) + continue + } + + const existing = await accountDB.invite.findOne({ migratedFrom: invite._id.toString() }) + if (existing != null) { + continue + } + + const inviteRecord = { + migratedFrom: invite._id.toString(), + workspaceUuid, + expiresOn: invite.exp, + emailPattern: invite.emailMask, + remainingUses: invite.limit, + role: invite.role ?? AccountRole.User + } + + await accountDB.invite.insertOne(inviteRecord) + + invitesProcessed++ + if (invitesProcessed % 100 === 0) { + ctx.info('Processed invites:', { invitesProcessed }) + } + } catch (err: any) { + ctx.error('Failed to restore invite', { _id: invite._id, err }) + } + } + + ctx.info('Total invites processed:', { invitesProcessed }) + + const dataId = workspace.workspace + const url = newWorkspaceUrl + const uuid = workspaceUuid + + const wsIds = { + uuid, + dataId, + url + } + + const queue = getPlatformQueue('tool', workspace.region) + const wsProducer = queue.getProducer(ctx, QueueTopic.Workspace) + + await wsProducer.send(ctx, uuid, [workspaceEvents.restoring()]) + + let pipeline: Pipeline | undefined + try { + pipeline = await createBackupPipeline(ctx, dbUrl, txes, { + externalStorage: workspaceStorage, + usePassedCtx: true + })(ctx, wsIds, createEmptyBroadcastOps(), null) + if (pipeline === undefined) { + ctx.error('failed to restore, pipeline is undefined', { dataId }) + return + } + await sendTransactorEvent(uuid, 'force-maintenance') + + await restore(ctx, pipeline, wsIds, backupWsStorage, { + date: -1, + merge: false, + parallel: 1, + recheck: false + }) + + await sendTransactorEvent(uuid, 'force-close') + + ctx.info('workspace restored', { dataId }) + await wsProducer.send(ctx, uuid, [workspaceEvents.restored()]) + + await accountDB.workspaceStatus.update({ workspaceUuid: uuid }, { mode: oldMode }) + } catch (err) { + ctx.error('failed to restore backup of the workspace', { url, dataId, err }) + } finally { + await pipeline?.close() + await queue.shutdown() + await workspaceStorage?.close() + } + } catch (err: any) { + ctx.error('Failed to restore workspace', { url: workspace.workspaceUrl, workspace: workspace.workspace, err }) + } +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 66bb096b89..1126d9886c 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -51,7 +51,11 @@ import { } from '@hcengineering/server-pipeline' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' -import { getMongoAccountDB } from '@hcengineering/account-service' +import { + getMongoAccountDB, + type Account as OldAccount, + type Workspace as OldWorkspace +} from '@hcengineering/account-service' import { faker } from '@faker-js/faker' import { getPlatformQueue } from '@hcengineering/kafka' @@ -113,7 +117,8 @@ import { migrateMergedAccounts, migrateTrustedV6Accounts, moveAccountDbFromMongoToPG, - restoreFromv6All + restoreFromv6All, + restoreTrustedV6Workspace } from './db' import { performGithubAccountMigrations } from './github' import { performGmailAccountMigrations } from './gmail' @@ -2750,6 +2755,96 @@ export function devTool ( }, dbUrl) }) + program + .command('restore-v6-from-storage ') + .description('Restore a workspace from v6 backup storage with accounts info') + .option('-r, --region ', 'Region to restore workspace to') + .option('-b, --branding ', 'Branding to restore workspace with', 'huly') + .option('-s, --suffix ', 'Url suffix if conflicting', 'bold') + .option('-f, --force', 'Force restore if the same uuid', false) + .action(async (workspace, accsRoot, cmd: { suffix: string, region: string, branding: string, force: boolean }) => { + const bucketName = process.env.BUCKET_NAME + if (bucketName === '' || bucketName == null) { + console.error('please provide bucket name env') + process.exit(1) + } + + const backupStorageConfig = storageConfigFromEnv(process.env.BACKUP_STORAGE) + const backupStorageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) + const backupIds = { uuid: bucketName as WorkspaceUuid, dataId: bucketName as WorkspaceDataId, url: '' } + const backupAccsStorage = await createStorageBackupStorage(toolCtx, backupStorageAdapter, backupIds, accsRoot) + const v6AccountsFile = 'account.accounts.json' + const v6WorkspacesFile = 'account.workspaces.json' + const v6InvitesFile = 'account.invites.json' + + if (!(await backupAccsStorage.exists(v6AccountsFile))) { + toolCtx.error('file not present', { file: v6AccountsFile }) + throw new Error(`${v6AccountsFile} should be present to restore`) + } + if (!(await backupAccsStorage.exists(v6WorkspacesFile))) { + toolCtx.error('file not present', { file: v6WorkspacesFile }) + throw new Error(`${v6WorkspacesFile} should be present to restore`) + } + if (!(await backupAccsStorage.exists(v6InvitesFile))) { + toolCtx.error('file not present', { file: v6InvitesFile }) + throw new Error(`${v6InvitesFile} should be present to restore`) + } + + const v6Workspaces = JSON.parse((await backupAccsStorage.loadFile(v6WorkspacesFile)).toString()) as OldWorkspace[] + const v6Workspace = v6Workspaces.find((it) => it.workspace === workspace) + + if (v6Workspace == null) { + toolCtx.error('workspace not found in the accounts backup', { workspace }) + throw new Error(`workspace ${workspace} not found in the accounts backup`) + } + + const uniqueWorkspaceAccounts = new Set((v6Workspace.accounts ?? []).map((it) => it.toString())) + const v6AccountsRaw = JSON.parse((await backupAccsStorage.loadFile(v6AccountsFile)).toString()) as any[] + const v6WorkspaceAccountsRaw = v6AccountsRaw.filter((acc) => uniqueWorkspaceAccounts.has(acc._id.toString())) + + const v6WorkspaceAccounts: OldAccount[] = [] + for (const rawAccount of v6WorkspaceAccountsRaw) { + const hashTypedArray = rawAccount.hash != null ? new Uint8Array(rawAccount.hash.data) : null + const saltTypedArray = new Uint8Array(rawAccount.salt.data) + + v6WorkspaceAccounts.push({ + ...rawAccount, + hash: hashTypedArray != null ? Buffer.from(hashTypedArray.buffer) : null, + salt: Buffer.from(saltTypedArray.buffer) + }) + } + + let v6Invites = JSON.parse((await backupAccsStorage.loadFile(v6InvitesFile)).toString()) as any[] + v6Invites = v6Invites.filter((invite: any) => invite.workspace.name === v6Workspace.workspace) + + const { txes, dbUrl } = prepareTools() + const backupWsStorage = await createStorageBackupStorage( + toolCtx, + backupStorageAdapter, + backupIds, + v6Workspace.uuid ?? v6Workspace.workspace + ) + + const storageConfig = storageConfigFromEnv() + const workspaceStorage: StorageAdapter = buildStorageFromConfig(storageConfig) + const { suffix, region, branding, force } = cmd + + await withAccountDatabase(async (pgDb) => { + await restoreTrustedV6Workspace( + toolCtx, + pgDb, + v6Workspace, + v6WorkspaceAccounts, + v6Invites, + backupWsStorage, + workspaceStorage, + txes, + dbUrl, + { conflictSuffix: suffix, region, branding, force } + ) + }, dbUrl) + }) + extendProgram?.(program) process.on('unhandledRejection', (reason, promise) => {