UBERF-13485: Restore v6 from storage tool (#9777)

Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com>
2026-09-09 19:27:44 +02:00 · 2025-09-04 16:16:18 +07:00
parent db9c52a1bd
commit c3902bb9aa
2 changed files with 306 additions and 26 deletions
@@ -55,7 +55,7 @@ import { type MongoClient } from 'mongodb'
 import type postgres from 'postgres'
 import { type Row } from 'postgres'
 import { getToolToken } from './utils'
-import { createFileBackupStorage, restore } from '@hcengineering/server-backup'
+import { type BackupStorage, createFileBackupStorage, restore } from '@hcengineering/server-backup'
 import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
 import { getPlatformQueue } from '@hcengineering/kafka'
 import {
@@ -1167,7 +1167,8 @@ export async function migrateTrustedV6Accounts (
  accountDB: AccountDB,
  mongoDb: v6MongoAccountDB,
  dryRun: boolean,
-  skipWorkspaces: Set<string>
+  skipWorkspaces: Set<string>,
+  conflictSuffix?: string
 ): Promise<void> {
  // Mapping between <ObjectId, UUID>
  const accountsIdToUuid: Record<string, AccountUuid> = {}
@@ -1229,13 +1230,16 @@ export async function migrateTrustedV6Accounts (
      }

      try {
-        const workspaceUuid = await migrateWorkspace(
-          workspace,
-          accountDB,
-          accountsIdToUuid,
-          accountsEmailToUuid,
-          dryRun
-        )
+        const [workspaceUuid] =
+          (await migrateWorkspace(
+            workspace,
+            accountDB,
+            accountsIdToUuid,
+            accountsEmailToUuid,
+            dryRun,
+            'manual-creation',
+            conflictSuffix
+          )) ?? []

        if (workspaceUuid !== undefined) {
          workspacesIdToUuid[workspace.workspace] = workspaceUuid
@@ -1394,8 +1398,12 @@ async function migrateWorkspace (
  accountsIdToUuid: Record<string, AccountUuid>,
  accountsEmailToUuid: Record<string, AccountUuid>,
  dryRun = true,
-  forcedMode?: WorkspaceMode
-): Promise<WorkspaceUuid | undefined> {
+  forcedMode?: WorkspaceMode,
+  conflictSuffix?: string,
+  throwExisting?: boolean,
+  region?: string,
+  branding?: string
+): Promise<[WorkspaceUuid, string] | undefined> {
  if (workspace.workspaceUrl == null) {
    console.log('No workspace url, skipping', workspace.workspace)
    return
@@ -1406,14 +1414,19 @@ async function migrateWorkspace (
    console.log('No account found for workspace', workspace.workspace, 'created by', workspace.createdBy)
  }

-  const existingByUrl = await accountDB.workspace.findOne({ url: workspace.workspaceUrl })
+  let existingByUrl = await accountDB.workspace.findOne({ url: workspace.workspaceUrl })
  const existingByUuid = await accountDB.workspace.findOne({ uuid: workspace.uuid })

  let workspaceUuid: WorkspaceUuid
+  let url = workspace.workspaceUrl

  if (existingByUuid == null) {
-    let url = workspace.workspaceUrl
+    if (existingByUrl != null && (conflictSuffix ?? '') !== '') {
+      url = `${url}-${conflictSuffix}`
+      existingByUrl = await accountDB.workspace.findOne({ url })
+    }
    if (existingByUrl != null) {
+      console.log('Conflicting workspace url', url)
      // generate new url
      url = `${url}-${generateId('-')}`
      console.log('Generating new url', url)
@@ -1424,8 +1437,8 @@ async function migrateWorkspace (
      name: workspace.workspaceName,
      url,
      dataId: workspace.workspace,
-      branding: workspace.branding,
-      region: workspace.region,
+      branding: branding ?? workspace.branding,
+      region: region ?? workspace.region,
      createdBy,
      billingAccount: createdBy,
      createdOn: workspace.createdOn ?? Date.now()
@@ -1438,6 +1451,10 @@ async function migrateWorkspace (
      workspaceUuid = generateUuid() as WorkspaceUuid
    }
  } else {
+    if (throwExisting === true) {
+      throw new Error(`Workspace with the same uuid ${workspace.uuid} already exists`)
+    }
+
    workspaceUuid = existingByUuid.uuid
  }

@@ -1488,7 +1505,7 @@ async function migrateWorkspace (
    }
  }

-  return workspaceUuid
+  return [workspaceUuid, url]
 }

 export async function restoreFromv6All (
@@ -1581,14 +1598,15 @@ export async function restoreFromv6All (

      try {
        // Create active workspaces as archived until they are actually restored
-        const workspaceUuid = await migrateWorkspace(
-          workspace,
-          accountDB,
-          accountsIdToUuid,
-          accountsEmailToUuid,
-          false,
-          isActive ? 'archived' : undefined
-        )
+        const [workspaceUuid] =
+          (await migrateWorkspace(
+            workspace,
+            accountDB,
+            accountsIdToUuid,
+            accountsEmailToUuid,
+            false,
+            isActive ? 'archived' : undefined
+          )) ?? []

        if (workspaceUuid !== undefined) {
          workspacesIdToUuid[workspace.workspace] = workspaceUuid
@@ -1720,3 +1738,170 @@ export async function restoreFromv6All (
    ctx.error('Failed to restore v6 dump', { err })
  }
 }
+
+export async function restoreTrustedV6Workspace (
+  ctx: MeasureMetricsContext,
+  accountDB: AccountDB,
+  workspace: OldWorkspace,
+  accounts: OldAccount[],
+  invites: any[],
+  backupWsStorage: BackupStorage,
+  workspaceStorage: StorageAdapter,
+  txes: Tx[],
+  dbUrl: string,
+  opts?: {
+    conflictSuffix?: string
+    region?: string
+    branding?: string
+    force?: boolean
+  }
+): Promise<void> {
+  const { conflictSuffix, region, branding, force } = opts ?? {}
+  // Mapping between <ObjectId, UUID>
+  const accountsIdToUuid: Record<string, AccountUuid> = {}
+  // Mapping between <email, UUID>
+  const accountsEmailToUuid: Record<string, AccountUuid> = {}
+  let workspaceUuid: WorkspaceUuid | undefined
+  let newWorkspaceUrl: string | undefined
+
+  ctx.info('Restoring workspace accounts...')
+
+  let accountsProcessed = 0
+  for (const account of accounts) {
+    try {
+      const accountUuid = await migrateAccount(account, accountDB, false)
+      if (accountUuid == null) {
+        ctx.warn('Account not restored', account)
+        continue
+      }
+
+      accountsIdToUuid[account._id.toString()] = accountUuid
+      accountsEmailToUuid[account.email] = accountUuid
+
+      accountsProcessed++
+      if (accountsProcessed % 100 === 0) {
+        ctx.info('Processed accounts:', { accountsProcessed })
+      }
+    } catch (err: any) {
+      ctx.error('Failed to restore account', { _id: account._id, email: account.email, err })
+    }
+  }
+
+  ctx.info('Total accounts processed:', { accountsProcessed })
+
+  const oldMode = workspace.mode
+
+  try {
+    // Create workspace with manual-creation mode until it is restored
+    ;[workspaceUuid, newWorkspaceUrl] =
+      (await migrateWorkspace(
+        workspace,
+        accountDB,
+        accountsIdToUuid,
+        accountsEmailToUuid,
+        false,
+        'manual-creation',
+        conflictSuffix,
+        force !== true,
+        region,
+        branding
+      )) ?? []
+
+    if (workspaceUuid === undefined) {
+      ctx.error('Workspace uuid not set', { workspace: workspace.workspace })
+      throw new Error(`Workspace uuid not set ${workspace.workspace}`)
+    }
+
+    if (newWorkspaceUrl == null) {
+      ctx.error('Workspace url not set', { workspace: workspace.workspace })
+      throw new Error(`Workspace url not set ${workspace.workspace}`)
+    }
+
+    let invitesProcessed = 0
+    for (const invite of invites) {
+      try {
+        if (workspace.workspace !== invite.workspace.name) {
+          ctx.error(
+            `Invite workspace ${invite.workspace.name} doesn't match workspace being restored ${workspace.workspace}`
+          )
+          continue
+        }
+
+        const existing = await accountDB.invite.findOne({ migratedFrom: invite._id.toString() })
+        if (existing != null) {
+          continue
+        }
+
+        const inviteRecord = {
+          migratedFrom: invite._id.toString(),
+          workspaceUuid,
+          expiresOn: invite.exp,
+          emailPattern: invite.emailMask,
+          remainingUses: invite.limit,
+          role: invite.role ?? AccountRole.User
+        }
+
+        await accountDB.invite.insertOne(inviteRecord)
+
+        invitesProcessed++
+        if (invitesProcessed % 100 === 0) {
+          ctx.info('Processed invites:', { invitesProcessed })
+        }
+      } catch (err: any) {
+        ctx.error('Failed to restore invite', { _id: invite._id, err })
+      }
+    }
+
+    ctx.info('Total invites processed:', { invitesProcessed })
+
+    const dataId = workspace.workspace
+    const url = newWorkspaceUrl
+    const uuid = workspaceUuid
+
+    const wsIds = {
+      uuid,
+      dataId,
+      url
+    }
+
+    const queue = getPlatformQueue('tool', workspace.region)
+    const wsProducer = queue.getProducer<QueueWorkspaceMessage>(ctx, QueueTopic.Workspace)
+
+    await wsProducer.send(ctx, uuid, [workspaceEvents.restoring()])
+
+    let pipeline: Pipeline | undefined
+    try {
+      pipeline = await createBackupPipeline(ctx, dbUrl, txes, {
+        externalStorage: workspaceStorage,
+        usePassedCtx: true
+      })(ctx, wsIds, createEmptyBroadcastOps(), null)
+      if (pipeline === undefined) {
+        ctx.error('failed to restore, pipeline is undefined', { dataId })
+        return
+      }
+      await sendTransactorEvent(uuid, 'force-maintenance')
+
+      await restore(ctx, pipeline, wsIds, backupWsStorage, {
+        date: -1,
+        merge: false,
+        parallel: 1,
+        recheck: false
+      })
+
+      await sendTransactorEvent(uuid, 'force-close')
+
+      ctx.info('workspace restored', { dataId })
+      await wsProducer.send(ctx, uuid, [workspaceEvents.restored()])
+
+      await accountDB.workspaceStatus.update({ workspaceUuid: uuid }, { mode: oldMode })
+    } catch (err) {
+      ctx.error('failed to restore backup of the workspace', { url, dataId, err })
+    } finally {
+      await pipeline?.close()
+      await queue.shutdown()
+      await workspaceStorage?.close()
+    }
+  } catch (err: any) {
+    ctx.error('Failed to restore workspace', { url: workspace.workspaceUrl, workspace: workspace.workspace, err })
+  }
+}
@@ -51,7 +51,11 @@ import {
 } from '@hcengineering/server-pipeline'
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
 import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
-import { getMongoAccountDB } from '@hcengineering/account-service'
+import {
+  getMongoAccountDB,
+  type Account as OldAccount,
+  type Workspace as OldWorkspace
+} from '@hcengineering/account-service'

 import { faker } from '@faker-js/faker'
 import { getPlatformQueue } from '@hcengineering/kafka'
@@ -113,7 +117,8 @@ import {
  migrateMergedAccounts,
  migrateTrustedV6Accounts,
  moveAccountDbFromMongoToPG,
-  restoreFromv6All
+  restoreFromv6All,
+  restoreTrustedV6Workspace
 } from './db'
 import { performGithubAccountMigrations } from './github'
 import { performGmailAccountMigrations } from './gmail'
@@ -2750,6 +2755,96 @@ export function devTool (
      }, dbUrl)
    })

+  program
+    .command('restore-v6-from-storage <workspace> <accsRoot>')
+    .description('Restore a workspace from v6 backup storage with accounts info')
+    .option('-r, --region <region>', 'Region to restore workspace to')
+    .option('-b, --branding <branding>', 'Branding to restore workspace with', 'huly')
+    .option('-s, --suffix <suffix>', 'Url suffix if conflicting', 'bold')
+    .option('-f, --force', 'Force restore if the same uuid', false)
+    .action(async (workspace, accsRoot, cmd: { suffix: string, region: string, branding: string, force: boolean }) => {
+      const bucketName = process.env.BUCKET_NAME
+      if (bucketName === '' || bucketName == null) {
+        console.error('please provide bucket name env')
+        process.exit(1)
+      }
+
+      const backupStorageConfig = storageConfigFromEnv(process.env.BACKUP_STORAGE)
+      const backupStorageAdapter = createStorageFromConfig(backupStorageConfig.storages[0])
+      const backupIds = { uuid: bucketName as WorkspaceUuid, dataId: bucketName as WorkspaceDataId, url: '' }
+      const backupAccsStorage = await createStorageBackupStorage(toolCtx, backupStorageAdapter, backupIds, accsRoot)
+      const v6AccountsFile = 'account.accounts.json'
+      const v6WorkspacesFile = 'account.workspaces.json'
+      const v6InvitesFile = 'account.invites.json'
+
+      if (!(await backupAccsStorage.exists(v6AccountsFile))) {
+        toolCtx.error('file not present', { file: v6AccountsFile })
+        throw new Error(`${v6AccountsFile} should be present to restore`)
+      }
+      if (!(await backupAccsStorage.exists(v6WorkspacesFile))) {
+        toolCtx.error('file not present', { file: v6WorkspacesFile })
+        throw new Error(`${v6WorkspacesFile} should be present to restore`)
+      }
+      if (!(await backupAccsStorage.exists(v6InvitesFile))) {
+        toolCtx.error('file not present', { file: v6InvitesFile })
+        throw new Error(`${v6InvitesFile} should be present to restore`)
+      }
+
+      const v6Workspaces = JSON.parse((await backupAccsStorage.loadFile(v6WorkspacesFile)).toString()) as OldWorkspace[]
+      const v6Workspace = v6Workspaces.find((it) => it.workspace === workspace)
+
+      if (v6Workspace == null) {
+        toolCtx.error('workspace not found in the accounts backup', { workspace })
+        throw new Error(`workspace ${workspace} not found in the accounts backup`)
+      }
+
+      const uniqueWorkspaceAccounts = new Set((v6Workspace.accounts ?? []).map((it) => it.toString()))
+      const v6AccountsRaw = JSON.parse((await backupAccsStorage.loadFile(v6AccountsFile)).toString()) as any[]
+      const v6WorkspaceAccountsRaw = v6AccountsRaw.filter((acc) => uniqueWorkspaceAccounts.has(acc._id.toString()))
+
+      const v6WorkspaceAccounts: OldAccount[] = []
+      for (const rawAccount of v6WorkspaceAccountsRaw) {
+        const hashTypedArray = rawAccount.hash != null ? new Uint8Array(rawAccount.hash.data) : null
+        const saltTypedArray = new Uint8Array(rawAccount.salt.data)
+
+        v6WorkspaceAccounts.push({
+          ...rawAccount,
+          hash: hashTypedArray != null ? Buffer.from(hashTypedArray.buffer) : null,
+          salt: Buffer.from(saltTypedArray.buffer)
+        })
+      }
+
+      let v6Invites = JSON.parse((await backupAccsStorage.loadFile(v6InvitesFile)).toString()) as any[]
+      v6Invites = v6Invites.filter((invite: any) => invite.workspace.name === v6Workspace.workspace)
+
+      const { txes, dbUrl } = prepareTools()
+      const backupWsStorage = await createStorageBackupStorage(
+        toolCtx,
+        backupStorageAdapter,
+        backupIds,
+        v6Workspace.uuid ?? v6Workspace.workspace
+      )
+
+      const storageConfig = storageConfigFromEnv()
+      const workspaceStorage: StorageAdapter = buildStorageFromConfig(storageConfig)
+      const { suffix, region, branding, force } = cmd
+
+      await withAccountDatabase(async (pgDb) => {
+        await restoreTrustedV6Workspace(
+          toolCtx,
+          pgDb,
+          v6Workspace,
+          v6WorkspaceAccounts,
+          v6Invites,
+          backupWsStorage,
+          workspaceStorage,
+          txes,
+          dbUrl,
+          { conflictSuffix: suffix, region, branding, force }
+        )
+      }, dbUrl)
+    })
+
  extendProgram?.(program)

  process.on('unhandledRejection', (reason, promise) => {