qfix: backups using pipeline (#9396)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

.vscode/launch.json

@@ -43,7 +43,7 @@
         "APM_SERVER_URL2": "http://localhost:8200",
         "METRICS_CONSOLE": "false",
         "METRICS_FILE": "${workspaceRoot}/metrics.txt", // Show metrics in console evert 30 seconds.,
-        "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
+        "STORAGE_CONFIG": "datalake|http://huly.local:4030",
         "SERVER_SECRET": "secret",
         "ENABLE_CONSOLE": "true",
         "COLLABORATOR_URL": "ws://localhost:3078",
@@ -83,11 +83,11 @@
         // "DB_URL": "postgresql://postgres:example@localhost:5432",
         "DB_URL": "postgresql://root@localhost:26257/defaultdb?sslmode=disable",
         // "GREEN_URL": "http://huly.local:6767?token=secret",
-        "SERVER_PORT": "3332",
+        "SERVER_PORT": "3334",
         "APM_SERVER_URL2": "http://localhost:8200",
         "METRICS_CONSOLE": "false",
         "METRICS_FILE": "${workspaceRoot}/metrics.txt", // Show metrics in console evert 30 seconds.,
-        "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
+        "STORAGE_CONFIG": "datalake|http://huly.local:4030",
         "SERVER_SECRET": "secret",
         "ENABLE_CONSOLE": "true",
         "COLLABORATOR_URL": "ws://localhost:3078",
@@ -126,7 +126,7 @@
         // "DB_URL": "postgresql://postgres:example@localhost:5432",
         "DB_URL": "postgresql://root@huly.local:26258/defaultdb?sslmode=disable",
         // "GREEN_URL": "http://huly.local:6767?token=secret",
-        "SERVER_PORT": "3335",
+        "SERVER_PORT": "3334",
         "METRICS_CONSOLE": "false",
         "DEBUG_PRINT_SQL": "true",
         "METRICS_FILE": "${workspaceRoot}/metrics.txt", // Show metrics in console evert 30 seconds.,
@@ -406,19 +406,30 @@
       "name": "Debug backup tool",
       "type": "node",
       "request": "launch",
-      "args": ["src/__start.ts", "backup", "../../../hardware/dump/felix", "felix", "--full"],
+      "args": [
+        "src/__start.ts",
+        "backup-restore",
+        "/Users/haiodo/Develop/private/platform/tests/sanity-ws",
+        "sanity-ws",
+        "--upgrade"
+      ],
       "env": {
-        "MINIO_ACCESS_KEY": "minioadmin",
-        "MINIO_SECRET_KEY": "minioadmin",
-        "MINIO_ENDPOINT": "localhost",
-        "SERVER_SECRET": "secret",
+        "MODEL_VERSION": "0.7.153",
+        "STORAGE_CONFIG": "datalake|http://huly.local:4030",
+        "ACCOUNTS_URL": "http://localhost:3003",
+        "TRANSACTOR_URL": "ws://localhost:3334",
+        "ACCOUNT_DB_URL": "postgresql://root@localhost:26258/defaultdb?sslmode=disable",
         "MONGO_URL": "mongodb://localhost:27018",
-        "ACCOUNTS_URL": "http://localhost:3003"
+        "ELASTIC_URL": "http://localhost:9201",
+        "SERVER_SECRET": "secret",
+        "DB_URL": "postgresql://root@localhost:26258/defaultdb?sslmode=disable",
+        "QUEUE_CONFIG": "localhost:19093;-staging"
       },
-      "smartStep": true,
-      "sourceMapRenames": true,
+      "smartStep": false,
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
       "sourceMaps": true,
+      "sourceMapRenames": false,
+      "skipFiles": ["<node_internals>/**"],
       "cwd": "${workspaceRoot}/dev/tool",
       "protocol": "inspector",
       "outputCapture": "std",

dev/docker-compose.yaml

@@ -462,30 +462,30 @@ services:
       - HULY_TOKEN_SECRET=secret
       - HULY_ACCOUNT_SERVICE=http://huly.local:3000
     restart: unless-stopped
-  # backup-cockroach:
-  #   image: hardcoreeng/backup
-  #   extra_hosts:
-  #     - 'huly.local:host-gateway'
-  #   depends_on:
-  #     minio:
-  #       condition: service_healthy
-  #     cockroach:
-  #       condition: service_started
-  #     stats:
-  #       condition: service_started
-  #     account:
-  #       condition: service_started
-  #   environment:
-  #     - SECRET=secret
-  #     - ACCOUNTS_URL=http://huly.local:3000
-  #     - STATS_URL=http://huly.local:4900
-  #     - DB_URL=${DB_CR_URL}
-  #     - REGION=cockroach
-  #     - BUCKET_NAME=backups
-  #     - INTERVAL=3600
-  #     - STORAGE=minio|minio?accessKey=minioadmin&secretKey=minioadmin
-  #     - WORKSPACE_STORAGE=minio|minio?accessKey=minioadmin&secretKey=minioadmin
-  #   restart: unless-stopped
+  backup-cockroach:
+    image: hardcoreeng/backup
+    extra_hosts:
+      - 'huly.local:host-gateway'
+    depends_on:
+      minio:
+        condition: service_healthy
+      cockroach:
+        condition: service_started
+      stats:
+        condition: service_started
+      account:
+        condition: service_started
+    environment:
+      - SECRET=secret
+      - ACCOUNTS_URL=http://huly.local:3000
+      - STATS_URL=http://huly.local:4900
+      - DB_URL=${DB_CR_URL}
+      - REGION=cockroach
+      - BUCKET_NAME=backups
+      - INTERVAL=60
+      - STORAGE=minio|minio?accessKey=minioadmin&secretKey=minioadmin
+      - WORKSPACE_STORAGE=${STORAGE_CONFIG}
+    restart: unless-stopped
   # backup-api:
   #   image: hardcoreeng/backup-api
   #   extra_hosts:

dev/tool/package.json

@@ -21,7 +21,7 @@
     "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging",
     "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool",
     "run-local-mongo": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4700 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:19092' node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js",
-    "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 STORAGE_CONFIG='datalake|http://localhost:4030' ACCOUNT_DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable  TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 REGION_INFO='cockroach|CockroachDB' MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:19092' node --expose-gc --max-old-space-size=18000 $TOOL_OPT ./bundle/bundle.js",
+    "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 STORAGE_CONFIG='datalake|http://huly.local:4030' ACCOUNT_DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable  TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 REGION_INFO='cockroach|CockroachDB' MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:19092' node --expose-gc --max-old-space-size=18000 $TOOL_OPT ./bundle/bundle.js",
     "run-local-brk": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js",
     "run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js",
     "upgrade-mongo": "rushx run-local-mongo upgrade-workspace -- $1",

dev/tool/src/index.ts

@@ -35,10 +35,13 @@ import {
   compactBackup,
   createFileBackupStorage,
   createStorageBackupStorage,
-  restore
+  restore,
+  backupDownload
 } from '@hcengineering/server-backup'
 import serverClientPlugin, { getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client'
 import {
+  createBackupPipeline,
+  createEmptyBroadcastOps,
   registerAdapterFactory,
   registerDestroyFactory,
   registerServerPlugins,
@@ -82,7 +85,6 @@ import {
   getMongoClient,
   shutdownMongo
 } from '@hcengineering/mongo'
-import { backupDownload } from '@hcengineering/server-backup/src/backup'
 
 import { getModelVersion } from '@hcengineering/model-all'
 import {
@@ -94,6 +96,7 @@ import {
 import {
   QueueTopic,
   workspaceEvents,
+  type Pipeline,
   type QueueWorkspaceMessage,
   type StorageAdapter
 } from '@hcengineering/server-core'
@@ -114,6 +117,7 @@ import { performGmailAccountMigrations } from './gmail'
 import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
 
 import { createRestClient } from '@hcengineering/api-client'
+import { sendTransactorEvent } from '@hcengineering/server-tool'
 import { existsSync } from 'fs'
 import { mkdir, writeFile } from 'fs/promises'
 import { dirname } from 'path'
@@ -688,72 +692,6 @@ export function devTool (
   //                       )
   //                     }
 
-  //                     const destroyer = getWorkspaceDestroyAdapter(dbUrl)
-
-  // program
-  //   .command('restore-all')
-  //   .description('Restore workspaces to selected region DB...')
-  //   .option('-t|--timeout [timeout]', 'Timeout in days', '60')
-  //   .option('-r|--region [region]', 'Timeout in days', '')
-  //   .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '')
-  //   .option('-d|--dry [dry]', 'Dry run', false)
-  //   .action(async (cmd: { timeout: string, workspace: string, region: string, dry: boolean, account: string }) => {
-  //     const { txes, dbUrl } = prepareTools()
-
-  //     const bucketName = process.env.BUCKET_NAME
-  //     if (bucketName === '' || bucketName == null) {
-  //       console.error('please provide butket name env')
-  //       process.exit(1)
-  //     }
-
-  //     const token = generateToken(systemAccountEmail, getWorkspaceId(''))
-  //     const workspaces = (await listAccountWorkspaces(token, cmd.region))
-  //       .sort((a, b) => {
-  //         const bsize = b.backupInfo?.backupSize ?? 0
-  //         const asize = a.backupInfo?.backupSize ?? 0
-  //         return bsize - asize
-  //       })
-  //       .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace)
-
-  //     for (const ws of workspaces) {
-  //       const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24)
-
-  //       toolCtx.warn('--- restoring workspace', {
-  //         url: ws.workspaceUrl,
-  //         id: ws.workspace,
-  //         lastVisitDays,
-  //         backupSize: ws.backupInfo?.blobsSize ?? 0,
-  //         mode: ws.mode
-  //       })
-  //       if (cmd.dry) {
-  //         continue
-  //       }
-  //       try {
-  //         const st = Date.now()
-  //         await backupRestore(
-  //           toolCtx,
-  //           dbUrl,
-  //           bucketName,
-  //           ws,
-  //           (dbUrl, storageAdapter) => {
-  //             const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, {
-  //               externalStorage: storageAdapter,
-  //               usePassedCtx: true
-  //             })
-  //             return factory
-  //           },
-  //           [DOMAIN_BLOB]
-  //         )
-  //         const ed = Date.now()
-  //         toolCtx.warn('--- restoring complete', {
-  //           time: ed - st
-  //         })
-  //       } catch (err: any) {
-  //         toolCtx.error('REstore of f workspace failedarchive workspace', { workspace: ws.workspace })
-  //       }
-  //     }
-  //   })
-
   // program
   //   .command('backup-all')
   //   .description('Backup all workspaces...')
@@ -1004,6 +942,7 @@ export function devTool (
       ) => {
         const storage = await createFileBackupStorage(dirName)
         await withAccountDatabase(async (db) => {
+          const { txes, dbUrl } = prepareTools()
           const ws = await getWorkspace(db, workspace)
           if (ws === null) {
             throw new Error(`workspace ${workspace} not found`)
@@ -1013,21 +952,51 @@ export function devTool (
             dataId: ws.dataId,
             url: ws.url
           }
-          const endpoint = await getWorkspaceTransactorEndpoint(ws.uuid)
+          const storageConfig = storageConfigFromEnv()
 
-          await backup(toolCtx, endpoint, wsIds, storage, {
-            force: cmd.force,
-            include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())),
-            skipDomains: (cmd.skip ?? '').split(';').map((it) => it.trim()),
-            timeout: 0,
-            connectTimeout: parseInt(cmd.timeout) * 1000,
-            blobDownloadLimit: parseInt(cmd.blobLimit),
-            skipBlobContentTypes: cmd.contentTypes
-              .split(';')
-              .map((it) => it.trim())
-              .filter((it) => it.length > 0),
-            keepSnapshots: parseInt(cmd.keepSnapshots)
-          })
+          const workspaceStorage: StorageAdapter = buildStorageFromConfig(storageConfig)
+
+          let pipeline: Pipeline | undefined
+          try {
+            pipeline = await createBackupPipeline(toolCtx, dbUrl, txes, {
+              externalStorage: workspaceStorage,
+              usePassedCtx: true
+            })(
+              toolCtx,
+              {
+                uuid: ws.uuid,
+                url: ws.url ?? '',
+                dataId: ws.dataId
+              },
+              createEmptyBroadcastOps(),
+              null
+            )
+            if (pipeline === undefined) {
+              toolCtx.error('failed to restore, pipeline is undefined', { workspace })
+              return
+            }
+
+            await backup(toolCtx, pipeline, wsIds, storage, {
+              force: cmd.force,
+              include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())),
+              skipDomains: (cmd.skip ?? '').split(';').map((it) => it.trim()),
+              timeout: 0,
+              connectTimeout: parseInt(cmd.timeout) * 1000,
+              blobDownloadLimit: parseInt(cmd.blobLimit),
+              skipBlobContentTypes: cmd.contentTypes
+                .split(';')
+                .map((it) => it.trim())
+                .filter((it) => it.length > 0),
+              keepSnapshots: parseInt(cmd.keepSnapshots)
+            })
+          } catch (err: any) {
+            toolCtx.error('Failed to backup workspace', { err, workspace })
+          } finally {
+            if (pipeline !== undefined) {
+              await pipeline.close()
+            }
+            await workspaceStorage.close()
+          }
         })
       }
     )
@@ -1145,7 +1114,6 @@ export function devTool (
     .option('-c, --recheck', 'Force hash recheck on server', false)
     .option('-i, --include <include>', 'A list of ; separated domain names to include during backup', '*')
     .option('-s, --skip <skip>', 'A list of ; separated domain names to skip during backup', '')
-    .option('--use-storage <useStorage>', 'Use workspace storage adapter from env variable', '')
     .option('--upgrade', 'Upgrade workspace', false)
     .option(
       '--history-file <historyFile>',
@@ -1170,6 +1138,7 @@ export function devTool (
         }
       ) => {
         await withAccountDatabase(async (db) => {
+          const { txes, dbUrl } = prepareTools()
           const ws = await getWorkspace(db, workspaceId)
           if (ws === null) {
             throw new Error(`workspace ${workspaceId} not found`)
@@ -1182,32 +1151,58 @@ export function devTool (
             url: ws.url
           }
           const storage = await createFileBackupStorage(dirName)
-          const storageConfig = cmd.useStorage !== '' ? storageConfigFromEnv(process.env[cmd.useStorage]) : undefined
+          const storageConfig = storageConfigFromEnv()
 
           const queue = getPlatformQueue('tool', ws.region)
           const wsProducer = queue.getProducer<QueueWorkspaceMessage>(toolCtx, QueueTopic.Workspace)
 
           await wsProducer.send(ws.uuid, [workspaceEvents.restoring()])
 
-          const workspaceStorage: StorageAdapter | undefined =
-            storageConfig !== undefined ? buildStorageFromConfig(storageConfig) : undefined
-          await restore(toolCtx, await getWorkspaceTransactorEndpoint(workspace), wsIds, storage, {
-            date: parseInt(date ?? '-1'),
-            merge: cmd.merge,
-            parallel: parseInt(cmd.parallel ?? '1'),
-            recheck: cmd.recheck,
-            include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';')),
-            skip: new Set(cmd.skip.split(';')),
-            storageAdapter: workspaceStorage,
-            historyFile: cmd.historyFile
-          })
+          const workspaceStorage: StorageAdapter = buildStorageFromConfig(storageConfig)
 
-          if (cmd.upgrade) {
-            await doUpgrade(toolCtx, workspace, true, true)
+          let pipeline: Pipeline | undefined
+          try {
+            pipeline = await createBackupPipeline(toolCtx, dbUrl, txes, {
+              externalStorage: workspaceStorage,
+              usePassedCtx: true
+            })(
+              toolCtx,
+              {
+                uuid: ws.uuid,
+                url: ws.url ?? '',
+                dataId: ws.dataId
+              },
+              createEmptyBroadcastOps(),
+              null
+            )
+            if (pipeline === undefined) {
+              toolCtx.error('failed to restore, pipeline is undefined', { workspaceId })
+              return
+            }
+            await sendTransactorEvent(workspace, 'force-maintenance')
+
+            await restore(toolCtx, pipeline, wsIds, storage, {
+              date: parseInt(date ?? '-1'),
+              merge: cmd.merge,
+              parallel: parseInt(cmd.parallel ?? '1'),
+              recheck: cmd.recheck,
+              include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';')),
+              skip: new Set(cmd.skip.split(';')),
+              historyFile: cmd.historyFile
+            })
+
+            if (cmd.upgrade) {
+              await doUpgrade(toolCtx, workspace, true, true)
+            } else {
+              await sendTransactorEvent(workspace, 'force-close')
+            }
+
+            console.log('workspace restored')
+            await wsProducer.send(ws.uuid, [workspaceEvents.restored()])
+          } catch (err) {
+            toolCtx.error('failed to restore', { err })
           }
-
-          console.log('workspace restored')
-          await wsProducer.send(ws.uuid, [workspaceEvents.restored()])
+          await pipeline?.close()
           await queue.shutdown()
           await workspaceStorage?.close()
         })

dev/tool/src/utils.ts

@@ -1,7 +1,5 @@
 import { getWorkspaceById, getWorkspaceByUrl, type AccountDB, type Workspace } from '@hcengineering/account'
 import {
-  systemAccountUuid,
-  type WorkspaceUuid,
   type AttachedData,
   type AttachedDoc,
   type Class,
@@ -9,10 +7,9 @@ import {
   type DocumentUpdate,
   type Ref,
   type Space,
-  type TxOperations
+  type TxOperations,
+  type WorkspaceUuid
 } from '@hcengineering/core'
-import { getTransactorEndpoint } from '@hcengineering/server-client'
-import { generateToken } from '@hcengineering/server-token'
 
 export async function findOrUpdateAttached<T extends AttachedDoc> (
   client: TxOperations,
@@ -61,13 +58,4 @@ export async function getWorkspace (db: AccountDB, workspace: string): Promise<W
   return wsObj
 }
 
-export function getToolToken (workspace?: WorkspaceUuid): string {
-  return generateToken(systemAccountUuid, workspace, { service: 'tool' })
-}
-
-export async function getWorkspaceTransactorEndpoint (
-  workspace: WorkspaceUuid,
-  type: 'external' | 'internal' = 'external'
-): Promise<string> {
-  return await getTransactorEndpoint(getToolToken(workspace), type)
-}
+export { getToolToken, getWorkspaceTransactorEndpoint } from '@hcengineering/server-tool'

dev/tool/src/workspace.ts

@@ -20,26 +20,13 @@ import core, {
   type Client as CoreClient,
   type Doc,
   DOMAIN_TX,
-  type MeasureContext,
   type Ref,
   type Tx,
-  type WorkspaceDataId,
-  type WorkspaceIds,
-  type WorkspaceInfoWithStatus,
   type WorkspaceUuid
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo'
-import { createStorageBackupStorage, restore } from '@hcengineering/server-backup'
-import {
-  createDummyStorageAdapter,
-  type PipelineFactory,
-  type StorageAdapter,
-  wrapPipeline
-} from '@hcengineering/server-core'
-import { createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
 import { connect } from '@hcengineering/server-tool'
 import { generateModelDiff, printDiff } from './mdiff'
-import { createEmptyBroadcastOps } from '@hcengineering/server-pipeline'
 
 export async function diffWorkspace (mongoUrl: string, dbName: string, rawTxes: Tx[]): Promise<void> {
   const client = getMongoClient(mongoUrl)
@@ -117,60 +104,3 @@ export async function updateField (
     await connection.close()
   }
 }
-
-export async function backupRestore (
-  ctx: MeasureContext,
-  dbURL: string,
-  bucketName: string,
-  workspace: WorkspaceInfoWithStatus,
-  pipelineFactoryFactory: (mongoUrl: string, storage: StorageAdapter) => PipelineFactory,
-  skipDomains: string[]
-): Promise<boolean> {
-  const storageEnv = process.env.STORAGE
-  if (storageEnv === undefined) {
-    console.error('please provide STORAGE env')
-    process.exit(1)
-  }
-  if (bucketName.trim() === '') {
-    console.error('please provide butket name env')
-    process.exit(1)
-  }
-  const backupStorageConfig = storageConfigFromEnv(storageEnv)
-
-  const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0])
-
-  const workspaceStorage = createDummyStorageAdapter()
-  const pipelineFactory = pipelineFactoryFactory(dbURL, workspaceStorage)
-
-  try {
-    const storage = await createStorageBackupStorage(
-      ctx,
-      storageAdapter,
-      {
-        uuid: 'backup' as WorkspaceUuid,
-        url: bucketName,
-        dataId: bucketName as WorkspaceDataId
-      },
-      workspace.dataId ?? workspace.uuid
-    )
-    const wsUrl: WorkspaceIds = {
-      uuid: workspace.uuid,
-      dataId: workspace.dataId,
-      url: workspace.url
-    }
-    const result: boolean = await ctx.with('restore', {}, (ctx) =>
-      restore(ctx, '', wsUrl, storage, {
-        date: -1,
-        skip: new Set(skipDomains),
-        recheck: false,
-        storageAdapter: workspaceStorage,
-        getConnection: async () => {
-          return wrapPipeline(ctx, await pipelineFactory(ctx, wsUrl, createEmptyBroadcastOps(), null), wsUrl)
-        }
-      })
-    )
-    return result
-  } finally {
-    await storageAdapter.close()
-  }
-}

plugins/setting-assets/lang/cs.json

@@ -132,6 +132,7 @@
     "BackupFileDownload": "Stáhnout soubor",
     "BackupFiles": "Soubory záloh",
     "BackupNoBackup": "V současné době nejsou k dispozici žádné zálohy.",
+    "NonBackupedBlobs": "Non Backed up blobs",
     "Calendar": "Kalendář",
     "StartOfTheWeek": "Začátek týdne",
     "SystemSetupString": "Nastavení systému ({day})",

plugins/setting-assets/lang/de.json

@@ -132,6 +132,7 @@
     "BackupFileDownload": "Datei herunterladen",
     "BackupFiles": "Sicherungsdateien",
     "BackupNoBackup": "Derzeit sind keine Sicherungen verfügbar.",
+    "NonBackupedBlobs": "Nicht gesicherte Blobs",
     "Calendar": "Kalender",
     "StartOfTheWeek": "Wochenstart",
     "SystemSetupString": "Systemkonfiguration ({day})",

plugins/setting-assets/lang/en.json

@@ -132,6 +132,7 @@
     "BackupFileDownload": "Download File",
     "BackupFiles": "Backup Files",
     "BackupNoBackup": "No backups are currently available.",
+    "NonBackupedBlobs": "Non Backed up blobs",
     "Calendar": "Calendar",
     "StartOfTheWeek": "Start of the week",
     "SystemSetupString": "System Setup ({day})",

plugins/setting-assets/lang/es.json

@@ -123,6 +123,7 @@
     "BackupFileDownload": "Descargar archivo",
     "BackupFiles": "Archivos de copia de seguridad",
     "BackupNoBackup": "No hay copias de seguridad disponibles actualmente.",
+    "NonBackupedBlobs": "Blobs no respaldados",
     "Calendar": "Calendario",
     "StartOfTheWeek": "Inicio de la semana",
     "SystemSetupString": "Configuración del sistema ({day})",

plugins/setting-assets/lang/fr.json

@@ -132,12 +132,13 @@
     "BackupFileDownload": "Télécharger le fichier",
     "BackupFiles": "Fichiers de sauvegarde",
     "BackupNoBackup": "Aucune sauvegarde n'est actuellement disponible.",
+    "NonBackupedBlobs": "Non Backed up blobs",
     "Calendar": "Calendrier",
     "StartOfTheWeek": "Début de semaine",
     "SystemSetupString": "Configuration du système ({day})",
     "DefaultString": "Par défaut ({day})",
     "AddAttribute": "Ajouter un attribut",
-    "WorkspaceNamePattern": "Le nom doit comporter 40 caractères ou moins, ne doit pas être vide et ne doit pas contenir de caractères spéciaux (<, >, /)" ,
+    "WorkspaceNamePattern": "Le nom doit comporter 40 caractères ou moins, ne doit pas être vide et ne doit pas contenir de caractères spéciaux (<, >, /)",
     "Mailboxes": "Boîtes aux lettres",
     "CreateMailbox": "Créer une boîte aux lettres",
     "CreateMailboxPlaceholder": "my-cool-name",

plugins/setting-assets/lang/it.json

@@ -132,6 +132,7 @@
     "BackupFileDownload": "Scarica file",
     "BackupFiles": "File di backup",
     "BackupNoBackup": "Non sono attualmente disponibili backup.",
+    "NonBackupedBlobs": "Non Backed up blobs",
     "Calendar": "Calendario",
     "StartOfTheWeek": "Inizio settimana",
     "SystemSetupString": "Configurazione del sistema ({day})",

plugins/setting-assets/lang/ja.json

@@ -132,6 +132,7 @@
     "BackupFileDownload": "ファイルをダウンロード",
     "BackupFiles": "バックアップファイル",
     "BackupNoBackup": "現在、利用可能なバックアップはありません。",
+    "NonBackupedBlobs": "バックアップされていないブロブ",
     "Calendar": "カレンダー",
     "StartOfTheWeek": "週の開始",
     "SystemSetupString": "システム設定 ({day})",

plugins/setting-assets/lang/pt.json

@@ -123,6 +123,7 @@
     "BackupFileDownload": "Baixar Arquivo",
     "BackupFiles": "Arquivos de Backup",
     "BackupNoBackup": "Não há backups disponíveis no momento.",
+    "NonBackupedBlobs": "Non Backed up blobs",
     "Calendar": "Calendário",
     "StartOfTheWeek": "Início da semana",
     "SystemSetupString": "Configuração do sistema ({day})",

plugins/setting-assets/lang/ru.json

@@ -132,6 +132,7 @@
     "BackupFileDownload": "Скачать файл",
     "BackupFiles": "Файлы резервной копии",
     "BackupNoBackup": "В настоящее время резервные копии недоступны.",
+    "NonBackupedBlobs": "Не резервируемые блобы",
     "Calendar": "Календарь",
     "StartOfTheWeek": "Начало недели",
     "SystemSetupString": "Системная настройка ({day})",

plugins/setting-assets/lang/zh.json

@@ -132,6 +132,7 @@
     "BackupFileDownload": "下载文件",
     "BackupFiles": "备份文件",
     "BackupNoBackup": "当前没有可用的备份。",
+    "NonBackupedBlobs": "未备份的 Blob",
     "Calendar": "日历",
     "StartOfTheWeek": "本周开始",
     "SystemSetupString": "系统设置({day})",

plugins/setting-resources/src/components/Backup.svelte

@@ -15,7 +15,7 @@
 <script lang="ts">
   import { AccountRole, getCurrentAccount, hasAccountRole } from '@hcengineering/core'
   import { getMetadata } from '@hcengineering/platform'
-  import presentation from '@hcengineering/presentation'
+  import presentation, { getFileUrl } from '@hcengineering/presentation'
   import { Breadcrumb, Button, Expandable, Header, Label, Loading } from '@hcengineering/ui'
   import Scroller from '@hcengineering/ui/src/components/Scroller.svelte'
   import view from '@hcengineering/view'
@@ -34,6 +34,7 @@
   let backupInfo:
   | {
     files: { name: string, size: number }[]
+    extraBlobs: { name: string, size: number, contentType: string }[]
     error?: string
     info?: BackupInfo
   }
@@ -116,14 +117,14 @@
 
     return ''
   }
-  function getFileUrl (filename: string): string {
+  function getBackupFileUrl (filename: string): string {
     return `${backupUrl}/${workspaceUuid}/${filename}`
   }
 
-  async function downloadFile (filename: string): Promise<void> {
+  async function doDownload (downloadUrl: string, filename?: string): Promise<void> {
     const a = document.createElement('a')
     try {
-      const response = await fetch(getFileUrl(filename), {
+      const response = await fetch(downloadUrl, {
         headers: {
           Authorization: `Bearer ${token}`
         }
@@ -132,7 +133,9 @@
       const blob = await response.blob()
       const url = window.URL.createObjectURL(blob)
       a.href = url
-      a.download = filename
+      if (filename !== undefined) {
+        a.download = filename
+      }
       document.body.appendChild(a)
       a.click()
       window.URL.revokeObjectURL(url)
@@ -142,12 +145,23 @@
       document.body.removeChild(a)
     }
   }
+
+  async function downloadFile (filename: string): Promise<void> {
+    const url = getBackupFileUrl(filename)
+    await doDownload(url, filename)
+  }
+  async function downloadBlobFile (filename: string): Promise<void> {
+    const url = getFileUrl(filename)
+    await doDownload(url, filename)
+  }
   let copied = false
   let copied2 = false
 
   $: snapshots = backupInfo?.info?.snapshots ?? []
 
   $: snapshorsr = [...(backupInfo?.info?.snapshots ?? [])].reverse()
+
+  $: blobs = (backupInfo?.extraBlobs ?? []).sort((a, b) => b.size - a.size)
 </script>
 
 <div class="hulyComponent flex-no-shrink">
@@ -281,7 +295,6 @@
           <div class="error">{backupInfo.error}</div>
         {/if}
       </Expandable>
-
       <Expandable expanded={true} bordered>
         <svelte:fragment slot="title">
           <div class="p-1">
@@ -312,13 +325,13 @@
             <div class="wrap">
               <Label label={setting.string.BackupLinkInfo} />
               <div class="select-text anti-component p-3 border-b-1 border-divider-color">
-                {getFileUrl('index.html')}
+                {getBackupFileUrl('index.html')}
               </div>
             </div>
             <Button
               label={copied ? view.string.Copied : view.string.CopyToClipboard}
               on:click={() => {
-                void navigator.clipboard.writeText(getFileUrl('index.html')).then(() => {
+                void navigator.clipboard.writeText(getBackupFileUrl('index.html')).then(() => {
                   copied = true
                   setTimeout(() => {
                     copied = false
@@ -344,6 +357,28 @@
           </div>
         </div>
       </Expandable>
+      <Expandable expanded={false} bordered>
+        <svelte:fragment slot="title">
+          <div class="p-1">
+            <Label label={setting.string.NonBackupedBlobs} />
+            {blobs.length}
+          </div>
+        </svelte:fragment>
+        <div class="p-1">
+          {#each blobs as file}
+            <div class="file-item">
+              <div class="file-name">{file.name}</div>
+              <div class="file-info">
+                <span class="file-size">{getSize(file.size)}</span>
+                <span class="file-content-type">{file.contentType}</span>
+                <div class="file-actions">
+                  <Button label={setting.string.BackupFileDownload} on:click={() => downloadBlobFile(file.name)} />
+                </div>
+              </div>
+            </div>
+          {/each}
+        </div>
+      </Expandable>
     {/if}
   </Scroller>
 </div>

plugins/setting/src/index.ts

@@ -233,6 +233,7 @@ export default plugin(settingId, {
     BackupFileDownload: '' as IntlString,
     BackupFiles: '' as IntlString,
     BackupNoBackup: '' as IntlString,
+    NonBackupedBlobs: '' as IntlString,
     AddAttribute: '' as IntlString,
     Mailboxes: '' as IntlString,
     CreateMailbox: '' as IntlString,

pods/server/src/server_http.ts

@@ -247,14 +247,21 @@ export function startHttpServer (
         case 'profile-stop': {
           profiling = false
           if (sessions.profiling?.stop != null) {
-            void sessions.profiling.stop().then((profile) => {
-              ctx.warn(
-                '---------------------------------------------PROFILING SESSION STOPPED---------------------------------------------',
-                {}
-              )
-              res.writeHead(200, { 'Content-Type': 'application/json' })
-              res.end(profile ?? '{ error: "no profiling" }')
-            })
+            void sessions.profiling
+              .stop()
+              .then((profile) => {
+                ctx.warn(
+                  '---------------------------------------------PROFILING SESSION STOPPED---------------------------------------------',
+                  {}
+                )
+                res.writeHead(200, { 'Content-Type': 'application/json' })
+                res.end(profile ?? '{ error: "no profiling" }')
+              })
+              .catch((err) => {
+                ctx.error('error', { err })
+                res.writeHead(500)
+                res.end()
+              })
           } else {
             res.writeHead(404)
             res.end()
@@ -262,9 +269,20 @@ export function startHttpServer (
 
           return
         }
+        case 'force-maintenance': {
+          const wsId = req.query.wsId as WorkspaceUuid
+          void sessions.forceMaintenance(ctx, wsId ?? payload.workspace).catch((err) => {
+            ctx.error('error', { err })
+          })
+          res.writeHead(200)
+          res.end()
+          return
+        }
         case 'force-close': {
           const wsId = req.query.wsId as WorkspaceUuid
-          void sessions.forceClose(wsId ?? payload.workspace)
+          void sessions.forceClose(wsId ?? payload.workspace).catch((err) => {
+            ctx.error('error', { err })
+          })
           res.writeHead(200)
           res.end()
           return

server/backup-service/src/index.ts

@@ -13,9 +13,9 @@
 // limitations under the License.
 //
 import {
-  type MeasureContext,
   systemAccountUuid,
   type Branding,
+  type MeasureContext,
   type WorkspaceIds,
   type WorkspaceInfoWithStatus
 } from '@hcengineering/core'
@@ -62,7 +62,6 @@ export function startBackup (
     storageAdapter,
     { ...config, Token: token },
     pipelineFactory,
-    workspaceStorageAdapter,
     (ctx, workspace, branding, externalStorage) => {
       return getConfig(ctx, mainDbUrl, workspace, branding, externalStorage)
     },
@@ -122,7 +121,6 @@ export async function backupWorkspace (
       storageAdapter,
       { ...config, Token: token },
       pipelineFactory,
-      workspaceStorageAdapter,
       (ctx, workspace, branding, externalStorage) => {
         return getConfig(ctx, mainDbUrl, workspace, branding, externalStorage)
       },

server/backup/src/index.ts

@@ -13,4 +13,7 @@
 // limitations under the License.
 //
 export * from './backup'
+export * from './restore'
 export * from './service'
+export * from './types'
+export * from './utils'

server/backup/src/restore.ts

@@ -0,0 +1,535 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { Analytics } from '@hcengineering/analytics'
+import core, {
+  Doc,
+  Domain,
+  DOMAIN_BLOB,
+  MeasureContext,
+  RateLimiter,
+  Ref,
+  toIdMap,
+  TxProcessor,
+  type Blob,
+  type LowLevelStorage,
+  type TxCUD,
+  type WorkspaceIds
+} from '@hcengineering/core'
+import { BlobClient } from '@hcengineering/server-client'
+import { BackupClientOps, createDummyStorageAdapter, type Pipeline } from '@hcengineering/server-core'
+import { deepEqual } from 'fast-equals'
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
+import { extract } from 'tar-stream'
+import { createGunzip, gunzipSync } from 'zlib'
+import { BackupStorage } from './storage'
+import type { BackupInfo } from './types'
+import { doTrimHash, loadDigest, migradeBlobData } from './utils'
+export * from './storage'
+
+const dataUploadSize = 2 * 1024 * 1024
+
+const defaultLevel = 9
+
+/**
+ * @public
+ * Restore state of DB to specified point.
+ *
+ * Recheck mean we download and compare every document on our side and if found difference upload changed version to server.
+ */
+export async function restore (
+  ctx: MeasureContext,
+  pipeline: Pipeline,
+  wsIds: WorkspaceIds,
+  storage: BackupStorage,
+  opt: {
+    date: number
+    merge?: boolean
+    parallel?: number
+    recheck?: boolean
+    include?: Set<string>
+    skip?: Set<string>
+    progress?: (progress: number) => Promise<void>
+    cleanIndexState?: boolean
+    historyFile?: string
+  }
+): Promise<boolean> {
+  const infoFile = 'backup.json.gz'
+  const workspaceId = wsIds.uuid
+  if (!(await storage.exists(infoFile))) {
+    ctx.error('file not pressent', { file: infoFile })
+    throw new Error(`${infoFile} should present to restore`)
+  }
+  const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString())
+  let snapshots = backupInfo.snapshots
+  if (opt.date !== -1) {
+    const bk = backupInfo.snapshots.findIndex((it) => it.date === opt.date)
+    if (bk === -1) {
+      ctx.error('could not restore to', { date: opt.date, file: infoFile, workspaceId })
+      throw new Error(`${infoFile} could not restore to ${opt.date}. Snapshot is missing.`)
+    }
+    snapshots = backupInfo.snapshots.slice(0, bk + 1)
+  } else {
+    opt.date = snapshots[snapshots.length - 1].date
+  }
+
+  if (backupInfo.domainHashes === undefined) {
+    backupInfo.domainHashes = {}
+  }
+  ctx.info('restore to ', { id: opt.date, date: new Date(opt.date).toDateString() })
+  const rsnapshots = Array.from(snapshots).reverse()
+
+  // Collect all possible domains
+  const domains = new Set<Domain>()
+  for (const s of snapshots) {
+    Object.keys(s.domains).forEach((it) => domains.add(it as Domain))
+  }
+
+  const historyFile: Record<string, string> =
+    opt.historyFile !== undefined && existsSync(opt.historyFile)
+      ? JSON.parse(readFileSync(opt.historyFile).toString())
+      : {}
+
+  const blobClient = new BlobClient(pipeline.context.storageAdapter ?? createDummyStorageAdapter(), wsIds)
+  console.log('connected')
+
+  // We need to find empty domains and clean them.
+  const allDomains = pipeline.context.hierarchy.domains()
+  for (const d of allDomains) {
+    domains.add(d)
+  }
+
+  // We do not backup elastic anymore
+  domains.delete('fulltext-blob' as Domain)
+  domains.delete('doc-index-state' as Domain)
+
+  let uploadedMb = 0
+  let uploaded = 0
+
+  let domainProgress = 0
+
+  const connection = pipeline.context.lowLevelStorage as LowLevelStorage
+  const ops = new BackupClientOps(connection)
+
+  const printUploaded = (msg: string, size: number): void => {
+    if (size == null) {
+      return
+    }
+    uploaded += size
+    const newDownloadedMb = Math.round(uploaded / (1024 * 1024))
+    const newId = Math.round(newDownloadedMb / 10)
+    if (uploadedMb !== newId) {
+      uploadedMb = newId
+      ctx.info('Uploaded', {
+        msg,
+        written: newDownloadedMb,
+        workspace: workspaceId
+      })
+    }
+  }
+
+  async function processDomain (c: Domain): Promise<void> {
+    const dHash = await connection.getDomainHash(ctx, c)
+    if (backupInfo.domainHashes[c] === dHash) {
+      ctx.info('no changes in domain', { domain: c })
+      return
+    }
+    const changeset = await loadDigest(ctx, storage, snapshots, c, opt.date)
+    // We need to load full changeset from server
+    const serverChangeset = new Map<Ref<Doc>, string>()
+
+    const oldUsed = process.memoryUsage().heapUsed
+    try {
+      global.gc?.()
+    } catch (err) {}
+    const mm = { old: oldUsed / (1024 * 1024), current: process.memoryUsage().heapUsed / (1024 * 1024) }
+    if (mm.old > mm.current + mm.current / 10) {
+      ctx.info('memory-stats', mm)
+    }
+
+    let idx: number | undefined
+    let loaded = 0
+    let el = 0
+    let chunks = 0
+    let dataSize = 0
+    try {
+      while (true) {
+        if (opt.progress !== undefined) {
+          await opt.progress?.(domainProgress)
+        }
+        const st = Date.now()
+        const it = await ops.loadChunk(ctx, c, idx)
+        dataSize += it.size ?? 0
+        chunks++
+
+        idx = it.idx
+        el += Date.now() - st
+
+        for (const { id, hash } of it.docs) {
+          serverChangeset.set(id as Ref<Doc>, hash)
+          loaded++
+        }
+
+        if (el > 2500) {
+          ctx.info('loaded from server', { domain: c, loaded, el, chunks, workspace: workspaceId })
+          el = 0
+          chunks = 0
+        }
+        if (it.finished) {
+          break
+        }
+      }
+    } finally {
+      if (idx !== undefined) {
+        await ops.closeChunk(ctx, idx)
+      }
+    }
+    ctx.info('loaded', {
+      domain: c,
+      loaded,
+      workspace: workspaceId,
+      dataSize: Math.round((dataSize / (1024 * 1024)) * 100) / 100
+    })
+    ctx.info('\tcompare documents', {
+      size: changeset.size,
+      serverSize: serverChangeset.size,
+      workspace: workspaceId
+    })
+
+    // Let's find difference
+    const docsToAdd = new Map(
+      opt.recheck === true // If recheck we check all documents.
+        ? Array.from(changeset.entries())
+        : Array.from(changeset.entries()).filter(
+          ([it]) =>
+            !serverChangeset.has(it) ||
+              (serverChangeset.has(it) && doTrimHash(serverChangeset.get(it)) !== doTrimHash(changeset.get(it)))
+        )
+    )
+    const docsToRemove = Array.from(serverChangeset.keys()).filter((it) => !changeset.has(it))
+
+    const docs: Doc[] = []
+    const blobs = new Map<string, { doc: Doc | undefined, buffer: Buffer | undefined }>()
+    let sendSize = 0
+    let totalSend = 0
+    async function sendChunk (doc: Doc | undefined, len: number): Promise<void> {
+      if (opt.progress !== undefined) {
+        await opt.progress?.(domainProgress)
+      }
+      if (doc !== undefined) {
+        docsToAdd.delete(doc._id)
+        docs.push(doc)
+      }
+      sendSize = sendSize + len
+
+      if (sendSize > dataUploadSize || (doc === undefined && docs.length > 0)) {
+        let docsToSend = docs
+        totalSend += docs.length
+        ctx.info('upload-' + c, {
+          docs: docs.length,
+          totalSend,
+          from: docsToAdd.size + totalSend,
+          sendSize,
+          workspace: workspaceId
+        })
+        // Correct docs without space
+        for (const d of docs) {
+          if (d.space == null) {
+            d.space = core.space.Workspace
+          }
+
+          if (TxProcessor.isExtendsCUD(d._class)) {
+            const tx = d as TxCUD<Doc>
+            if (tx.objectSpace == null) {
+              tx.objectSpace = core.space.Workspace
+            }
+          }
+        }
+
+        if (opt.recheck === true) {
+          // We need to download all documents and compare them.
+          const serverDocs = toIdMap(
+            await ops.loadDocs(
+              ctx,
+              c,
+              docs.map((it) => it._id)
+            )
+          )
+          docsToSend = docs.filter((doc) => {
+            const serverDoc = serverDocs.get(doc._id)
+            if (serverDoc !== undefined) {
+              const { '%hash%': _h1, ...dData } = doc as any
+              const { '%hash%': _h2, ...sData } = serverDoc as any
+
+              return !deepEqual(dData, sData)
+            }
+            return true
+          })
+        }
+        try {
+          await ops.upload(ctx, c, docsToSend)
+        } catch (err: any) {
+          ctx.error('error during upload', { err, docs: JSON.stringify(docs) })
+        }
+
+        docs.length = 0
+        sendSize = 0
+      }
+      printUploaded('upload', len)
+    }
+
+    let processed = 0
+
+    const blobUploader = new RateLimiter(10)
+
+    for (const s of rsnapshots) {
+      const d = s.domains[c]
+
+      if (d !== undefined && docsToAdd.size > 0) {
+        const sDigest = await loadDigest(ctx, storage, [s], c)
+        const requiredDocs = new Map(Array.from(sDigest.entries()).filter(([it]) => docsToAdd.has(it)))
+
+        let lastSendTime = Date.now()
+        async function sendBlob (blob: Blob, data: Buffer, next: (err?: any) => void): Promise<void> {
+          await blobUploader.add(async () => {
+            next()
+            let needSend = true
+            if (opt.historyFile !== undefined) {
+              if (historyFile[blob._id] === blob.etag) {
+                needSend = false
+              }
+            }
+
+            if (needSend) {
+              try {
+                await blobClient.upload(ctx, blob._id, blob.size, blob.contentType, data)
+                if (opt.historyFile !== undefined) {
+                  historyFile[blob._id] = blob.etag
+                  if (totalSend % 1000 === 0) {
+                    writeFileSync(opt.historyFile, JSON.stringify(historyFile, undefined, 2))
+                  }
+                }
+              } catch (err: any) {
+                ctx.warn('failed to upload blob', { _id: blob._id, err, workspace: wsIds.uuid })
+                next(err)
+              }
+            }
+            docsToAdd.delete(blob._id)
+            requiredDocs.delete(blob._id)
+            printUploaded('upload:' + blobUploader.processingQueue.size, data.length)
+            totalSend++
+            if (lastSendTime < Date.now()) {
+              lastSendTime = Date.now() + 2500
+
+              ctx.info('upload ' + c, {
+                totalSend,
+                from: docsToAdd.size + totalSend,
+                sendSize,
+                workspace: workspaceId
+              })
+            }
+          })
+        }
+
+        if (requiredDocs.size > 0) {
+          ctx.info('updating', { domain: c, requiredDocs: requiredDocs.size, workspace: workspaceId })
+          // We have required documents here.
+          for (const sf of d.storage ?? []) {
+            if (docsToAdd.size === 0) {
+              break
+            }
+            ctx.info('processing', { storageFile: sf, processed, workspace: wsIds.url })
+            try {
+              const readStream = await storage.load(sf)
+              const ex = extract()
+
+              ex.on('entry', (headers, stream, next) => {
+                const name = headers.name ?? ''
+                processed++
+                // We found blob data
+                if (requiredDocs.has(name as Ref<Doc>)) {
+                  const chunks: Buffer[] = []
+                  stream.on('data', (chunk) => {
+                    chunks.push(chunk)
+                  })
+                  stream.on('end', () => {
+                    const bf = Buffer.concat(chunks as any)
+                    const d = blobs.get(name)
+                    if (d === undefined) {
+                      blobs.set(name, { doc: undefined, buffer: bf })
+                      next()
+                    } else {
+                      blobs.delete(name)
+                      const blob = d?.doc as Blob
+                      ;(blob as any)['%hash%'] = changeset.get(blob._id)
+                      let sz = blob.size
+                      if (Number.isNaN(sz) || sz !== bf.length) {
+                        sz = bf.length
+                      }
+                      void sendBlob(blob, bf, next).catch((err) => {
+                        ctx.error('failed to send blob', { message: err.message })
+                      })
+                    }
+                  })
+                } else if (name.endsWith('.json') && requiredDocs.has(name.substring(0, name.length - 5) as Ref<Doc>)) {
+                  const chunks: Buffer[] = []
+                  const bname = name.substring(0, name.length - 5)
+                  stream.on('data', (chunk) => {
+                    chunks.push(chunk)
+                  })
+                  stream.on('end', () => {
+                    const bf = Buffer.concat(chunks as any)
+                    let doc: Doc
+                    try {
+                      doc = JSON.parse(bf.toString()) as Doc
+                    } catch (err) {
+                      ctx.warn('failed to parse blob metadata', { name, workspace: wsIds.url, err })
+                      next()
+                      return
+                    }
+
+                    if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') {
+                      const data = migradeBlobData(doc as Blob, changeset.get(doc._id) as string)
+                      const d = blobs.get(bname) ?? (data !== '' ? Buffer.from(data, 'base64') : undefined)
+                      if (d === undefined) {
+                        blobs.set(bname, { doc, buffer: undefined })
+                        next()
+                      } else {
+                        blobs.delete(bname)
+                        const blob = doc as Blob
+                        const buff = d instanceof Buffer ? d : d.buffer
+                        if (buff != null) {
+                          void sendBlob(blob, d instanceof Buffer ? d : (d.buffer as Buffer), next).catch((err) => {
+                            ctx.error('failed to send blob', { err })
+                          })
+                        } else {
+                          next()
+                        }
+                      }
+                    } else {
+                      ;(doc as any)['%hash%'] = changeset.get(doc._id)
+                      void sendChunk(doc, bf.length)
+                        .finally(() => {
+                          requiredDocs.delete(doc._id)
+                          next()
+                        })
+                        .catch((err) => {
+                          ctx.error('failed to sendChunk', { err })
+                          next(err)
+                        })
+                    }
+                  })
+                } else {
+                  next()
+                }
+                stream.resume() // just auto drain the stream
+              })
+
+              await blobUploader.waitProcessing()
+
+              const unzip = createGunzip({ level: defaultLevel })
+
+              const endPromise = new Promise((resolve, reject) => {
+                ex.on('finish', () => {
+                  resolve(null)
+                })
+
+                readStream.on('end', () => {
+                  readStream.destroy()
+                })
+                readStream.pipe(unzip).on('error', (err) => {
+                  readStream.destroy()
+                  reject(err)
+                })
+                unzip.pipe(ex)
+              })
+
+              await endPromise
+            } catch (err: any) {
+              ctx.error('failed to processing', { storageFile: sf, processed, workspace: wsIds.url })
+            }
+          }
+        }
+      }
+    }
+
+    await sendChunk(undefined, 0)
+    async function performCleanOfDomain (docsToRemove: Ref<Doc>[], c: Domain): Promise<void> {
+      ctx.info('cleanup', { toRemove: docsToRemove.length, workspace: workspaceId, domain: c })
+      while (docsToRemove.length > 0) {
+        const part = docsToRemove.splice(0, 10000)
+        try {
+          await ops.clean(ctx, c, part)
+        } catch (err: any) {
+          ctx.error('failed to clean, will retry', { error: err.message, workspaceId })
+          docsToRemove.push(...part)
+        }
+      }
+    }
+    if (c !== DOMAIN_BLOB) {
+      // Clean domain documents if not blob
+      if (docsToRemove.length > 0 && opt.merge !== true) {
+        await performCleanOfDomain(docsToRemove, c)
+      }
+    }
+  }
+
+  const limiter = new RateLimiter(opt.parallel ?? 1)
+
+  try {
+    let i = 0
+    for (const c of domains) {
+      if (opt.progress !== undefined) {
+        await opt.progress?.(domainProgress)
+      }
+      if (opt.include !== undefined && !opt.include.has(c)) {
+        continue
+      }
+      if (opt.skip?.has(c) === true) {
+        continue
+      }
+      await limiter.add(async () => {
+        ctx.info('processing domain', { domain: c, workspaceId })
+        let retry = 5
+        let delay = 1
+        while (retry > 0) {
+          retry--
+          try {
+            await processDomain(c)
+            if (delay > 1) {
+              ctx.warn('retry-success', { retry, delay, workspaceId })
+            }
+            break
+          } catch (err: any) {
+            ctx.error('failed to process domain', { err, domain: c, workspaceId })
+            if (retry !== 0) {
+              ctx.warn('cool-down to retry', { delay, domain: c, workspaceId })
+              await new Promise((resolve) => setTimeout(resolve, delay * 1000))
+              delay++
+            }
+          }
+        }
+        domainProgress = Math.round(i / domains.size) * 100
+        i++
+      })
+    }
+    await limiter.waitProcessing()
+  } catch (err: any) {
+    Analytics.handleError(err)
+    return false
+  }
+  return true
+}

server/backup/src/service.ts

@@ -14,28 +14,22 @@
 //
 
 import { Analytics } from '@hcengineering/analytics'
-import core, {
-  DOMAIN_TX,
+import {
   groupByArray,
-  Hierarchy,
   isActiveMode,
-  ModelDb,
   RateLimiter,
   reduceCalls,
-  SortingOrder,
   systemAccountUuid,
   WorkspaceDataId,
   WorkspaceUuid,
   type BackupStatus,
   type Branding,
   type MeasureContext,
-  type Tx,
   type WorkspaceIds,
   type WorkspaceInfoWithStatus
 } from '@hcengineering/core'
 import { getAccountClient } from '@hcengineering/server-client'
 import {
-  wrapPipeline,
   type DbConfiguration,
   type Pipeline,
   type PipelineFactory,
@@ -43,8 +37,9 @@ import {
 } from '@hcengineering/server-core'
 import { generateToken } from '@hcengineering/server-token'
 import { clearInterval } from 'node:timers'
-import { backup, restore } from '.'
 import { createStorageBackupStorage } from './storage'
+import { backup } from './backup'
+import { restore } from './restore'
 
 export interface BackupConfig {
   AccountsURL: string
@@ -71,7 +66,6 @@ class BackupWorker {
     readonly storageAdapter: StorageAdapter,
     readonly config: BackupConfig,
     readonly pipelineFactory: PipelineFactory,
-    readonly workspaceStorageAdapter: StorageAdapter,
     readonly getConfig: (
       ctx: MeasureContext,
       workspace: WorkspaceIds,
@@ -303,11 +297,23 @@ class BackupWorker {
         dataId: ws.dataId,
         url: ws.url
       }
+      pipeline = await this.pipelineFactory(
+        ctx,
+        wsIds,
+        {
+          broadcast: () => {},
+          broadcastSessions: () => {}
+        },
+        null
+      )
+      if (pipeline === undefined) {
+        throw new Error('Pipeline is undefined, cannot proceed with backup')
+      }
       const result = await ctx.with(
         'backup',
         {},
         (ctx) =>
-          backup(ctx, '', wsIds, storage, {
+          backup(ctx, pipeline as Pipeline, wsIds, storage, {
             skipDomains: this.skipDomains,
             force: true,
             timeout: this.config.Timeout * 1000,
@@ -316,48 +322,6 @@ class BackupWorker {
             blobDownloadLimit: this.downloadLimit,
             skipBlobContentTypes: ['video/', 'audio/', 'image/'],
             fullVerify: this.fullCheck,
-            storageAdapter: this.workspaceStorageAdapter,
-            getLastTx: async (): Promise<Tx | undefined> => {
-              const config = this.getConfig(ctx, wsIds, null, this.workspaceStorageAdapter)
-              const adapterConf = config.adapters[config.domains[DOMAIN_TX]]
-              const hierarchy = new Hierarchy()
-              const modelDb = new ModelDb(hierarchy)
-              const txAdapter = await adapterConf.factory(
-                ctx,
-                hierarchy,
-                adapterConf.url,
-                wsIds,
-                modelDb,
-                this.workspaceStorageAdapter
-              )
-              try {
-                await txAdapter.init?.(ctx, {})
-
-                return (
-                  await txAdapter.rawFindAll<Tx>(
-                    DOMAIN_TX,
-                    { objectSpace: { $ne: core.space.Model } },
-                    { limit: 1, sort: { modifiedOn: SortingOrder.Descending } }
-                  )
-                ).shift()
-              } finally {
-                await txAdapter.close()
-              }
-            },
-            getConnection: async () => {
-              if (pipeline === undefined) {
-                pipeline = await this.pipelineFactory(
-                  ctx,
-                  wsIds,
-                  {
-                    broadcast: () => {},
-                    broadcastSessions: () => {}
-                  },
-                  null
-                )
-              }
-              return wrapPipeline(ctx, pipeline, wsIds)
-            },
             progress: (progress) => {
               return notify?.(progress) ?? Promise.resolve()
             }
@@ -409,7 +373,6 @@ export function backupService (
   storage: StorageAdapter,
   config: BackupConfig,
   pipelineFactory: PipelineFactory,
-  workspaceStorageAdapter: StorageAdapter,
   getConfig: (
     ctx: MeasureContext,
     workspace: WorkspaceIds,
@@ -419,7 +382,7 @@ export function backupService (
   region: string,
   recheck?: boolean
 ): () => void {
-  const backupWorker = new BackupWorker(storage, config, pipelineFactory, workspaceStorageAdapter, getConfig, region)
+  const backupWorker = new BackupWorker(storage, config, pipelineFactory, getConfig, region)
 
   const shutdown = (): void => {
     void backupWorker.close()
@@ -435,7 +398,6 @@ export async function doBackupWorkspace (
   storage: StorageAdapter,
   config: BackupConfig,
   pipelineFactory: PipelineFactory,
-  workspaceStorageAdapter: StorageAdapter,
   getConfig: (
     ctx: MeasureContext,
     workspace: WorkspaceIds,
@@ -448,16 +410,7 @@ export async function doBackupWorkspace (
   fullCheck: boolean = false,
   notify?: (progress: number) => Promise<void>
 ): Promise<boolean> {
-  const backupWorker = new BackupWorker(
-    storage,
-    config,
-    pipelineFactory,
-    workspaceStorageAdapter,
-    getConfig,
-    region,
-    skipDomains,
-    fullCheck
-  )
+  const backupWorker = new BackupWorker(storage, config, pipelineFactory, getConfig, region, skipDomains, fullCheck)
   backupWorker.downloadLimit = downloadLimit
   const result = await backupWorker.doBackup(ctx, workspace, notify)
   await backupWorker.close()
@@ -470,7 +423,6 @@ export async function doRestoreWorkspace (
   backupAdapter: StorageAdapter,
   bucketName: string,
   pipelineFactory: PipelineFactory,
-  workspaceStorageAdapter: StorageAdapter,
   skipDomains: string[],
   cleanIndexState: boolean,
   notify?: (progress: number) => Promise<void>
@@ -482,32 +434,29 @@ export async function doRestoreWorkspace (
   const ctx = rootCtx.newChild('doRestore', {})
   let pipeline: Pipeline | undefined
   try {
+    pipeline = await pipelineFactory(
+      ctx,
+      wsIds,
+      {
+        broadcast: () => {},
+        broadcastSessions: () => {}
+      },
+      null
+    )
+    if (pipeline === undefined) {
+      throw new Error('Pipeline is undefined, cannot proceed with restore')
+    }
     const restoreIds = { uuid: bucketName as WorkspaceUuid, dataId: bucketName as WorkspaceDataId, url: '' }
     const storage = await createStorageBackupStorage(ctx, backupAdapter, restoreIds, wsIds.dataId ?? wsIds.uuid)
     const result: boolean = await ctx.with(
       'restore',
       {},
       (ctx) =>
-        restore(ctx, '', wsIds, storage, {
+        restore(ctx, pipeline as Pipeline, wsIds, storage, {
           date: -1,
           skip: new Set(skipDomains),
           recheck: false, // Do not need to recheck
-          storageAdapter: workspaceStorageAdapter,
           cleanIndexState,
-          getConnection: async () => {
-            if (pipeline === undefined) {
-              pipeline = await pipelineFactory(
-                ctx,
-                wsIds,
-                {
-                  broadcast: () => {},
-                  broadcastSessions: () => {}
-                },
-                null
-              )
-            }
-            return wrapPipeline(ctx, pipeline, wsIds)
-          },
           progress: (progress) => {
             return notify?.(progress) ?? Promise.resolve()
           }

server/backup/src/types.ts

@@ -0,0 +1,97 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { Doc, Domain, Ref, type BackupStatus, type WorkspaceUuid } from '@hcengineering/core'
+export * from './storage'
+
+/**
+ * Blob data from s3 storage
+ * @public
+ */
+export interface BlobData extends Doc {
+  name: string
+  size: number
+  type: string
+  provider?: string // If node defined, will be default one
+  base64Data: string // base64 encoded data
+}
+
+/**
+ * @public
+ */
+export interface Snapshot {
+  added: Map<Ref<Doc>, string>
+  updated: Map<Ref<Doc>, string>
+  removed: Ref<Doc>[]
+}
+
+/**
+ * @public
+ */
+export interface SnapshotV6 {
+  added: Record<Ref<Doc>, string>
+  updated: Record<Ref<Doc>, string>
+  removed: Ref<Doc>[]
+}
+
+/**
+ * @public
+ */
+export interface DomainData {
+  snapshot?: string // 0.6 json snapshot
+  snapshots?: string[]
+  storage?: string[]
+
+  // Some statistics
+  added: number
+  updated: number
+  removed: number
+}
+
+/**
+ * @public
+ */
+export interface BackupSnapshot {
+  // _id => hash of added items.
+  domains: Record<Domain, DomainData>
+  date: number
+
+  compacting?: boolean
+  stIndex: number // Snapshot index
+}
+
+/**
+ * @public
+ */
+export interface BackupInfo {
+  workspace: WorkspaceUuid
+  version: string
+  snapshots: BackupSnapshot[]
+  snapshotsIndex?: number
+  lastTxId?: string
+
+  // A hash of current domain transactions, so we could skip all other checks if same.
+  domainHashes: Record<Domain, string>
+
+  migrations: Record<string, boolean | string>
+  dataSize?: number
+  blobsSize?: number
+  backupSize?: number
+}
+
+export interface BackupResult extends Omit<BackupStatus, 'backups' | 'lastBackup'> {
+  result: boolean
+}

server/client/src/blob.ts

@@ -19,51 +19,18 @@ import { Buffer } from 'node:buffer'
 
 // Will use temporary file to store huge content into
 export class BlobClient {
-  transactorAPIUrl: string
   index: number
   constructor (
-    readonly transactorUrl: string,
-    readonly token: string,
-    readonly workspace: WorkspaceIds,
-    readonly opt?: {
-      storageAdapter?: StorageAdapter
-    }
+    readonly storageAdapter: StorageAdapter,
+    readonly workspace: WorkspaceIds
   ) {
     this.index = 0
-    let url = transactorUrl
-    if (url.endsWith('/')) {
-      url = url.slice(0, url.length - 1)
-    }
-
-    this.transactorAPIUrl = url.replaceAll('wss://', 'https://').replace('ws://', 'http://') + '/api/v1/blob'
   }
 
   async checkFile (ctx: MeasureContext, name: string): Promise<boolean> {
-    if (this.opt?.storageAdapter !== undefined) {
-      const obj = await this.opt?.storageAdapter.stat(ctx, this.workspace, name)
-      if (obj !== undefined) {
-        return true
-      }
-    }
-    for (let i = 0; i < 5; i++) {
-      try {
-        const response = await fetch(this.transactorAPIUrl + `?name=${encodeURIComponent(name)}`, {
-          headers: {
-            Authorization: 'Bearer ' + this.token,
-            Range: 'bytes=0-1'
-          }
-        })
-        if (response.status === 404) {
-          return false
-        }
-        const buff = await response.arrayBuffer()
-        return buff.byteLength > 0
-      } catch (err: any) {
-        if (i === 4) {
-          ctx.error('Failed to check file', { name, error: err })
-        }
-        await new Promise<void>((resolve) => setTimeout(resolve, 10))
-      }
+    const obj = await this.storageAdapter.stat(ctx, this.workspace, name)
+    if (obj !== undefined) {
+      return true
     }
     return false
   }
@@ -81,75 +48,22 @@ export class BlobClient {
     const chunkSize = 50 * 1024 * 1024
 
     // Use ranges to iterave through file with retry if required.
-    while (written < size || size === -1) {
+    while (written < size) {
       let i = 0
-      let response: Response | undefined
       for (; i < 5; i++) {
         try {
-          const st = Date.now()
-          let chunk: Buffer
-
-          if (this.opt?.storageAdapter !== undefined) {
-            const chunks: Buffer[] = []
-            const readable = await this.opt.storageAdapter.partial(ctx, this.workspace, name, written, chunkSize)
-            await new Promise<void>((resolve) => {
-              readable.on('data', (chunk) => {
-                chunks.push(chunk)
-              })
-              readable.on('end', () => {
-                readable.destroy()
-                resolve()
-              })
+          const chunks: Buffer[] = []
+          const readable = await this.storageAdapter.partial(ctx, this.workspace, name, written, chunkSize)
+          await new Promise<void>((resolve) => {
+            readable.on('data', (chunk) => {
+              chunks.push(chunk)
             })
-            chunk = Buffer.concat(chunks)
-          } else {
-            const header: Record<string, string> = {
-              Authorization: 'Bearer ' + this.token
-            }
-
-            if (!(size !== -1 && written === 0 && size < chunkSize)) {
-              header.Range = `bytes=${written}-${size === -1 ? written + chunkSize : Math.min(size - 1, written + chunkSize)}`
-            }
-
-            response = await fetch(this.transactorAPIUrl + `?name=${encodeURIComponent(name)}`, { headers: header })
-            if (header.Range != null) {
-              ctx.info('fetch part', { time: Date.now() - st, blobId: name, written, size })
-            }
-            if (response.status === 403) {
-              i = 5
-              // No file, so make it empty
-              throw new Error(`Unauthorized ${this.transactorAPIUrl}/${this.workspace.uuid}/${name}`)
-            }
-            if (response.status === 404) {
-              i = 5
-              // No file, so make it empty
-              throw new Error(`No file for ${this.transactorAPIUrl}/${this.workspace.uuid}/${name}`)
-            }
-            if (response.status === 416) {
-              if (size === -1) {
-                size = parseInt((response.headers.get('content-range') ?? '').split('*/')[1])
-                continue
-              }
-
-              // No file, so make it empty
-              throw new Error(`No file for ${this.transactorAPIUrl}/${this.workspace.uuid}/${name}`)
-            }
-            chunk = Buffer.from(await response.arrayBuffer())
-
-            if (header.Range == null) {
-              size = chunk.length
-            }
-            // We need to parse
-            // 'Content-Range': `bytes ${start}-${end}/${size}`
-            // To determine if something is left
-            const range = response.headers.get('Content-Range')
-            if (range !== null) {
-              const [, total] = range.split(' ')[1].split('/')
-              if (total !== undefined) {
-                size = parseInt(total)
-              }
-            }
-          }
+            readable.on('end', () => {
+              readable.destroy()
+              resolve()
+            })
+          })
+          const chunk = Buffer.concat(chunks)
 
           await new Promise<void>((resolve, reject) => {
             writable.write(chunk, (err) => {
@@ -189,44 +103,6 @@ export class BlobClient {
   }
 
   async upload (ctx: MeasureContext, name: string, size: number, contentType: string, buffer: Buffer): Promise<void> {
-    if (this.opt?.storageAdapter !== undefined) {
-      await this.opt.storageAdapter.put(ctx, this.workspace, name, buffer, contentType, size)
-    } else {
-      // TODO: We need to improve this logig, to allow restore of huge blobs
-      for (let i = 0; i < 5; i++) {
-        try {
-          const resp = await (
-            await fetch(
-              this.transactorAPIUrl +
-                `?name=${encodeURIComponent(name)}&contentType=${encodeURIComponent(contentType)}&size=${size}`,
-              {
-                keepalive: true,
-                method: 'PUT',
-                headers: {
-                  Authorization: 'Bearer ' + this.token,
-                  'Content-Type': contentType
-                },
-                body: buffer
-              }
-            )
-          ).text()
-          try {
-            const json = JSON.parse(resp)
-            if (json.error !== undefined) {
-              ctx.error('failed to upload file, error from server', { name, message: json.error })
-              return
-            }
-          } catch (err) {
-            console.log(err)
-          }
-          break
-        } catch (err: any) {
-          if (i === 4) {
-            ctx.error('failed to upload file', { name, message: err.message, cause: err.cause?.message })
-            throw err
-          }
-        }
-      }
-    }
+    await this.storageAdapter.put(ctx, this.workspace, name, buffer, contentType, size)
   }
 }

server/core/src/storage.ts

@@ -18,7 +18,7 @@ export function getBucketId (workspace: WorkspaceUuid): string {
   return workspace
 }
 
-const chunkSize = 5000
+const chunkSize = 50000
 
 /**
  * @public
@@ -45,6 +45,7 @@ export class BackupClientOps {
     idx?: number
   ): Promise<{
       idx: number
+      size?: number
       docs: DocInfo[]
       finished: boolean
     }> {

server/core/src/types.ts

@@ -727,6 +727,8 @@ export interface SessionManager {
 
   forceClose: (wsId: WorkspaceUuid, ignoreSocket?: ConnectionSocket) => Promise<void>
 
+  forceMaintenance: (ctx: MeasureContext, workspaceId: WorkspaceUuid) => Promise<void>
+
   closeWorkspaces: (ctx: MeasureContext) => Promise<void>
 
   scheduleMaintenance: (timeMinutes: number, message?: string) => void

server/server/src/sessionManager.ts

@@ -250,13 +250,18 @@ export class TSessionManager implements SessionManager {
       }
 
       // Wait some time for new client to appear before closing workspace.
-      if (workspace.sessions.size === 0 && workspace.closing === undefined && workspace.workspaceInitCompleted) {
+      if (
+        !workspace.maintenance &&
+        workspace.sessions.size === 0 &&
+        workspace.closing === undefined &&
+        workspace.workspaceInitCompleted
+      ) {
         workspace.softShutdown--
         if (workspace.softShutdown <= 0) {
           this.ctx.warn('closing workspace, no users', {
             workspace: workspace.wsId.url,
             wsId,
-            upgrade: workspace.upgrade
+            upgrade: workspace.maintenance
           })
           workspace.closing = this.performWorkspaceCloseCheck(workspace)
         }
@@ -474,7 +479,7 @@ export class TSessionManager implements SessionManager {
     }
 
     if (token.extra?.model === 'upgrade') {
-      if (workspace.upgrade) {
+      if (workspace.maintenance) {
         ctx.warn('reconnect workspace in upgrade', {
           account: token.account,
           workspace: workspaceUuid,
@@ -491,7 +496,7 @@ export class TSessionManager implements SessionManager {
         await this.switchToUpgradeSession(token, ctx.parent ?? ctx, workspace, ws)
       }
     } else {
-      if (workspace.upgrade) {
+      if (workspace.maintenance || this.maintenanceWorkspaces.has(workspace.wsId.uuid)) {
         ctx.warn('connect during upgrade', {
           account: token.account,
           workspace: workspace.wsId.url,
@@ -512,6 +517,8 @@ export class TSessionManager implements SessionManager {
     socialIds: []
   }
 
+  maintenanceWorkspaces = new Set<WorkspaceUuid>()
+
   async addSession (
     ctx: MeasureContext,
     ws: ConnectionSocket,
@@ -624,7 +631,7 @@ export class TSessionManager implements SessionManager {
     }
 
     // Mark as upgrade, to prevent any new clients to connect during close
-    workspace.upgrade = true
+    workspace.maintenance = true
     // If upgrade client is used.
     // Drop all existing clients
     await this.doCloseAll(workspace, 0, 'upgrade', ws)
@@ -644,7 +651,7 @@ export class TSessionManager implements SessionManager {
   }
 
   doBroadcast (ws: Workspace, tx: Tx[], target?: AccountUuid | AccountUuid[], exclude?: AccountUuid[]): void {
-    if (ws.upgrade) {
+    if (ws.maintenance) {
       return
     }
     if (target !== undefined && !Array.isArray(target)) {
@@ -725,7 +732,7 @@ export class TSessionManager implements SessionManager {
       })
       return
     }
-    if (workspace?.upgrade ?? false) {
+    if (workspace?.maintenance ?? false) {
       return
     }
 
@@ -789,7 +796,7 @@ export class TSessionManager implements SessionManager {
       workspaceIds,
       branding
     )
-    workspace.upgrade = upgrade
+    workspace.maintenance = upgrade
     this.workspaces.set(token.workspace, workspace)
 
     return workspace
@@ -895,7 +902,7 @@ export class TSessionManager implements SessionManager {
               const user = sessionRef.session.getUser()
               if (workspace !== undefined) {
                 const another = Array.from(workspace.sessions.values()).findIndex((p) => p.session.getUser() === user)
-                if (another === -1 && !workspace.upgrade) {
+                if (another === -1 && !workspace.maintenance) {
                   void workspace.with(async (pipeline) => {
                     await pipeline.closeSession(ctx, sessionRef.session.sessionId)
                     // await communicationApi.closeSession(sessionRef.session.sessionId)
@@ -924,11 +931,22 @@ export class TSessionManager implements SessionManager {
     }
   }
 
+  async forceMaintenance (ctx: MeasureContext, workspaceId: WorkspaceUuid): Promise<void> {
+    const workspace = this.workspaces.get(workspaceId)
+    this.maintenanceWorkspaces.add(workspaceId)
+    if (workspace !== undefined) {
+      workspace.maintenance = true
+      ctx.info('force-maintenance', { workspaceId })
+      await this.doCloseAll(workspace, 99, 'force-close')
+    }
+  }
+
   async forceClose (wsId: WorkspaceUuid, ignoreSocket?: ConnectionSocket): Promise<void> {
     const ws = this.workspaces.get(wsId)
+    this.maintenanceWorkspaces.delete(wsId)
     if (ws !== undefined) {
       this.ctx.warn('force-close', { name: ws.wsId.url })
-      ws.upgrade = true // We need to similare upgrade to refresh all clients.
+      ws.maintenance = true // We need to similare upgrade to refresh all clients.
       ws.closing = this.doCloseAll(ws, 99, 'force-close', ignoreSocket)
       this.workspaces.delete(wsId)
       await ws.closing
@@ -1194,7 +1212,7 @@ export class TSessionManager implements SessionManager {
         // TODO: we chould allow this only for admin or system accounts
         let done = false
         const wsRef = this.workspaces.get(workspaceId)
-        if (wsRef?.upgrade ?? false) {
+        if (wsRef?.maintenance ?? false) {
           done = true
           this.ctx.warn('FORCE CLOSE', { workspace: workspaceId })
           // In case of upgrade, we need to force close workspace not in interval handler

server/server/src/workspace.ts

@@ -29,7 +29,7 @@ export type WorkspacePipelineFactory = () => Promise<Pipeline>
  */
 export class Workspace {
   pipeline?: Pipeline | Promise<Pipeline>
-  upgrade: boolean = false
+  maintenance: boolean = false
   closing?: Promise<void>
 
   workspaceInitCompleted: boolean = false

server/tool/src/index.ts

@@ -32,7 +32,6 @@ import core, {
   platformNow,
   platformNowDiff,
   type Ref,
-  systemAccountUuid,
   type Tx,
   type TxOperations,
   type WithLookup,
@@ -59,13 +58,14 @@ import toolPlugin from './plugin'
 import { MigrateClientImpl } from './upgrade'
 
 import { getMetadata, PlatformError, unknownError } from '@hcengineering/platform'
-import { generateToken } from '@hcengineering/server-token'
 import fs from 'fs'
 import * as yaml from 'js-yaml'
 import path from 'path'
+import { sendTransactorEvent } from './utils'
 
 export * from './connect'
 export * from './plugin'
+export * from './utils'
 export { toolPlugin as default }
 
 export class FileModelLogger implements ModelLogger {
@@ -384,16 +384,8 @@ export async function upgradeModel (
 
   // We need to send reboot for workspace
   ctx.info('send force close', { workspace: wsIds, transactorUrl })
-  const serverEndpoint = transactorUrl.replaceAll('wss://', 'https://').replace('ws://', 'http://')
-  const token = generateToken(systemAccountUuid, wsIds.uuid, { service: 'tool', admin: 'true' })
 
-  try {
-    await fetch(serverEndpoint + `/api/v1/manage?token=${token}&operation=force-close`, {
-      method: 'PUT'
-    })
-  } catch (err: any) {
-    // Ignore error if transactor is not yet ready
-  }
+  await sendTransactorEvent(wsIds.uuid, 'force-close')
   return model
 }
 

server/tool/src/utils.ts

@@ -0,0 +1,34 @@
+import { systemAccountUuid, type WorkspaceUuid } from '@hcengineering/core'
+import { getTransactorEndpoint } from '@hcengineering/server-client'
+import { generateToken } from '@hcengineering/server-token'
+
+export function getToolToken (workspace?: WorkspaceUuid): string {
+  return generateToken(systemAccountUuid, workspace, { service: 'tool', admin: 'true' })
+}
+
+export async function getWorkspaceTransactorEndpoint (
+  workspace: WorkspaceUuid,
+  type: 'external' | 'internal' = 'external'
+): Promise<string> {
+  return await getTransactorEndpoint(getToolToken(workspace), type)
+}
+
+export async function sendTransactorEvent (
+  workspace: WorkspaceUuid,
+  operation: 'force-maintenance' | 'force-close',
+  type: 'external' | 'internal' = 'external'
+): Promise<void> {
+  const token = getToolToken(workspace)
+  const serverEndpoint = (await getTransactorEndpoint(token, type))
+    .replaceAll('wss://', 'https://')
+    .replace('ws://', 'http://')
+
+  try {
+    console.info('send transactor event', operation, 'to', serverEndpoint)
+    await fetch(serverEndpoint + `/api/v1/manage?token=${token}&operation=${operation}`, {
+      method: 'PUT'
+    })
+  } catch (err: any) {
+    // Ignore error if transactor is not yet ready
+  }
+}

server/workspace-service/src/service.ts

@@ -616,7 +616,6 @@ export class WorkspaceWorker {
           KeepSnapshots: 7 * 12
         },
         pipelineFactory,
-        workspaceStorageAdapter,
         (ctx, workspace, branding, externalStorage) => {
           return getConfig(ctx, dbUrl, ctx, {
             externalStorage,
@@ -702,7 +701,6 @@ export class WorkspaceWorker {
         opt.backup.backupStorage,
         opt.backup.bucketName,
         pipelineFactory,
-        workspaceStorageAdapter,
         [DOMAIN_BLOB],
         true,
         (_p: number) => {

services/backup/backup-api-pod/src/server.ts

@@ -57,10 +57,12 @@ async function traverseBackup (
   ctx: MeasureContext,
   storage: BackupStorage,
   snapshot: (date: number) => void,
-  addFile: (name: string, size?: number) => void
+  addFile: (name: string, size?: number) => void,
+  addBlobInfo: (info: Record<string, [string, number]>) => void
 ): Promise<BackupInfo | undefined> {
   const infoFile = 'backup.json.gz'
   const sizeFile = 'backup.size.gz'
+  const blobInfoFile = 'blob-info.json.gz'
 
   const backupInfoFile = await storage.loadFile(infoFile)
 
@@ -80,6 +82,16 @@ async function traverseBackup (
     console.error('failed to load size file', err)
   }
 
+  let blobInfo: Record<string, [string, number]> = {}
+  try {
+    const blobInfoData = await storage.loadFile(blobInfoFile)
+    if (blobInfoData != null) {
+      blobInfo = JSON.parse(gunzipSync(blobInfoData).toString())
+    }
+  } catch (err: any) {
+    console.error('failed to load size file', err)
+  }
+
   const addFileSize = async (file: string | undefined | null): Promise<void> => {
     if (file != null) {
       addFile(file, sizeInfo?.[file])
@@ -101,6 +113,7 @@ async function traverseBackup (
   }
   sizeInfo[infoFile] = backupInfoFile.length
   await addFileSize(infoFile)
+  addBlobInfo(blobInfo)
   return backupInfo
 }
 
@@ -218,7 +231,8 @@ export async function createServer (ctx: MeasureContext, config: Config): Promis
             backupInfoHTML += `<span><a href="${name}">${name}</a> Size: ${size ?? 'unknown'}  <a href="${
               name + '.hash'
             }">etag</a><br/></span>\n`
-          }
+          },
+          (info) => {}
         )
       } catch (err: any) {
         backupInfoHTML += `Error: ${err.message}`
@@ -235,10 +249,12 @@ export async function createServer (ctx: MeasureContext, config: Config): Promis
     if (file === 'index.json') {
       const jsonData: {
         files: { name: string, size?: number }[]
+        extraBlobs: { name: string, size: number, contentType: string }[]
         error?: string
         info?: BackupInfo
       } = {
-        files: []
+        files: [],
+        extraBlobs: []
       }
 
       try {
@@ -248,6 +264,11 @@ export async function createServer (ctx: MeasureContext, config: Config): Promis
           (snapshot) => {},
           (name, size) => {
             jsonData.files.push({ name, size })
+          },
+          (info) => {
+            for (const [name, [contentType, size]] of Object.entries(info)) {
+              jsonData.extraBlobs.push({ name, size, contentType })
+            }
           }
         )
       } catch (err: any) {

tests/tool-local.sh

@@ -1,7 +1,5 @@
 export MODEL_VERSION=$(node ../common/scripts/show_version.js)
-export MINIO_ACCESS_KEY=minioadmin
-export MINIO_SECRET_KEY=minioadmin
-export MINIO_ENDPOINT=localhost:9000
+export STORAGE_CONFIG="datalake|http://localhost:4030"
 export MONGO_URL=mongodb://localhost:27017
 export DB_URL=postgresql://root@localhost:26257/defaultdb?sslmode=disable
 export ACCOUNT_DB_URL=postgresql://root@localhost:26257/defaultdb?sslmode=disable

tests/tool-pg.sh

@@ -1,9 +1,7 @@
 #!/usr/bin/env bash
 
 export MODEL_VERSION=$(node ../common/scripts/show_version.js)
-export MINIO_ACCESS_KEY=minioadmin
-export MINIO_SECRET_KEY=minioadmin
-export MINIO_ENDPOINT=localhost:9002
+export STORAGE_CONFIG="datalake|http://localhost:4031"
 export ACCOUNTS_URL=http://localhost:3003
 export TRANSACTOR_URL=ws://localhost:3334
 export ACCOUNT_DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable

tests/tool.sh

@@ -1,9 +1,7 @@
 #!/usr/bin/env bash
 
 export MODEL_VERSION=$(node ../common/scripts/show_version.js)
-export MINIO_ACCESS_KEY=minioadmin
-export MINIO_SECRET_KEY=minioadmin
-export MINIO_ENDPOINT=localhost:9002
+export STORAGE_CONFIG="datalake|http://localhost:4031"
 export ACCOUNTS_URL=http://localhost:3003
 export TRANSACTOR_URL=ws://localhost:3334
 export ACCOUNT_DB_URL=mongodb://localhost:27018