diff --git a/.vscode/launch.json b/.vscode/launch.json index 14a3d55d35..d37b883f27 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -43,7 +43,7 @@ "APM_SERVER_URL2": "http://localhost:8200", "METRICS_CONSOLE": "false", "METRICS_FILE": "${workspaceRoot}/metrics.txt", // Show metrics in console evert 30 seconds., - "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin", + "STORAGE_CONFIG": "datalake|http://huly.local:4030", "SERVER_SECRET": "secret", "ENABLE_CONSOLE": "true", "COLLABORATOR_URL": "ws://localhost:3078", @@ -83,11 +83,11 @@ // "DB_URL": "postgresql://postgres:example@localhost:5432", "DB_URL": "postgresql://root@localhost:26257/defaultdb?sslmode=disable", // "GREEN_URL": "http://huly.local:6767?token=secret", - "SERVER_PORT": "3332", + "SERVER_PORT": "3334", "APM_SERVER_URL2": "http://localhost:8200", "METRICS_CONSOLE": "false", "METRICS_FILE": "${workspaceRoot}/metrics.txt", // Show metrics in console evert 30 seconds., - "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin", + "STORAGE_CONFIG": "datalake|http://huly.local:4030", "SERVER_SECRET": "secret", "ENABLE_CONSOLE": "true", "COLLABORATOR_URL": "ws://localhost:3078", @@ -126,7 +126,7 @@ // "DB_URL": "postgresql://postgres:example@localhost:5432", "DB_URL": "postgresql://root@huly.local:26258/defaultdb?sslmode=disable", // "GREEN_URL": "http://huly.local:6767?token=secret", - "SERVER_PORT": "3335", + "SERVER_PORT": "3334", "METRICS_CONSOLE": "false", "DEBUG_PRINT_SQL": "true", "METRICS_FILE": "${workspaceRoot}/metrics.txt", // Show metrics in console evert 30 seconds., @@ -406,19 +406,30 @@ "name": "Debug backup tool", "type": "node", "request": "launch", - "args": ["src/__start.ts", "backup", "../../../hardware/dump/felix", "felix", "--full"], + "args": [ + "src/__start.ts", + "backup-restore", + "/Users/haiodo/Develop/private/platform/tests/sanity-ws", + "sanity-ws", + "--upgrade" + ], "env": { - "MINIO_ACCESS_KEY": "minioadmin", - "MINIO_SECRET_KEY": "minioadmin", - "MINIO_ENDPOINT": "localhost", - "SERVER_SECRET": "secret", + "MODEL_VERSION": "0.7.153", + "STORAGE_CONFIG": "datalake|http://huly.local:4030", + "ACCOUNTS_URL": "http://localhost:3003", + "TRANSACTOR_URL": "ws://localhost:3334", + "ACCOUNT_DB_URL": "postgresql://root@localhost:26258/defaultdb?sslmode=disable", "MONGO_URL": "mongodb://localhost:27018", - "ACCOUNTS_URL": "http://localhost:3003" + "ELASTIC_URL": "http://localhost:9201", + "SERVER_SECRET": "secret", + "DB_URL": "postgresql://root@localhost:26258/defaultdb?sslmode=disable", + "QUEUE_CONFIG": "localhost:19093;-staging" }, - "smartStep": true, - "sourceMapRenames": true, + "smartStep": false, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "sourceMaps": true, + "sourceMapRenames": false, + "skipFiles": ["/**"], "cwd": "${workspaceRoot}/dev/tool", "protocol": "inspector", "outputCapture": "std", diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 3191c6b184..65844a49a5 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -462,30 +462,30 @@ services: - HULY_TOKEN_SECRET=secret - HULY_ACCOUNT_SERVICE=http://huly.local:3000 restart: unless-stopped - # backup-cockroach: - # image: hardcoreeng/backup - # extra_hosts: - # - 'huly.local:host-gateway' - # depends_on: - # minio: - # condition: service_healthy - # cockroach: - # condition: service_started - # stats: - # condition: service_started - # account: - # condition: service_started - # environment: - # - SECRET=secret - # - ACCOUNTS_URL=http://huly.local:3000 - # - STATS_URL=http://huly.local:4900 - # - DB_URL=${DB_CR_URL} - # - REGION=cockroach - # - BUCKET_NAME=backups - # - INTERVAL=3600 - # - STORAGE=minio|minio?accessKey=minioadmin&secretKey=minioadmin - # - WORKSPACE_STORAGE=minio|minio?accessKey=minioadmin&secretKey=minioadmin - # restart: unless-stopped + backup-cockroach: + image: hardcoreeng/backup + extra_hosts: + - 'huly.local:host-gateway' + depends_on: + minio: + condition: service_healthy + cockroach: + condition: service_started + stats: + condition: service_started + account: + condition: service_started + environment: + - SECRET=secret + - ACCOUNTS_URL=http://huly.local:3000 + - STATS_URL=http://huly.local:4900 + - DB_URL=${DB_CR_URL} + - REGION=cockroach + - BUCKET_NAME=backups + - INTERVAL=60 + - STORAGE=minio|minio?accessKey=minioadmin&secretKey=minioadmin + - WORKSPACE_STORAGE=${STORAGE_CONFIG} + restart: unless-stopped # backup-api: # image: hardcoreeng/backup-api # extra_hosts: diff --git a/dev/tool/package.json b/dev/tool/package.json index 6f1f31c1b8..e245ba46ec 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -21,7 +21,7 @@ "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool", "run-local-mongo": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4700 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:19092' node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js", - "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 STORAGE_CONFIG='datalake|http://localhost:4030' ACCOUNT_DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 REGION_INFO='cockroach|CockroachDB' MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:19092' node --expose-gc --max-old-space-size=18000 $TOOL_OPT ./bundle/bundle.js", + "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 STORAGE_CONFIG='datalake|http://huly.local:4030' ACCOUNT_DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 REGION_INFO='cockroach|CockroachDB' MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:19092' node --expose-gc --max-old-space-size=18000 $TOOL_OPT ./bundle/bundle.js", "run-local-brk": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js", "run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js", "upgrade-mongo": "rushx run-local-mongo upgrade-workspace -- $1", diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index eb8ba864f8..eaa9adb502 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -35,10 +35,13 @@ import { compactBackup, createFileBackupStorage, createStorageBackupStorage, - restore + restore, + backupDownload } from '@hcengineering/server-backup' import serverClientPlugin, { getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client' import { + createBackupPipeline, + createEmptyBroadcastOps, registerAdapterFactory, registerDestroyFactory, registerServerPlugins, @@ -82,7 +85,6 @@ import { getMongoClient, shutdownMongo } from '@hcengineering/mongo' -import { backupDownload } from '@hcengineering/server-backup/src/backup' import { getModelVersion } from '@hcengineering/model-all' import { @@ -94,6 +96,7 @@ import { import { QueueTopic, workspaceEvents, + type Pipeline, type QueueWorkspaceMessage, type StorageAdapter } from '@hcengineering/server-core' @@ -114,6 +117,7 @@ import { performGmailAccountMigrations } from './gmail' import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils' import { createRestClient } from '@hcengineering/api-client' +import { sendTransactorEvent } from '@hcengineering/server-tool' import { existsSync } from 'fs' import { mkdir, writeFile } from 'fs/promises' import { dirname } from 'path' @@ -688,72 +692,6 @@ export function devTool ( // ) // } - // const destroyer = getWorkspaceDestroyAdapter(dbUrl) - - // program - // .command('restore-all') - // .description('Restore workspaces to selected region DB...') - // .option('-t|--timeout [timeout]', 'Timeout in days', '60') - // .option('-r|--region [region]', 'Timeout in days', '') - // .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '') - // .option('-d|--dry [dry]', 'Dry run', false) - // .action(async (cmd: { timeout: string, workspace: string, region: string, dry: boolean, account: string }) => { - // const { txes, dbUrl } = prepareTools() - - // const bucketName = process.env.BUCKET_NAME - // if (bucketName === '' || bucketName == null) { - // console.error('please provide butket name env') - // process.exit(1) - // } - - // const token = generateToken(systemAccountEmail, getWorkspaceId('')) - // const workspaces = (await listAccountWorkspaces(token, cmd.region)) - // .sort((a, b) => { - // const bsize = b.backupInfo?.backupSize ?? 0 - // const asize = a.backupInfo?.backupSize ?? 0 - // return bsize - asize - // }) - // .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) - - // for (const ws of workspaces) { - // const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - - // toolCtx.warn('--- restoring workspace', { - // url: ws.workspaceUrl, - // id: ws.workspace, - // lastVisitDays, - // backupSize: ws.backupInfo?.blobsSize ?? 0, - // mode: ws.mode - // }) - // if (cmd.dry) { - // continue - // } - // try { - // const st = Date.now() - // await backupRestore( - // toolCtx, - // dbUrl, - // bucketName, - // ws, - // (dbUrl, storageAdapter) => { - // const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { - // externalStorage: storageAdapter, - // usePassedCtx: true - // }) - // return factory - // }, - // [DOMAIN_BLOB] - // ) - // const ed = Date.now() - // toolCtx.warn('--- restoring complete', { - // time: ed - st - // }) - // } catch (err: any) { - // toolCtx.error('REstore of f workspace failedarchive workspace', { workspace: ws.workspace }) - // } - // } - // }) - // program // .command('backup-all') // .description('Backup all workspaces...') @@ -1004,6 +942,7 @@ export function devTool ( ) => { const storage = await createFileBackupStorage(dirName) await withAccountDatabase(async (db) => { + const { txes, dbUrl } = prepareTools() const ws = await getWorkspace(db, workspace) if (ws === null) { throw new Error(`workspace ${workspace} not found`) @@ -1013,21 +952,51 @@ export function devTool ( dataId: ws.dataId, url: ws.url } - const endpoint = await getWorkspaceTransactorEndpoint(ws.uuid) + const storageConfig = storageConfigFromEnv() - await backup(toolCtx, endpoint, wsIds, storage, { - force: cmd.force, - include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())), - skipDomains: (cmd.skip ?? '').split(';').map((it) => it.trim()), - timeout: 0, - connectTimeout: parseInt(cmd.timeout) * 1000, - blobDownloadLimit: parseInt(cmd.blobLimit), - skipBlobContentTypes: cmd.contentTypes - .split(';') - .map((it) => it.trim()) - .filter((it) => it.length > 0), - keepSnapshots: parseInt(cmd.keepSnapshots) - }) + const workspaceStorage: StorageAdapter = buildStorageFromConfig(storageConfig) + + let pipeline: Pipeline | undefined + try { + pipeline = await createBackupPipeline(toolCtx, dbUrl, txes, { + externalStorage: workspaceStorage, + usePassedCtx: true + })( + toolCtx, + { + uuid: ws.uuid, + url: ws.url ?? '', + dataId: ws.dataId + }, + createEmptyBroadcastOps(), + null + ) + if (pipeline === undefined) { + toolCtx.error('failed to restore, pipeline is undefined', { workspace }) + return + } + + await backup(toolCtx, pipeline, wsIds, storage, { + force: cmd.force, + include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())), + skipDomains: (cmd.skip ?? '').split(';').map((it) => it.trim()), + timeout: 0, + connectTimeout: parseInt(cmd.timeout) * 1000, + blobDownloadLimit: parseInt(cmd.blobLimit), + skipBlobContentTypes: cmd.contentTypes + .split(';') + .map((it) => it.trim()) + .filter((it) => it.length > 0), + keepSnapshots: parseInt(cmd.keepSnapshots) + }) + } catch (err: any) { + toolCtx.error('Failed to backup workspace', { err, workspace }) + } finally { + if (pipeline !== undefined) { + await pipeline.close() + } + await workspaceStorage.close() + } }) } ) @@ -1145,7 +1114,6 @@ export function devTool ( .option('-c, --recheck', 'Force hash recheck on server', false) .option('-i, --include ', 'A list of ; separated domain names to include during backup', '*') .option('-s, --skip ', 'A list of ; separated domain names to skip during backup', '') - .option('--use-storage ', 'Use workspace storage adapter from env variable', '') .option('--upgrade', 'Upgrade workspace', false) .option( '--history-file ', @@ -1170,6 +1138,7 @@ export function devTool ( } ) => { await withAccountDatabase(async (db) => { + const { txes, dbUrl } = prepareTools() const ws = await getWorkspace(db, workspaceId) if (ws === null) { throw new Error(`workspace ${workspaceId} not found`) @@ -1182,32 +1151,58 @@ export function devTool ( url: ws.url } const storage = await createFileBackupStorage(dirName) - const storageConfig = cmd.useStorage !== '' ? storageConfigFromEnv(process.env[cmd.useStorage]) : undefined + const storageConfig = storageConfigFromEnv() const queue = getPlatformQueue('tool', ws.region) const wsProducer = queue.getProducer(toolCtx, QueueTopic.Workspace) await wsProducer.send(ws.uuid, [workspaceEvents.restoring()]) - const workspaceStorage: StorageAdapter | undefined = - storageConfig !== undefined ? buildStorageFromConfig(storageConfig) : undefined - await restore(toolCtx, await getWorkspaceTransactorEndpoint(workspace), wsIds, storage, { - date: parseInt(date ?? '-1'), - merge: cmd.merge, - parallel: parseInt(cmd.parallel ?? '1'), - recheck: cmd.recheck, - include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';')), - skip: new Set(cmd.skip.split(';')), - storageAdapter: workspaceStorage, - historyFile: cmd.historyFile - }) + const workspaceStorage: StorageAdapter = buildStorageFromConfig(storageConfig) - if (cmd.upgrade) { - await doUpgrade(toolCtx, workspace, true, true) + let pipeline: Pipeline | undefined + try { + pipeline = await createBackupPipeline(toolCtx, dbUrl, txes, { + externalStorage: workspaceStorage, + usePassedCtx: true + })( + toolCtx, + { + uuid: ws.uuid, + url: ws.url ?? '', + dataId: ws.dataId + }, + createEmptyBroadcastOps(), + null + ) + if (pipeline === undefined) { + toolCtx.error('failed to restore, pipeline is undefined', { workspaceId }) + return + } + await sendTransactorEvent(workspace, 'force-maintenance') + + await restore(toolCtx, pipeline, wsIds, storage, { + date: parseInt(date ?? '-1'), + merge: cmd.merge, + parallel: parseInt(cmd.parallel ?? '1'), + recheck: cmd.recheck, + include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';')), + skip: new Set(cmd.skip.split(';')), + historyFile: cmd.historyFile + }) + + if (cmd.upgrade) { + await doUpgrade(toolCtx, workspace, true, true) + } else { + await sendTransactorEvent(workspace, 'force-close') + } + + console.log('workspace restored') + await wsProducer.send(ws.uuid, [workspaceEvents.restored()]) + } catch (err) { + toolCtx.error('failed to restore', { err }) } - - console.log('workspace restored') - await wsProducer.send(ws.uuid, [workspaceEvents.restored()]) + await pipeline?.close() await queue.shutdown() await workspaceStorage?.close() }) diff --git a/dev/tool/src/utils.ts b/dev/tool/src/utils.ts index ba297c9ed7..1bad8441f8 100644 --- a/dev/tool/src/utils.ts +++ b/dev/tool/src/utils.ts @@ -1,7 +1,5 @@ import { getWorkspaceById, getWorkspaceByUrl, type AccountDB, type Workspace } from '@hcengineering/account' import { - systemAccountUuid, - type WorkspaceUuid, type AttachedData, type AttachedDoc, type Class, @@ -9,10 +7,9 @@ import { type DocumentUpdate, type Ref, type Space, - type TxOperations + type TxOperations, + type WorkspaceUuid } from '@hcengineering/core' -import { getTransactorEndpoint } from '@hcengineering/server-client' -import { generateToken } from '@hcengineering/server-token' export async function findOrUpdateAttached ( client: TxOperations, @@ -61,13 +58,4 @@ export async function getWorkspace (db: AccountDB, workspace: string): Promise { - return await getTransactorEndpoint(getToolToken(workspace), type) -} +export { getToolToken, getWorkspaceTransactorEndpoint } from '@hcengineering/server-tool' diff --git a/dev/tool/src/workspace.ts b/dev/tool/src/workspace.ts index ef955129a3..e790dbc1fe 100644 --- a/dev/tool/src/workspace.ts +++ b/dev/tool/src/workspace.ts @@ -20,26 +20,13 @@ import core, { type Client as CoreClient, type Doc, DOMAIN_TX, - type MeasureContext, type Ref, type Tx, - type WorkspaceDataId, - type WorkspaceIds, - type WorkspaceInfoWithStatus, type WorkspaceUuid } from '@hcengineering/core' import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' -import { createStorageBackupStorage, restore } from '@hcengineering/server-backup' -import { - createDummyStorageAdapter, - type PipelineFactory, - type StorageAdapter, - wrapPipeline -} from '@hcengineering/server-core' -import { createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { connect } from '@hcengineering/server-tool' import { generateModelDiff, printDiff } from './mdiff' -import { createEmptyBroadcastOps } from '@hcengineering/server-pipeline' export async function diffWorkspace (mongoUrl: string, dbName: string, rawTxes: Tx[]): Promise { const client = getMongoClient(mongoUrl) @@ -117,60 +104,3 @@ export async function updateField ( await connection.close() } } - -export async function backupRestore ( - ctx: MeasureContext, - dbURL: string, - bucketName: string, - workspace: WorkspaceInfoWithStatus, - pipelineFactoryFactory: (mongoUrl: string, storage: StorageAdapter) => PipelineFactory, - skipDomains: string[] -): Promise { - const storageEnv = process.env.STORAGE - if (storageEnv === undefined) { - console.error('please provide STORAGE env') - process.exit(1) - } - if (bucketName.trim() === '') { - console.error('please provide butket name env') - process.exit(1) - } - const backupStorageConfig = storageConfigFromEnv(storageEnv) - - const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - - const workspaceStorage = createDummyStorageAdapter() - const pipelineFactory = pipelineFactoryFactory(dbURL, workspaceStorage) - - try { - const storage = await createStorageBackupStorage( - ctx, - storageAdapter, - { - uuid: 'backup' as WorkspaceUuid, - url: bucketName, - dataId: bucketName as WorkspaceDataId - }, - workspace.dataId ?? workspace.uuid - ) - const wsUrl: WorkspaceIds = { - uuid: workspace.uuid, - dataId: workspace.dataId, - url: workspace.url - } - const result: boolean = await ctx.with('restore', {}, (ctx) => - restore(ctx, '', wsUrl, storage, { - date: -1, - skip: new Set(skipDomains), - recheck: false, - storageAdapter: workspaceStorage, - getConnection: async () => { - return wrapPipeline(ctx, await pipelineFactory(ctx, wsUrl, createEmptyBroadcastOps(), null), wsUrl) - } - }) - ) - return result - } finally { - await storageAdapter.close() - } -} diff --git a/plugins/setting-assets/lang/cs.json b/plugins/setting-assets/lang/cs.json index 84aed77251..847f698226 100644 --- a/plugins/setting-assets/lang/cs.json +++ b/plugins/setting-assets/lang/cs.json @@ -132,6 +132,7 @@ "BackupFileDownload": "Stáhnout soubor", "BackupFiles": "Soubory záloh", "BackupNoBackup": "V současné době nejsou k dispozici žádné zálohy.", + "NonBackupedBlobs": "Non Backed up blobs", "Calendar": "Kalendář", "StartOfTheWeek": "Začátek týdne", "SystemSetupString": "Nastavení systému ({day})", diff --git a/plugins/setting-assets/lang/de.json b/plugins/setting-assets/lang/de.json index d783c99c43..099987fdee 100644 --- a/plugins/setting-assets/lang/de.json +++ b/plugins/setting-assets/lang/de.json @@ -132,6 +132,7 @@ "BackupFileDownload": "Datei herunterladen", "BackupFiles": "Sicherungsdateien", "BackupNoBackup": "Derzeit sind keine Sicherungen verfügbar.", + "NonBackupedBlobs": "Nicht gesicherte Blobs", "Calendar": "Kalender", "StartOfTheWeek": "Wochenstart", "SystemSetupString": "Systemkonfiguration ({day})", diff --git a/plugins/setting-assets/lang/en.json b/plugins/setting-assets/lang/en.json index e64799204b..e0e00d97db 100644 --- a/plugins/setting-assets/lang/en.json +++ b/plugins/setting-assets/lang/en.json @@ -132,6 +132,7 @@ "BackupFileDownload": "Download File", "BackupFiles": "Backup Files", "BackupNoBackup": "No backups are currently available.", + "NonBackupedBlobs": "Non Backed up blobs", "Calendar": "Calendar", "StartOfTheWeek": "Start of the week", "SystemSetupString": "System Setup ({day})", diff --git a/plugins/setting-assets/lang/es.json b/plugins/setting-assets/lang/es.json index 815d1172e4..fea4e9b12e 100644 --- a/plugins/setting-assets/lang/es.json +++ b/plugins/setting-assets/lang/es.json @@ -123,6 +123,7 @@ "BackupFileDownload": "Descargar archivo", "BackupFiles": "Archivos de copia de seguridad", "BackupNoBackup": "No hay copias de seguridad disponibles actualmente.", + "NonBackupedBlobs": "Blobs no respaldados", "Calendar": "Calendario", "StartOfTheWeek": "Inicio de la semana", "SystemSetupString": "Configuración del sistema ({day})", diff --git a/plugins/setting-assets/lang/fr.json b/plugins/setting-assets/lang/fr.json index 39a674fb8f..1030ee2943 100644 --- a/plugins/setting-assets/lang/fr.json +++ b/plugins/setting-assets/lang/fr.json @@ -132,12 +132,13 @@ "BackupFileDownload": "Télécharger le fichier", "BackupFiles": "Fichiers de sauvegarde", "BackupNoBackup": "Aucune sauvegarde n'est actuellement disponible.", + "NonBackupedBlobs": "Non Backed up blobs", "Calendar": "Calendrier", "StartOfTheWeek": "Début de semaine", "SystemSetupString": "Configuration du système ({day})", "DefaultString": "Par défaut ({day})", "AddAttribute": "Ajouter un attribut", - "WorkspaceNamePattern": "Le nom doit comporter 40 caractères ou moins, ne doit pas être vide et ne doit pas contenir de caractères spéciaux (<, >, /)" , + "WorkspaceNamePattern": "Le nom doit comporter 40 caractères ou moins, ne doit pas être vide et ne doit pas contenir de caractères spéciaux (<, >, /)", "Mailboxes": "Boîtes aux lettres", "CreateMailbox": "Créer une boîte aux lettres", "CreateMailboxPlaceholder": "my-cool-name", diff --git a/plugins/setting-assets/lang/it.json b/plugins/setting-assets/lang/it.json index c2171a31c5..efbad7ca67 100644 --- a/plugins/setting-assets/lang/it.json +++ b/plugins/setting-assets/lang/it.json @@ -132,6 +132,7 @@ "BackupFileDownload": "Scarica file", "BackupFiles": "File di backup", "BackupNoBackup": "Non sono attualmente disponibili backup.", + "NonBackupedBlobs": "Non Backed up blobs", "Calendar": "Calendario", "StartOfTheWeek": "Inizio settimana", "SystemSetupString": "Configurazione del sistema ({day})", diff --git a/plugins/setting-assets/lang/ja.json b/plugins/setting-assets/lang/ja.json index 22de00ff53..49eb009a1c 100644 --- a/plugins/setting-assets/lang/ja.json +++ b/plugins/setting-assets/lang/ja.json @@ -132,6 +132,7 @@ "BackupFileDownload": "ファイルをダウンロード", "BackupFiles": "バックアップファイル", "BackupNoBackup": "現在、利用可能なバックアップはありません。", + "NonBackupedBlobs": "バックアップされていないブロブ", "Calendar": "カレンダー", "StartOfTheWeek": "週の開始", "SystemSetupString": "システム設定 ({day})", diff --git a/plugins/setting-assets/lang/pt.json b/plugins/setting-assets/lang/pt.json index 1a5be24afc..1f6ba55deb 100644 --- a/plugins/setting-assets/lang/pt.json +++ b/plugins/setting-assets/lang/pt.json @@ -123,6 +123,7 @@ "BackupFileDownload": "Baixar Arquivo", "BackupFiles": "Arquivos de Backup", "BackupNoBackup": "Não há backups disponíveis no momento.", + "NonBackupedBlobs": "Non Backed up blobs", "Calendar": "Calendário", "StartOfTheWeek": "Início da semana", "SystemSetupString": "Configuração do sistema ({day})", diff --git a/plugins/setting-assets/lang/ru.json b/plugins/setting-assets/lang/ru.json index 4aac30db3c..0f92e7de04 100644 --- a/plugins/setting-assets/lang/ru.json +++ b/plugins/setting-assets/lang/ru.json @@ -132,6 +132,7 @@ "BackupFileDownload": "Скачать файл", "BackupFiles": "Файлы резервной копии", "BackupNoBackup": "В настоящее время резервные копии недоступны.", + "NonBackupedBlobs": "Не резервируемые блобы", "Calendar": "Календарь", "StartOfTheWeek": "Начало недели", "SystemSetupString": "Системная настройка ({day})", diff --git a/plugins/setting-assets/lang/zh.json b/plugins/setting-assets/lang/zh.json index e4a8bef6ce..a1e169b5d9 100644 --- a/plugins/setting-assets/lang/zh.json +++ b/plugins/setting-assets/lang/zh.json @@ -132,6 +132,7 @@ "BackupFileDownload": "下载文件", "BackupFiles": "备份文件", "BackupNoBackup": "当前没有可用的备份。", + "NonBackupedBlobs": "未备份的 Blob", "Calendar": "日历", "StartOfTheWeek": "本周开始", "SystemSetupString": "系统设置({day})", diff --git a/plugins/setting-resources/src/components/Backup.svelte b/plugins/setting-resources/src/components/Backup.svelte index 7151287ccc..b325d37b52 100644 --- a/plugins/setting-resources/src/components/Backup.svelte +++ b/plugins/setting-resources/src/components/Backup.svelte @@ -15,7 +15,7 @@
@@ -281,7 +295,6 @@
{backupInfo.error}
{/if} -
@@ -312,13 +325,13 @@
+ + +
+
+ +
+ {#each blobs as file} +
+
{file.name}
+
+ {getSize(file.size)} + {file.contentType} +
+
+
+
+ {/each} +
+ {/if} diff --git a/plugins/setting/src/index.ts b/plugins/setting/src/index.ts index 8f24ec0a0f..024abe5975 100644 --- a/plugins/setting/src/index.ts +++ b/plugins/setting/src/index.ts @@ -233,6 +233,7 @@ export default plugin(settingId, { BackupFileDownload: '' as IntlString, BackupFiles: '' as IntlString, BackupNoBackup: '' as IntlString, + NonBackupedBlobs: '' as IntlString, AddAttribute: '' as IntlString, Mailboxes: '' as IntlString, CreateMailbox: '' as IntlString, diff --git a/pods/server/src/server_http.ts b/pods/server/src/server_http.ts index aae3efc569..374b963c16 100644 --- a/pods/server/src/server_http.ts +++ b/pods/server/src/server_http.ts @@ -247,14 +247,21 @@ export function startHttpServer ( case 'profile-stop': { profiling = false if (sessions.profiling?.stop != null) { - void sessions.profiling.stop().then((profile) => { - ctx.warn( - '---------------------------------------------PROFILING SESSION STOPPED---------------------------------------------', - {} - ) - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(profile ?? '{ error: "no profiling" }') - }) + void sessions.profiling + .stop() + .then((profile) => { + ctx.warn( + '---------------------------------------------PROFILING SESSION STOPPED---------------------------------------------', + {} + ) + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(profile ?? '{ error: "no profiling" }') + }) + .catch((err) => { + ctx.error('error', { err }) + res.writeHead(500) + res.end() + }) } else { res.writeHead(404) res.end() @@ -262,9 +269,20 @@ export function startHttpServer ( return } + case 'force-maintenance': { + const wsId = req.query.wsId as WorkspaceUuid + void sessions.forceMaintenance(ctx, wsId ?? payload.workspace).catch((err) => { + ctx.error('error', { err }) + }) + res.writeHead(200) + res.end() + return + } case 'force-close': { const wsId = req.query.wsId as WorkspaceUuid - void sessions.forceClose(wsId ?? payload.workspace) + void sessions.forceClose(wsId ?? payload.workspace).catch((err) => { + ctx.error('error', { err }) + }) res.writeHead(200) res.end() return diff --git a/server/backup-service/src/index.ts b/server/backup-service/src/index.ts index d0f59eede5..beca2d62d6 100644 --- a/server/backup-service/src/index.ts +++ b/server/backup-service/src/index.ts @@ -13,9 +13,9 @@ // limitations under the License. // import { - type MeasureContext, systemAccountUuid, type Branding, + type MeasureContext, type WorkspaceIds, type WorkspaceInfoWithStatus } from '@hcengineering/core' @@ -62,7 +62,6 @@ export function startBackup ( storageAdapter, { ...config, Token: token }, pipelineFactory, - workspaceStorageAdapter, (ctx, workspace, branding, externalStorage) => { return getConfig(ctx, mainDbUrl, workspace, branding, externalStorage) }, @@ -122,7 +121,6 @@ export async function backupWorkspace ( storageAdapter, { ...config, Token: token }, pipelineFactory, - workspaceStorageAdapter, (ctx, workspace, branding, externalStorage) => { return getConfig(ctx, mainDbUrl, workspace, branding, externalStorage) }, diff --git a/server/backup/src/backup.ts b/server/backup/src/backup.ts index 837a14d46e..db09888f61 100644 --- a/server/backup/src/backup.ts +++ b/server/backup/src/backup.ts @@ -14,10 +14,7 @@ // limitations under the License. // -import { Analytics } from '@hcengineering/analytics' import core, { - BackupClient, - Client as CoreClient, Doc, Domain, DOMAIN_BLOB, @@ -26,661 +23,52 @@ import core, { DOMAIN_TRANSIENT, DOMAIN_TX, MeasureContext, - MeasureMetricsContext, - RateLimiter, Ref, SortingOrder, - systemAccountUuid, toIdMap, TxProcessor, - type BackupStatus, type Blob, + type LowLevelStorage, type Tx, type TxCUD, - type WorkspaceIds, - type WorkspaceUuid + type WorkspaceIds } from '@hcengineering/core' -import { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client' -import { estimateDocSize, type StorageAdapter } from '@hcengineering/server-core' -import { generateToken } from '@hcengineering/server-token' +import { BlobClient } from '@hcengineering/server-client' +import { BackupClientOps, createDummyStorageAdapter, estimateDocSize, type Pipeline } from '@hcengineering/server-core' import { deepEqual } from 'fast-equals' -import { - createReadStream, - createWriteStream, - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - statSync, - writeFileSync -} from 'node:fs' +import { createReadStream, createWriteStream, mkdtempSync } from 'node:fs' import { rm } from 'node:fs/promises' -import { basename, dirname } from 'node:path' +import { basename } from 'node:path' import { PassThrough } from 'node:stream' import { createGzip } from 'node:zlib' import { join } from 'path' -import { Writable } from 'stream' -import { extract, Pack, pack } from 'tar-stream' -import { createGunzip, gunzipSync, gzipSync } from 'zlib' +import { Pack, pack } from 'tar-stream' +import { gunzipSync, gzipSync } from 'zlib' import { BackupStorage } from './storage' +import { type BackupInfo, type BackupResult, type BackupSnapshot, type DomainData, type Snapshot } from './types' +import { + checkBackupIntegrity, + compactBackup, + doTrimHash, + extendZero, + loadDigest, + rebuildSizeInfo, + verifyDocsFromSnapshot, + writeChanges +} from './utils' export * from './storage' const dataBlobSize = 250 * 1024 * 1024 -const dataUploadSize = 2 * 1024 * 1024 const batchSize = 5000 const defaultLevel = 9 -/** - * Blob data from s3 storage - * @public - */ -interface BlobData extends Doc { - name: string - size: number - type: string - provider?: string // If node defined, will be default one - base64Data: string // base64 encoded data -} - -/** - * @public - */ -export interface Snapshot { - added: Map, string> - updated: Map, string> - removed: Ref[] -} - -/** - * @public - */ -export interface SnapshotV6 { - added: Record, string> - updated: Record, string> - removed: Ref[] -} - -/** - * @public - */ -export interface DomainData { - snapshot?: string // 0.6 json snapshot - snapshots?: string[] - storage?: string[] - - // Some statistics - added: number - updated: number - removed: number -} - -/** - * @public - */ -export interface BackupSnapshot { - // _id => hash of added items. - domains: Record - date: number - - compacting?: boolean - stIndex: number // Snapshot index -} - -/** - * @public - */ -export interface BackupInfo { - workspace: WorkspaceUuid - version: string - snapshots: BackupSnapshot[] - snapshotsIndex?: number - lastTxId?: string - - // A hash of current domain transactions, so we could skip all other checks if same. - domainHashes: Record - - migrations: Record -} - -async function loadDigest ( - ctx: MeasureContext, - storage: BackupStorage, - snapshots: BackupSnapshot[], - domain: Domain, - date?: number -): Promise, string>> { - const result = new Map, string>() - for (const s of snapshots) { - const d = s.domains[domain] - - // Load old JSON snapshot - if (d?.snapshot !== undefined) { - try { - const dChanges: SnapshotV6 = JSON.parse( - gunzipSync(new Uint8Array(await storage.loadFile(d.snapshot))).toString() - ) - for (const [k, v] of Object.entries(dChanges.added)) { - result.set(k as Ref, v) - } - for (const [k, v] of Object.entries(dChanges.updated)) { - result.set(k as Ref, v) - } - for (const d of dChanges.removed) { - result.delete(d) - } - } catch (err: any) { - ctx.warn('failed to load digest', { snapshot: d.snapshot }) - } - } - for (const snapshot of d?.snapshots ?? []) { - try { - const dataBlob = gunzipSync(new Uint8Array(await storage.loadFile(snapshot))) - .toString() - .split('\n') - const addedCount = parseInt(dataBlob.shift() ?? '0') - const added = dataBlob.splice(0, addedCount) - for (const it of added) { - const [k, v] = it.split(';') - result.set(k as Ref, v) - } - - const updatedCount = parseInt(dataBlob.shift() ?? '0') - const updated = dataBlob.splice(0, updatedCount) - for (const it of updated) { - const [k, v] = it.split(';') - result.set(k as Ref, v) - } - - const removedCount = parseInt(dataBlob.shift() ?? '0') - const removed = dataBlob.splice(0, removedCount) - for (const k of removed) { - result.delete(k as Ref) - } - } catch (err: any) { - ctx.warn('digest is broken', { domain, err: err.message, snapshot }) - } - } - // Stop if stop date is matched and provided - if (date !== undefined && date === s.date) { - break - } - } - // ctx.info('load-digest', { domain, snapshots: snapshots.length, documents: result.size }) - return result -} -async function verifyDigest ( - ctx: MeasureContext, - storage: BackupStorage, - snapshots: BackupSnapshot[], - domain: Domain -): Promise<{ modified: boolean, modifiedFiles: string[] }> { - ctx = ctx.newChild('verify digest', { domain, count: snapshots.length }) - ctx.info('verify-digest', { domain, count: snapshots.length }) - let modified = false - const modifiedFiles: string[] = [] - for (const s of snapshots) { - const d = s.domains[domain] - if (d === undefined) { - continue - } - - const storageToRemove = new Set() - // We need to verify storage has all necessary resources - ctx.info('checking', { domain }) - // We have required documents here. - const validDocs = new Set>() - const zeroEntres = new Set>() - - for (const sf of d.storage ?? []) { - const blobs = new Map() - try { - ctx.info('checking storage', { sf }) - const readStream = await storage.load(sf) - const ex = extract() - - ex.on('entry', (headers, stream, next) => { - const name = headers.name ?? '' - // We found blob data - if (name.endsWith('.json')) { - const chunks: Buffer[] = [] - const bname = name.substring(0, name.length - 5) - stream.on('data', (chunk) => { - chunks.push(chunk) - }) - stream.on('end', () => { - try { - const bf = Buffer.concat(chunks as any) - const doc = JSON.parse(bf.toString()) as Doc - if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') { - const data = migradeBlobData(doc as Blob, '') - const d = blobs.get(bname) ?? (data !== '' ? Buffer.from(data, 'base64') : undefined) - if (d === undefined) { - blobs.set(bname, { doc, buffer: undefined }) - } else { - blobs.delete(bname) - } - } - validDocs.add(bname as Ref) - } catch (err: any) { - // If not a json, skip - } - next() - }) - } else { - if (headers.size === 0) { - zeroEntres.add(name as any) - } - next() - } - stream.resume() // just auto drain the stream - }) - - const unzip = createGunzip({ level: defaultLevel }) - const endPromise = new Promise((resolve, reject) => { - readStream.on('error', (err) => { - console.error(err) - reject(err) - }) - ex.on('finish', () => { - resolve(null) - }) - unzip.on('error', (err) => { - ctx.error('error during reading of', { sf, err }) - modified = true - storageToRemove.add(sf) - resolve(null) - }) - }) - - readStream.on('end', () => { - readStream.destroy() - }) - readStream.pipe(unzip) - unzip.pipe(ex) - - await endPromise - } catch (err: any) { - ctx.error('error during reading of', { sf, err }) - // In case of invalid archive, we need to - // We need to remove broken storage file - modified = true - storageToRemove.add(sf) - } - } - - // Clear zero files, they potentially wrong downloaded. - for (const zz of zeroEntres.values()) { - validDocs.delete(zz) - } - if (storageToRemove.size > 0) { - modified = true - d.storage = (d.storage ?? []).filter((it) => !storageToRemove.has(it)) - modifiedFiles.push(...Array.from(storageToRemove)) - for (const sf of storageToRemove) { - await storage.delete(sf) - } - } - let mfiles: string[] = [] - ;({ modified, modifiedFiles: mfiles } = await updateDigest(d, ctx, storage, validDocs, modified, domain)) - modifiedFiles.push(...mfiles) - } - ctx.end() - return { modified, modifiedFiles } -} - -async function updateDigest ( - d: DomainData, - ctx: MeasureContext, - storage: BackupStorage, - validDocs: Set>, - modified: boolean, - domain: Domain -): Promise<{ modified: boolean, modifiedFiles: string[] }> { - const digestToRemove = new Set() - const modifiedFiles: string[] = [] - for (const snapshot of d?.snapshots ?? []) { - try { - ctx.info('checking', { snapshot }) - const changes: Snapshot = { - added: new Map(), - removed: [], - updated: new Map() - } - let lmodified = false - try { - const dataBlob = gunzipSync(new Uint8Array(await storage.loadFile(snapshot))) - .toString() - .split('\n') - const addedCount = parseInt(dataBlob.shift() ?? '0') - const added = dataBlob.splice(0, addedCount) - for (const it of added) { - const [k, v] = it.split(';') - if (validDocs.has(k as any)) { - changes.added.set(k as Ref, v) - } else { - lmodified = true - } - } - - const updatedCount = parseInt(dataBlob.shift() ?? '0') - const updated = dataBlob.splice(0, updatedCount) - for (const it of updated) { - const [k, v] = it.split(';') - if (validDocs.has(k as any)) { - changes.updated.set(k as Ref, v) - } else { - lmodified = true - } - } - - const removedCount = parseInt(dataBlob.shift() ?? '0') - const removed = dataBlob.splice(0, removedCount) - changes.removed = removed as Ref[] - if (addedCount === 0 && removedCount === 0 && updatedCount === 0) { - // Empty digest, need to clean - digestToRemove.add(snapshot) - lmodified = true - } - } catch (err: any) { - ctx.warn('failed during processing of snapshot file, it will be skipped', { snapshot }) - digestToRemove.add(snapshot) - modified = true - } - - if (lmodified) { - modified = true - if (digestToRemove.has(snapshot)) { - await storage.delete(snapshot) // No need for digest, lets' remove it - } else { - // Store changes without missing files - await writeChanges(storage, snapshot, changes) - } - } - } catch (err: any) { - digestToRemove.add(snapshot) - modifiedFiles.push(snapshot) - ctx.warn('digest is broken', { domain, err: err.message, snapshot }) - modified = true - } - } - d.snapshots = (d.snapshots ?? []).filter((it) => !digestToRemove.has(it)) - return { modified, modifiedFiles } -} - -async function write (chunk: any, stream: Writable): Promise { - let needDrain = false - await new Promise((resolve, reject) => { - needDrain = !stream.write(chunk, (err) => { - if (err != null) { - reject(err) - } else { - resolve(null) - } - }) - }) - if (needDrain) { - await new Promise((resolve) => stream.once('drain', resolve)) - } -} - -async function writeChanges (storage: BackupStorage, snapshot: string, changes: Snapshot): Promise { - const snapshotWritable = await storage.write(snapshot) - const writable = createGzip({ level: defaultLevel }) - writable.pipe(snapshotWritable) - - // Write size - await write(`${changes.added.size}\n`, writable) - for (const [k, v] of changes.added.entries()) { - await write(`${k};${v}\n`, writable) - } - await write(`${changes.updated.size}\n`, writable) - for (const [k, v] of changes.updated.entries()) { - await write(`${k};${v}\n`, writable) - } - await write(`${changes.removed.length}\n`, writable) - for (const k of changes.removed) { - await write(`${k}\n`, writable) - } - writable.end() - await new Promise((resolve) => { - writable.flush(() => { - resolve(null) - }) - }) -} - -/** - * @public - */ -export async function cloneWorkspace ( - ctx: MeasureContext, - transactorUrl: string, - sourceWorkspaceId: WorkspaceUuid, - targetWorkspaceId: WorkspaceUuid, - clearTime: boolean = true, - progress: (value: number) => Promise, - storageAdapter: StorageAdapter -): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // await ctx.with( - // 'clone-workspace', - // {}, - // async (ctx) => { - // const sourceConnection = await ctx.with( - // 'connect-source', - // {}, - // async (ctx) => - // (await connect(transactorUrl, sourceWorkspaceId, undefined, { - // mode: 'backup' - // })) as unknown as CoreClient & BackupClient - // ) - // const targetConnection = await ctx.with( - // 'connect-target', - // {}, - // async (ctx) => - // (await connect(transactorUrl, targetWorkspaceId, undefined, { - // mode: 'backup', - // model: 'upgrade', - // admin: 'true' - // })) as unknown as CoreClient & BackupClient - // ) - // try { - // const domains = sourceConnection - // .getHierarchy() - // .domains() - // .filter((it) => it !== DOMAIN_TRANSIENT && it !== DOMAIN_MODEL) - - // let i = 0 - // for (const c of domains) { - // ctx.info('clone domain...', { domain: c, workspace: targetWorkspaceId }) - - // // We need to clean target connection before copying something. - // await ctx.with('clean-domain', { domain: c }, (ctx) => cleanDomain(ctx, targetConnection, c)) - - // const changes: Snapshot = { - // added: new Map(), - // updated: new Map(), - // removed: [] - // } - - // let idx: number | undefined - - // // update digest tar - // const needRetrieveChunks: Ref[][] = [] - - // let processed = 0 - // let domainProgress = 0 - // let st = Date.now() - // // Load all digest from collection. - // await ctx.with('retrieve-domain-info', { domain: c }, async (ctx) => { - // while (true) { - // try { - // const it = await ctx.with('load-chunk', {}, () => sourceConnection.loadChunk(c, idx)) - // idx = it.idx - - // let needRetrieve: Ref[] = [] - - // for (const { id, hash } of it.docs) { - // processed++ - // if (Date.now() - st > 2500) { - // ctx.info('processed', { processed, time: Date.now() - st, workspace: targetWorkspaceId }) - // st = Date.now() - // } - - // changes.added.set(id as Ref, hash) - // needRetrieve.push(id as Ref) - - // if (needRetrieve.length > 200) { - // needRetrieveChunks.push(needRetrieve) - // needRetrieve = [] - // } - // } - // if (needRetrieve.length > 0) { - // needRetrieveChunks.push(needRetrieve) - // } - // if (it.finished) { - // ctx.info('processed-end', { processed, time: Date.now() - st, workspace: targetWorkspaceId }) - // await ctx.with('close-chunk', {}, async () => { - // await sourceConnection.closeChunk(idx as number) - // }) - // break - // } - // } catch (err: any) { - // ctx.error('failed to clone', { err, workspace: targetWorkspaceId }) - // if (idx !== undefined) { - // await ctx.with('load-chunk', {}, () => sourceConnection.closeChunk(idx as number)) - // } - // // Try again - // idx = undefined - // processed = 0 - // } - // } - // }) - // await ctx.with('clone-domain', { domain: c }, async (ctx) => { - // while (needRetrieveChunks.length > 0) { - // const needRetrieve = needRetrieveChunks.shift() as Ref[] - - // ctx.info('Retrieve chunk:', { count: needRetrieve.length }) - // let docs: Doc[] = [] - // try { - // docs = await ctx.with('load-docs', {}, (ctx) => sourceConnection.loadDocs(c, needRetrieve)) - // if (clearTime) { - // docs = prepareClonedDocuments(docs) - // } - // const executor = new RateLimiter(10) - // for (const d of docs) { - // if (d._class === core.class.Blob) { - // const blob = d as Blob - // await executor.add(async () => { - // try { - // ctx.info('clone blob', { name: blob._id, contentType: blob.contentType }) - // const readable = await storageAdapter.get(ctx, sourceWorkspaceId, blob._id) - // const passThrue = new PassThrough() - // readable.pipe(passThrue) - // await storageAdapter.put( - // ctx, - // targetWorkspaceId, - // blob._id, - // passThrue, - // blob.contentType, - // blob.size - // ) - // } catch (err: any) { - // Analytics.handleError(err) - // console.error(err) - // } - // domainProgress++ - // await progress((100 / domains.length) * i + (100 / domains.length / processed) * domainProgress) - // }) - // } else { - // domainProgress++ - // } - // } - // await executor.waitProcessing() - // await ctx.with('upload-docs', {}, (ctx) => targetConnection.upload(c, docs), { length: docs.length }) - // await progress((100 / domains.length) * i + (100 / domains.length / processed) * domainProgress) - // } catch (err: any) { - // console.log(err) - // Analytics.handleError(err) - // // Put back. - // needRetrieveChunks.push(needRetrieve) - // continue - // } - // } - // }) - - // i++ - // await progress((100 / domains.length) * i) - // } - // } catch (err: any) { - // console.error(err) - // Analytics.handleError(err) - // } finally { - // ctx.info('end clone') - // await ctx.with('close-source', {}, async (ctx) => { - // await sourceConnection.close() - // }) - // await ctx.with('close-target', {}, async (ctx) => { - // await targetConnection.sendForceClose() - // await targetConnection.close() - // }) - // } - // }, - // { - // source: sourceWorkspaceId, - // target: targetWorkspaceId - // } - // ) -} - -// async function cleanDomain (ctx: MeasureContext, connection: CoreClient & BackupClient, domain: Domain): Promise { -// // Load all digest from collection. -// let idx: number | undefined -// const ids: Ref[] = [] -// while (true) { -// try { -// const it = await connection.loadChunk(domain, idx) -// idx = it.idx - -// ids.push(...it.docs.map((it) => it.id as Ref)) -// if (it.finished) { -// break -// } -// } catch (err: any) { -// console.error(err) -// if (idx !== undefined) { -// await connection.closeChunk(idx) -// } -// } -// } -// while (ids.length > 0) { -// const part = ids.splice(0, 5000) -// await connection.clean(domain, part) -// } -// } - -function doTrimHash (s: string | undefined): string | undefined { - if (s == null) { - return undefined - } - if (s.startsWith('"') && s.endsWith('"')) { - return s.slice(1, s.length - 1) - } - return s -} - -export interface BackupResult extends Omit { - result: boolean -} - /** * @public */ export async function backup ( ctx: MeasureContext, - transactorUrl: string, + pipeline: Pipeline, wsIds: WorkspaceIds, storage: BackupStorage, options: { @@ -691,9 +79,6 @@ export async function backup ( connectTimeout: number skipBlobContentTypes: string[] blobDownloadLimit: number - getLastTx?: () => Promise - getConnection?: () => Promise - storageAdapter?: StorageAdapter // Return true in case isCanceled?: () => boolean progress?: (progress: number) => Promise @@ -738,11 +123,13 @@ export async function backup ( } const st = Date.now() - let connection!: CoreClient & BackupClient - let printEnd = true + const connection = new BackupClientOps(pipeline.context.lowLevelStorage as LowLevelStorage) + const printEnd = true const tmpRoot = mkdtempSync('huly') + const forcedFullCheck = '2' + try { let backupInfo: BackupInfo = { workspace: workspaceId, @@ -751,14 +138,21 @@ export async function backup ( domainHashes: {}, migrations: { zeroCheckSize: true, // Assume already checked for new backups - forcedFullCheck: '1' // A force to full recheck. - } + forcedFullCheck // A force to full recheck. + }, + dataSize: 0, + blobsSize: 0, + backupSize: 0 } + let blobInfo: Record = {} + // Version 0.6.2, format of digest file is changed to const infoFile = 'backup.json.gz' + const blobInfoFile = 'blob-info.json.gz' + if (await storage.exists(infoFile)) { backupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) } @@ -768,7 +162,9 @@ export async function backup ( backupInfo.migrations = {} } - const forcedFullCheck = '2' + if (await storage.exists(blobInfoFile)) { + blobInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(blobInfoFile))).toString()) + } // Apply verification to backup, since we know it should have broken blobs if (backupInfo.migrations.zeroCheckSize == null) { @@ -816,60 +212,33 @@ export async function backup ( let lastTx: Tx | undefined - let lastTxChecked = false - // Skip backup if there is no transaction changes. - if (options.getLastTx !== undefined && !fullCheck) { - lastTx = await options.getLastTx() - if (lastTx !== undefined) { - if (lastTx._id === backupInfo.lastTxId && !options.force) { - printEnd = false - ctx.info('No transaction changes. Skipping backup.', { workspace: workspaceId }) - result.result = false - return result - } - } - lastTxChecked = true - } - const token = - options.token ?? - generateToken(systemAccountUuid, workspaceId, { - service: 'backup', - mode: 'backup' - }) - ctx.warn('starting backup', { workspace: workspaceId }) - connection = - options.getConnection !== undefined - ? await options.getConnection() - : ((await createClient(transactorUrl, token, undefined, options.connectTimeout)) as CoreClient & BackupClient) - - if (!lastTxChecked && !fullCheck) { - lastTx = await connection.findOne( - core.class.Tx, - { objectSpace: { $ne: core.space.Model } }, - { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } - ) + if (!fullCheck) { + lastTx = ( + await pipeline.findAll( + ctx, + core.class.Tx, + { objectSpace: { $ne: core.space.Model } }, + { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } + ) + ).shift() if (lastTx !== undefined) { if (lastTx._id === backupInfo.lastTxId && !options.force) { ctx.info('No transaction changes. Skipping backup.', { workspace: workspaceId }) - if (options.getConnection === undefined) { - await connection.close() - } result.result = false return result } } } - const blobClient = new BlobClient(transactorUrl, token, wsIds, { storageAdapter: options.storageAdapter }) + const blobClient = new BlobClient(pipeline.context.storageAdapter ?? createDummyStorageAdapter(), wsIds) const domains = [ DOMAIN_BLOB, DOMAIN_MODEL_TX, DOMAIN_TX, - ...connection - .getHierarchy() + ...pipeline.context.hierarchy .domains() .filter( (it) => @@ -925,31 +294,28 @@ export async function backup ( } } + type RetriavableChunks = Map, { size?: number, contentType?: string, hash: string }> async function loadChangesFromServer ( ctx: MeasureContext, domain: Domain, digest: Map, string>, changes: Snapshot, same: Map, string> - ): Promise<{ changed: number, needRetrieveChunks: Ref[][] }> { + ): Promise<{ changed: number, needRetrieveChunks: RetriavableChunks[] }> { let idx: number | undefined let processed = 0 let st = Date.now() let changed: number = 0 - const needRetrieveChunks: Ref[][] = [] + const needRetrieveChunks: RetriavableChunks[] = [] const oldHash = new Map, string>() - function removeFromNeedRetrieve (needRetrieve: Ref[], id: string): void { - const pos = needRetrieve.indexOf(id as Ref) - if (pos !== -1) { - needRetrieve.splice(pos, 1) + function removeFromNeedRetrieve (needRetrieve: RetriavableChunks, id: Ref): void { + if (needRetrieve.delete(id)) { processed-- changed-- } for (const ch of needRetrieveChunks) { - const pos = ch.indexOf(id as Ref) - if (pos !== -1) { - ch.splice(pos, 1) + if (ch.delete(id)) { processed-- changed-- } @@ -957,7 +323,7 @@ export async function backup ( } while (true) { try { - const currentChunk = await ctx.with('loadChunk', {}, () => connection.loadChunk(domain, idx)) + const currentChunk = await ctx.with('loadChunk', {}, () => connection.loadChunk(ctx, domain, idx)) if (domain === DOMAIN_BLOB) { result.blobsSize += currentChunk.size ?? 0 } else { @@ -967,9 +333,9 @@ export async function backup ( idx = currentChunk.idx ops++ - let needRetrieve: Ref[] = [] + let needRetrieve: RetriavableChunks = new Map() - for (const { id, hash, contentType } of currentChunk.docs) { + for (const { id, hash, contentType, size } of currentChunk.docs) { processed++ if ( domain === DOMAIN_BLOB && @@ -977,8 +343,15 @@ export async function backup ( options.skipBlobContentTypes.length > 0 && options.skipBlobContentTypes.some((it) => contentType.includes(it)) ) { + blobInfo[id] = [contentType ?? '', size ?? 0] continue } + + if (domain === DOMAIN_BLOB && size !== undefined && size > options.blobDownloadLimit * 1024 * 1024) { + blobInfo[id] = [contentType ?? '', size ?? 0] + continue + } + if (Date.now() - st > 2500) { ctx.info('processed', { processed, @@ -1003,7 +376,7 @@ export async function backup ( removeFromNeedRetrieve(needRetrieve, id as Ref) } changes.updated.set(id as Ref, serverDocHash) - needRetrieve.push(id as Ref) + needRetrieve.set(id as Ref, { size, contentType, hash }) changed++ } else if (changes.updated.has(id as Ref)) { // We have same @@ -1014,21 +387,21 @@ export async function backup ( } else { if (domain === DOMAIN_BLOB && changes.added.has(id as Ref)) { // We need to clean old need retrieve in case of duplicates. - removeFromNeedRetrieve(needRetrieve, id) + removeFromNeedRetrieve(needRetrieve, id as Ref) } changes.added.set(id as Ref, serverDocHash) - needRetrieve.push(id as Ref) + needRetrieve.set(id as Ref, { size, contentType, hash }) changed++ } - if (needRetrieve.length > batchSize) { + if (needRetrieve.size > batchSize) { needRetrieveChunks.push(needRetrieve) - needRetrieve = [] + needRetrieve = new Map() } } - if (needRetrieve.length > 0) { + if (needRetrieve.size > 0) { needRetrieveChunks.push(needRetrieve) - needRetrieve = [] + needRetrieve = new Map() } if (currentChunk.finished) { ctx.info('processed', { @@ -1038,7 +411,7 @@ export async function backup ( workspace: workspaceId }) await ctx.with('closeChunk', {}, async () => { - await connection.closeChunk(idx as number) + await connection.closeChunk(ctx, idx as number) }) break } @@ -1046,7 +419,7 @@ export async function backup ( ctx.error('failed to load chunks', { error: err }) if (idx !== undefined) { await ctx.with('closeChunk', {}, async () => { - await connection.closeChunk(idx as number) + await connection.closeChunk(ctx, idx as number) }) } // Try again @@ -1057,6 +430,7 @@ export async function backup ( return { changed, needRetrieveChunks } } + let domainChanges = 0 async function processDomain ( ctx: MeasureContext, domain: Domain, @@ -1085,7 +459,7 @@ export async function backup ( removed: 0 } - const dHash = await connection.getDomainHash(domain) + const dHash = await connection.getDomainHash(ctx, domain) if (backupInfo.domainHashes[domain] === dHash && !fullCheck) { ctx.info('no changes in domain', { domain }) return @@ -1116,7 +490,7 @@ export async function backup ( if (d == null) { continue } - let needRetrieve: Ref[] = [] + let needRetrieve: RetriavableChunks = new Map() const { modified, modifiedFiles } = await verifyDocsFromSnapshot( ctx, domain, @@ -1126,6 +500,7 @@ export async function backup ( same, async (docs) => { const serverDocs = await connection.loadDocs( + ctx, domain, docs.map((it) => it._id) ) @@ -1146,12 +521,12 @@ export async function backup ( const dsame = deepEqual(dData, sData) if (!dsame) { - needRetrieve.push(localDoc._id) + needRetrieve.set(localDoc._id, { hash: _h1 }) changes.updated.set(localDoc._id, same.get(localDoc._id) ?? '') // Docs are not same - if (needRetrieve.length > batchSize) { + if (needRetrieve.size > batchSize) { needRetrieveChunks.push(needRetrieve) - needRetrieve = [] + needRetrieve = new Map() } } } @@ -1163,15 +538,15 @@ export async function backup ( changed++ recheckSizes.push(...modifiedFiles) } - if (needRetrieve.length > 0) { + if (needRetrieve.size > 0) { needRetrieveChunks.push(needRetrieve) - needRetrieve = [] + needRetrieve = new Map() } } // We need to retrieve all documents from same not matched const sameArray: Ref[] = Array.from(same.keys()) while (sameArray.length > 0) { - const docs = sameArray.splice(0, batchSize) + const docs: RetriavableChunks = new Map(sameArray.splice(0, batchSize).map((it) => [it, { hash: '' }])) needRetrieveChunks.push(docs) } } else { @@ -1181,7 +556,7 @@ export async function backup ( if (progress !== undefined) { await progress(10) } - const totalChunks = needRetrieveChunks.flatMap((it) => it.length).reduce((p, c) => p + c, 0) + const totalChunks = needRetrieveChunks.flatMap((it) => it.size).reduce((p, c) => p + c, 0) let processed = 0 try { @@ -1194,25 +569,59 @@ export async function backup ( if (canceled()) { return } - const needRetrieve = needRetrieveChunks.shift() as Ref[] + const needRetrieve = needRetrieveChunks.shift() as RetriavableChunks - if (needRetrieve.length === 0) { + if (needRetrieve.size === 0) { continue } ctx.info('<<<< chunk', { - needRetrieve: needRetrieveChunks.reduce((v, docs) => v + docs.length, 0), - toLoad: needRetrieve.length, + needRetrieve: needRetrieveChunks.reduce((v, docs) => v + docs.size, 0), + toLoad: needRetrieve.size, workspace: workspaceId, lastSize: Math.round((lastSize * 100) / (1024 * 1024)) / 100 }) let docs: Doc[] = [] try { - docs = await ctx.with('<<<< load-docs', {}, async () => await connection.loadDocs(domain, needRetrieve)) + if (domain === DOMAIN_BLOB) { + // Try if all info already pressent + const toRetrieve: Ref[] = [] + docs = [] + for (const [id, val] of needRetrieve) { + if (val.size !== undefined && val.contentType !== undefined) { + const b: Blob = { + _id: id as Ref, + _class: core.class.Blob, + contentType: val.contentType, + size: val.size, + etag: val.hash, + modifiedBy: core.account.System, + modifiedOn: Date.now(), + provider: '', + space: core.space.Workspace, + version: null + } + docs.push(b) + } else { + toRetrieve.push(id) + } + } + if (toRetrieve.length > 0) { + docs = docs.concat( + await ctx.with('<<<< load-docs', {}, async () => await connection.loadDocs(ctx, domain, toRetrieve)) + ) + } + } else { + docs = await ctx.with( + '<<<< load-docs', + {}, + async () => await connection.loadDocs(ctx, domain, Array.from(needRetrieve.keys())) + ) + } lastSize = docs.reduce((p, it) => p + estimateDocSize(it), 0) - if (docs.length !== needRetrieve.length) { + if (docs.length !== needRetrieve.size) { ctx.error('failed to retrieve all documents', { docsLen: docs.length, - needRetrieve: needRetrieve.length + needRetrieve: needRetrieve.size }) } ops++ @@ -1225,10 +634,7 @@ export async function backup ( while (docs.length > 0) { // Chunk data into small pieces - if ( - (addedDocuments() > dataBlobSize || processedChanges.added.size + processedChanges.updated.size > 500000) && - _pack !== undefined - ) { + if (addedDocuments() > dataBlobSize && _pack !== undefined) { await _packClose() if (changed > 0) { @@ -1240,14 +646,15 @@ export async function backup ( domainInfo.updated += processedChanges.updated.size domainInfo.removed += processedChanges.removed.length - const snapshotFile = join(backupIndex, `${domain}-${snapshot.date}-${extendZero(snapshotIndex)}.snp.gz`) snapshotIndex++ + const snapshotFile = join(backupIndex, `${domain}-${snapshot.date}-${extendZero(snapshotIndex)}.snp.gz`) domainInfo.snapshots = [...(domainInfo.snapshots ?? []), snapshotFile] await writeChanges(storage, snapshotFile, processedChanges) processedChanges.added.clear() processedChanges.removed = [] processedChanges.updated.clear() + domainChanges++ await storage.writeFile( infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel, memLevel: 9 }) @@ -1352,7 +759,7 @@ export async function backup ( printDownloaded( '', descrJson.length, - needRetrieveChunks.reduce((v, docs) => v + docs.length, 0) + docs.length + needRetrieveChunks.reduce((v, docs) => v + docs.size, 0) + docs.length ) try { const buffers: Buffer[] = [] @@ -1391,7 +798,7 @@ export async function backup ( }) }) - printDownloaded('', blob.size, needRetrieveChunks.reduce((v, docs) => v + docs.length, 0) + docs.length) + printDownloaded('', blob.size, needRetrieveChunks.reduce((v, docs) => v + docs.size, 0) + docs.length) } catch (err: any) { if (err.message?.startsWith('No file for') === true) { ctx.error('failed to download blob', { message: err.message }) @@ -1411,7 +818,7 @@ export async function backup ( }) }) processChanges(d) - printDownloaded('', data.length, needRetrieveChunks.reduce((v, docs) => v + docs.length, 0) + docs.length) + printDownloaded('', data.length, needRetrieveChunks.reduce((v, docs) => v + docs.size, 0) + docs.length) } } } @@ -1430,8 +837,8 @@ export async function backup ( if (domainInfo.added + domainInfo.updated + domainInfo.removed > 0) { snapshot.domains[domain] = domainInfo - const snapshotFile = join(backupIndex, `${domain}-${snapshot.date}-${extendZero(snapshotIndex)}.snp.gz`) snapshotIndex++ + const snapshotFile = join(backupIndex, `${domain}-${snapshot.date}-${extendZero(snapshotIndex)}.snp.gz`) domainInfo.snapshots = [...(domainInfo.snapshots ?? []), snapshotFile] await writeChanges(storage, snapshotFile, processedChanges) } @@ -1440,6 +847,7 @@ export async function backup ( processedChanges.removed = [] processedChanges.updated.clear() await _packClose() + domainChanges++ // This will allow to retry in case of critical error. await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) } @@ -1473,19 +881,23 @@ export async function backup ( domainProgress++ await options.progress?.(Math.round((domainProgress / domains.length) * 10000) / 100) } - if (!canceled()) { - backupInfo.lastTxId = lastTx?._id ?? '0' // We could store last tx, since full backup is complete - await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) - } + result.result = true - if (!canceled()) { + if (!canceled() && domainChanges > 0) { backupInfo.lastTxId = lastTx?._id ?? '0' // We could store last tx, since full backup is complete backupInfo.migrations.forcedFullCheck = forcedFullCheck + backupInfo.dataSize = result.dataSize + backupInfo.blobsSize = result.blobsSize + backupInfo.backupSize = result.backupSize await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) - } - await rebuildSizeInfo(storage, recheckSizes, ctx, result, backupInfo, infoFile) + await rebuildSizeInfo(storage, recheckSizes, ctx, result, backupInfo, infoFile) + + // Same one more time with recalculated sizes + await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) + await storage.writeFile(blobInfoFile, gzipSync(JSON.stringify(blobInfo), { level: defaultLevel })) + } return result } catch (err: any) { @@ -1496,1565 +908,9 @@ export async function backup ( if (printEnd) { ctx.info('end backup', { workspace: workspaceId, totalTime: Date.now() - st }) } - if (options.getConnection === undefined && connection !== undefined) { - await connection.close() - } ctx.end() if (options.timeout !== -1) { clearInterval(timer) } } } - -async function rebuildSizeInfo ( - storage: BackupStorage, - recheckSizes: string[], - ctx: MeasureContext, - result: BackupResult, - backupInfo: BackupInfo, - infoFile: string -): Promise { - const sizeFile = 'backup.size.gz' - - let sizeInfo: Record = {} - - if (await storage.exists(sizeFile)) { - sizeInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(sizeFile))).toString()) - } - let processed = 0 - - for (const file of recheckSizes) { - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete sizeInfo[file] - } - - const addFileSize = async (file: string | undefined | null): Promise => { - if (file != null) { - try { - const sz = sizeInfo[file] - const fileSize = sz ?? (await storage.stat(file)) - if (sz === undefined) { - sizeInfo[file] = fileSize - processed++ - if (processed % 10 === 0) { - ctx.info('Calculate size processed', { processed, size: Math.round(result.backupSize / (1024 * 1024)) }) - } - } - result.backupSize += fileSize - } catch (err: any) { - ctx.error('failed to calculate size', { file, err }) - } - } - } - - // Let's calculate data size for backup - for (const sn of backupInfo.snapshots) { - for (const [, d] of Object.entries(sn.domains)) { - await addFileSize(d.snapshot) - for (const snp of d.snapshots ?? []) { - await addFileSize(snp) - } - for (const snp of d.storage ?? []) { - await addFileSize(snp) - } - } - } - await addFileSize(infoFile) - - await storage.writeFile(sizeFile, gzipSync(JSON.stringify(sizeInfo, undefined, 2), { level: defaultLevel })) -} - -/** - * @public - */ -export async function backupList (storage: BackupStorage): Promise { - const infoFile = 'backup.json.gz' - - if (!(await storage.exists(infoFile))) { - throw new Error(`${infoFile} should present to restore`) - } - const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) - console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) - for (const s of backupInfo.snapshots) { - console.log('snapshot: id:', s.date, ' date:', new Date(s.date)) - } -} - -/** - * @public - */ -export async function backupRemoveLast (storage: BackupStorage, date: number): Promise { - const infoFile = 'backup.json.gz' - - if (!(await storage.exists(infoFile))) { - throw new Error(`${infoFile} should present to restore`) - } - const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) - console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) - const old = backupInfo.snapshots.length - backupInfo.snapshots = backupInfo.snapshots.filter((it) => it.date < date) - if (old !== backupInfo.snapshots.length) { - console.log('removed snapshots: id:', old - backupInfo.snapshots.length) - - await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) - } -} - -/** - * @public - */ -export async function backupSize (storage: BackupStorage): Promise { - const infoFile = 'backup.json.gz' - - if (!(await storage.exists(infoFile))) { - throw new Error(`${infoFile} should present to restore`) - } - let size = 0 - - const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) - console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) - const addFileSize = async (file: string | undefined | null): Promise => { - if (file != null && (await storage.exists(file))) { - try { - const fileSize = await storage.stat(file) - console.log(file, fileSize) - size += fileSize - } catch (err: any) { - console.error('failed to calculate size', { file, err }) - } - } - } - - // Let's calculate data size for backup - for (const sn of backupInfo.snapshots) { - for (const [, d] of Object.entries(sn.domains)) { - await addFileSize(d.snapshot) - for (const snp of d.snapshots ?? []) { - await addFileSize(snp) - } - for (const snp of d.storage ?? []) { - await addFileSize(snp) - } - } - } - await addFileSize(infoFile) - - console.log('Backup size', size / (1024 * 1024), 'Mb') -} - -/** - * @public - */ -export async function backupDownload (storage: BackupStorage, storeIn: string, skipDomains: Set): Promise { - const infoFile = 'backup.json.gz' - const sizeFile = 'backup.size.gz' - - if (!(await storage.exists(infoFile))) { - throw new Error(`${infoFile} should present to restore`) - } - let size = 0 - - const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) - console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) - - let sizeInfo: Record = {} - if (existsSync(join(storeIn, sizeFile))) { - console.log('Parse size file') - sizeInfo = JSON.parse(gunzipSync(new Uint8Array(readFileSync(join(storeIn, sizeFile)))).toString()) - } - console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) - - const downloadFile = async (file: string | undefined | null, force: boolean = false): Promise => { - console.log('Checking file', file) - if (file != null) { - const target = join(storeIn, file) - const dir = dirname(target) - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }) - } - - const serverSize: number | undefined = sizeInfo[file] ?? (await storage.stat(file)) - if (serverSize !== sizeInfo[file]) { - sizeInfo[file] = serverSize - writeFileSync( - join(storeIn, sizeFile), - gzipSync(JSON.stringify(sizeInfo, undefined, 2), { level: defaultLevel }) - ) - } - - if (!existsSync(target) || force || (serverSize !== undefined && serverSize !== statSync(target).size)) { - try { - console.log('downloading', file, serverSize) - const readStream = await storage.load(file) - const outp = createWriteStream(target) - - readStream.pipe(outp) - await new Promise((resolve, reject) => { - readStream.on('error', (err) => { - console.error('failed to download file', { file, err }) - reject(err) - }) - outp.on('error', (err) => { - console.error('failed to write file', { file, err }) - reject(err) - }) - outp.on('finish', () => { - readStream.destroy() - outp.close() - resolve() - }) - readStream.on('end', () => { - outp.end() - }) - }) - size += serverSize - } catch (err: any) { - console.error('failed to calculate size', { file, err }) - } - } - } - } - - await downloadFile(infoFile, true) - // Let's calculate data size for backup - for (const sn of backupInfo.snapshots) { - for (const [k, d] of Object.entries(sn.domains)) { - console.log('processing', sn.date, k) - if (skipDomains.has(k)) { - continue - } - await downloadFile(d.snapshot) - for (const snp of d.snapshots ?? []) { - await downloadFile(snp) - } - for (const snp of d.storage ?? []) { - await downloadFile(snp) - } - } - } - - console.log('Backup size', size / (1024 * 1024), 'Mb') -} - -/** - * @public - */ -export async function backupFind ( - storage: BackupStorage, - id: Ref, - showAll: boolean, - domain?: string -): Promise { - const infoFile = 'backup.json.gz' - - if (!(await storage.exists(infoFile))) { - throw new Error(`${infoFile} should present to restore`) - } - const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) - console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) - - const toolCtx = new MeasureMetricsContext('', {}) - - const snapshots = backupInfo.snapshots - const rnapshots = Array.from(backupInfo.snapshots).reverse() - - // Collect all possible domains - const domains = new Set() - for (const s of snapshots) { - Object.keys(s.domains).forEach((it) => domains.add(it as Domain)) - } - - for (const dd of domains) { - if (domain !== undefined && dd !== domain) { - continue - } - console.log('checking:', dd) - const sDigest = await loadDigest(toolCtx, storage, snapshots, dd) - if (sDigest.has(id)) { - console.log('we found file') - let found = false - for (const sn of rnapshots) { - const ssDigest = await loadDigest(toolCtx, storage, [sn], dd) - if (!ssDigest.has(id)) { - continue - } - const d = sn.domains[dd] - if (found && !showAll) { - break - } - for (const sf of d?.storage ?? []) { - if (found && !showAll) { - break - } - console.log('processing', sf) - const readStream = await storage.load(sf) - const ex = extract() - ex.on('entry', (headers, stream, next) => { - if (headers.name === id + '.json') { - console.log('file found in:', sf) - - const chunks: Buffer[] = [] - stream.on('data', (chunk) => { - chunks.push(chunk) - }) - stream.on('end', () => { - const bf = Buffer.concat(chunks as any) - console.log('>>>>>>>>>>>') - console.log(JSON.stringify(JSON.parse(bf.toString()), undefined, 2)) - console.log('>>>>>>>>>>>') - next() - }) - - found = true - } else { - stream.resume() // auto drain for non-matching entries - next() // continue to the next entry - } - }) - - const endPromise = new Promise((resolve) => { - ex.on('finish', () => { - resolve(null) - }) - }) - const unzip = createGunzip({ level: defaultLevel }) - - readStream.on('end', () => { - readStream.destroy() - }) - readStream.pipe(unzip) - unzip.pipe(ex) - - await endPromise - } - } - } - } -} - -/** - * @public - * Restore state of DB to specified point. - * - * Recheck mean we download and compare every document on our side and if found difference upload changed version to server. - */ -export async function restore ( - ctx: MeasureContext, - transactorUrl: string, - wsIds: WorkspaceIds, - storage: BackupStorage, - opt: { - date: number - merge?: boolean - parallel?: number - recheck?: boolean - include?: Set - skip?: Set - getConnection?: () => Promise - storageAdapter?: StorageAdapter - token?: string - progress?: (progress: number) => Promise - cleanIndexState?: boolean - historyFile?: string - } -): Promise { - const infoFile = 'backup.json.gz' - const workspaceId = wsIds.uuid - if (!(await storage.exists(infoFile))) { - ctx.error('file not pressent', { file: infoFile }) - throw new Error(`${infoFile} should present to restore`) - } - const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) - let snapshots = backupInfo.snapshots - if (opt.date !== -1) { - const bk = backupInfo.snapshots.findIndex((it) => it.date === opt.date) - if (bk === -1) { - ctx.error('could not restore to', { date: opt.date, file: infoFile, workspaceId }) - throw new Error(`${infoFile} could not restore to ${opt.date}. Snapshot is missing.`) - } - snapshots = backupInfo.snapshots.slice(0, bk + 1) - } else { - opt.date = snapshots[snapshots.length - 1].date - } - - if (backupInfo.domainHashes === undefined) { - backupInfo.domainHashes = {} - } - ctx.info('restore to ', { id: opt.date, date: new Date(opt.date).toDateString() }) - const rsnapshots = Array.from(snapshots).reverse() - - // Collect all possible domains - const domains = new Set() - for (const s of snapshots) { - Object.keys(s.domains).forEach((it) => domains.add(it as Domain)) - } - - ctx.info('connecting:', { transactorUrl, workspace: workspaceId }) - - const historyFile: Record = - opt.historyFile !== undefined && existsSync(opt.historyFile) - ? JSON.parse(readFileSync(opt.historyFile).toString()) - : {} - - const token = - opt.token ?? - generateToken(systemAccountUuid, workspaceId, { - service: 'backup', - mode: 'backup', - model: 'upgrade' - }) - - const connection = - opt.getConnection !== undefined - ? await opt.getConnection() - : ((await createClient(transactorUrl, token)) as CoreClient & BackupClient) - - if (opt.getConnection === undefined) { - try { - let serverEndpoint = await getTransactorEndpoint(token, 'external') - serverEndpoint = serverEndpoint.replaceAll('wss://', 'https://').replace('ws://', 'http://') - await fetch(serverEndpoint + `/api/v1/manage?token=${token}&operation=force-close`, { - method: 'PUT' - }) - } catch (err: any) { - // Ignore - } - } - - const blobClient = new BlobClient(transactorUrl, token, wsIds, { storageAdapter: opt.storageAdapter }) - console.log('connected') - - // We need to find empty domains and clean them. - const allDomains = connection.getHierarchy().domains() - for (const d of allDomains) { - domains.add(d) - } - - // We do not backup elastic anymore - domains.delete('fulltext-blob' as Domain) - domains.delete('doc-index-state' as Domain) - - let uploadedMb = 0 - let uploaded = 0 - - let domainProgress = 0 - - const printUploaded = (msg: string, size: number): void => { - if (size == null) { - return - } - uploaded += size - const newDownloadedMb = Math.round(uploaded / (1024 * 1024)) - const newId = Math.round(newDownloadedMb / 10) - if (uploadedMb !== newId) { - uploadedMb = newId - ctx.info('Uploaded', { - msg, - written: newDownloadedMb, - workspace: workspaceId - }) - } - } - - async function processDomain (c: Domain): Promise { - const dHash = await connection.getDomainHash(c) - if (backupInfo.domainHashes[c] === dHash) { - ctx.info('no changes in domain', { domain: c }) - return - } - const changeset = await loadDigest(ctx, storage, snapshots, c, opt.date) - // We need to load full changeset from server - const serverChangeset = new Map, string>() - - const oldUsed = process.memoryUsage().heapUsed - try { - global.gc?.() - } catch (err) {} - const mm = { old: oldUsed / (1024 * 1024), current: process.memoryUsage().heapUsed / (1024 * 1024) } - if (mm.old > mm.current + mm.current / 10) { - ctx.info('memory-stats', mm) - } - - let idx: number | undefined - let loaded = 0 - let el = 0 - let chunks = 0 - let dataSize = 0 - try { - while (true) { - if (opt.progress !== undefined) { - await opt.progress?.(domainProgress) - } - const st = Date.now() - const it = await connection.loadChunk(c, idx) - dataSize += it.size ?? 0 - chunks++ - - idx = it.idx - el += Date.now() - st - - for (const { id, hash } of it.docs) { - serverChangeset.set(id as Ref, hash) - loaded++ - } - - if (el > 2500) { - ctx.info('loaded from server', { domain: c, loaded, el, chunks, workspace: workspaceId }) - el = 0 - chunks = 0 - } - if (it.finished) { - break - } - } - } finally { - if (idx !== undefined) { - await connection.closeChunk(idx) - } - } - ctx.info('loaded', { - domain: c, - loaded, - workspace: workspaceId, - dataSize: Math.round((dataSize / (1024 * 1024)) * 100) / 100 - }) - ctx.info('\tcompare documents', { - size: changeset.size, - serverSize: serverChangeset.size, - workspace: workspaceId - }) - - // Let's find difference - const docsToAdd = new Map( - opt.recheck === true // If recheck we check all documents. - ? Array.from(changeset.entries()) - : Array.from(changeset.entries()).filter( - ([it]) => - !serverChangeset.has(it) || - (serverChangeset.has(it) && doTrimHash(serverChangeset.get(it)) !== doTrimHash(changeset.get(it))) - ) - ) - const docsToRemove = Array.from(serverChangeset.keys()).filter((it) => !changeset.has(it)) - - const docs: Doc[] = [] - const blobs = new Map() - let sendSize = 0 - let totalSend = 0 - async function sendChunk (doc: Doc | undefined, len: number): Promise { - if (opt.progress !== undefined) { - await opt.progress?.(domainProgress) - } - if (doc !== undefined) { - docsToAdd.delete(doc._id) - docs.push(doc) - } - sendSize = sendSize + len - - if (sendSize > dataUploadSize || (doc === undefined && docs.length > 0)) { - let docsToSend = docs - totalSend += docs.length - ctx.info('upload-' + c, { - docs: docs.length, - totalSend, - from: docsToAdd.size + totalSend, - sendSize, - workspace: workspaceId - }) - // Correct docs without space - for (const d of docs) { - if (d.space == null) { - d.space = core.space.Workspace - } - - if (TxProcessor.isExtendsCUD(d._class)) { - const tx = d as TxCUD - if (tx.objectSpace == null) { - tx.objectSpace = core.space.Workspace - } - } - } - - if (opt.recheck === true) { - // We need to download all documents and compare them. - const serverDocs = toIdMap( - await connection.loadDocs( - c, - docs.map((it) => it._id) - ) - ) - docsToSend = docs.filter((doc) => { - const serverDoc = serverDocs.get(doc._id) - if (serverDoc !== undefined) { - const { '%hash%': _h1, ...dData } = doc as any - const { '%hash%': _h2, ...sData } = serverDoc as any - - return !deepEqual(dData, sData) - } - return true - }) - } - try { - await connection.upload(c, docsToSend) - } catch (err: any) { - ctx.error('error during upload', { err, docs: JSON.stringify(docs) }) - } - - docs.length = 0 - sendSize = 0 - } - printUploaded('upload', len) - } - - let processed = 0 - - const blobUploader = new RateLimiter(10) - - for (const s of rsnapshots) { - const d = s.domains[c] - - if (d !== undefined && docsToAdd.size > 0) { - const sDigest = await loadDigest(ctx, storage, [s], c) - const requiredDocs = new Map(Array.from(sDigest.entries()).filter(([it]) => docsToAdd.has(it))) - - let lastSendTime = Date.now() - async function sendBlob (blob: Blob, data: Buffer, next: (err?: any) => void): Promise { - await blobUploader.add(async () => { - next() - let needSend = true - if (opt.historyFile !== undefined) { - if (historyFile[blob._id] === blob.etag) { - needSend = false - } - } - - if (needSend) { - try { - await blobClient.upload(ctx, blob._id, blob.size, blob.contentType, data) - if (opt.historyFile !== undefined) { - historyFile[blob._id] = blob.etag - if (totalSend % 1000 === 0) { - writeFileSync(opt.historyFile, JSON.stringify(historyFile, undefined, 2)) - } - } - } catch (err: any) { - ctx.warn('failed to upload blob', { _id: blob._id, err, workspace: wsIds.uuid }) - next(err) - } - } - docsToAdd.delete(blob._id) - requiredDocs.delete(blob._id) - printUploaded('upload:' + blobUploader.processingQueue.size, data.length) - totalSend++ - if (lastSendTime < Date.now()) { - lastSendTime = Date.now() + 2500 - - ctx.info('upload ' + c, { - totalSend, - from: docsToAdd.size + totalSend, - sendSize, - workspace: workspaceId - }) - } - }) - } - - if (requiredDocs.size > 0) { - ctx.info('updating', { domain: c, requiredDocs: requiredDocs.size, workspace: workspaceId }) - // We have required documents here. - for (const sf of d.storage ?? []) { - if (docsToAdd.size === 0) { - break - } - ctx.info('processing', { storageFile: sf, processed, workspace: wsIds.url }) - try { - const readStream = await storage.load(sf) - const ex = extract() - - ex.on('entry', (headers, stream, next) => { - const name = headers.name ?? '' - processed++ - // We found blob data - if (requiredDocs.has(name as Ref)) { - const chunks: Buffer[] = [] - stream.on('data', (chunk) => { - chunks.push(chunk) - }) - stream.on('end', () => { - const bf = Buffer.concat(chunks as any) - const d = blobs.get(name) - if (d === undefined) { - blobs.set(name, { doc: undefined, buffer: bf }) - next() - } else { - blobs.delete(name) - const blob = d?.doc as Blob - ;(blob as any)['%hash%'] = changeset.get(blob._id) - let sz = blob.size - if (Number.isNaN(sz) || sz !== bf.length) { - sz = bf.length - } - void sendBlob(blob, bf, next).catch((err) => { - ctx.error('failed to send blob', { message: err.message }) - }) - } - }) - } else if (name.endsWith('.json') && requiredDocs.has(name.substring(0, name.length - 5) as Ref)) { - const chunks: Buffer[] = [] - const bname = name.substring(0, name.length - 5) - stream.on('data', (chunk) => { - chunks.push(chunk) - }) - stream.on('end', () => { - const bf = Buffer.concat(chunks as any) - let doc: Doc - try { - doc = JSON.parse(bf.toString()) as Doc - } catch (err) { - ctx.warn('failed to parse blob metadata', { name, workspace: wsIds.url, err }) - next() - return - } - - if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') { - const data = migradeBlobData(doc as Blob, changeset.get(doc._id) as string) - const d = blobs.get(bname) ?? (data !== '' ? Buffer.from(data, 'base64') : undefined) - if (d === undefined) { - blobs.set(bname, { doc, buffer: undefined }) - next() - } else { - blobs.delete(bname) - const blob = doc as Blob - const buff = d instanceof Buffer ? d : d.buffer - if (buff != null) { - void sendBlob(blob, d instanceof Buffer ? d : (d.buffer as Buffer), next).catch((err) => { - ctx.error('failed to send blob', { err }) - }) - } else { - next() - } - } - } else { - ;(doc as any)['%hash%'] = changeset.get(doc._id) - void sendChunk(doc, bf.length) - .finally(() => { - requiredDocs.delete(doc._id) - next() - }) - .catch((err) => { - ctx.error('failed to sendChunk', { err }) - next(err) - }) - } - }) - } else { - next() - } - stream.resume() // just auto drain the stream - }) - - await blobUploader.waitProcessing() - - const unzip = createGunzip({ level: defaultLevel }) - - const endPromise = new Promise((resolve, reject) => { - ex.on('finish', () => { - resolve(null) - }) - - readStream.on('end', () => { - readStream.destroy() - }) - readStream.pipe(unzip).on('error', (err) => { - readStream.destroy() - reject(err) - }) - unzip.pipe(ex) - }) - - await endPromise - } catch (err: any) { - ctx.error('failed to processing', { storageFile: sf, processed, workspace: wsIds.url }) - } - } - } - } - } - - await sendChunk(undefined, 0) - async function performCleanOfDomain (docsToRemove: Ref[], c: Domain): Promise { - ctx.info('cleanup', { toRemove: docsToRemove.length, workspace: workspaceId, domain: c }) - while (docsToRemove.length > 0) { - const part = docsToRemove.splice(0, 10000) - try { - await connection.clean(c, part) - } catch (err: any) { - ctx.error('failed to clean, will retry', { error: err.message, workspaceId }) - docsToRemove.push(...part) - } - } - } - if (c !== DOMAIN_BLOB) { - // Clean domain documents if not blob - if (docsToRemove.length > 0 && opt.merge !== true) { - await performCleanOfDomain(docsToRemove, c) - } - } - } - - const limiter = new RateLimiter(opt.parallel ?? 1) - - try { - let i = 0 - for (const c of domains) { - if (opt.progress !== undefined) { - await opt.progress?.(domainProgress) - } - if (opt.include !== undefined && !opt.include.has(c)) { - continue - } - if (opt.skip?.has(c) === true) { - continue - } - await limiter.add(async () => { - ctx.info('processing domain', { domain: c, workspaceId }) - let retry = 5 - let delay = 1 - while (retry > 0) { - retry-- - try { - await processDomain(c) - if (delay > 1) { - ctx.warn('retry-success', { retry, delay, workspaceId }) - } - break - } catch (err: any) { - ctx.error('failed to process domain', { err, domain: c, workspaceId }) - if (retry !== 0) { - ctx.warn('cool-down to retry', { delay, domain: c, workspaceId }) - await new Promise((resolve) => setTimeout(resolve, delay * 1000)) - delay++ - } - } - } - domainProgress = Math.round(i / domains.size) * 100 - i++ - }) - } - await limiter.waitProcessing() - } catch (err: any) { - Analytics.handleError(err) - return false - } finally { - if (opt.getConnection === undefined && connection !== undefined) { - await connection.sendForceClose() - await connection.close() - } - } - return true -} - -async function verifyDocsFromSnapshot ( - ctx: MeasureContext, - domain: Domain, - d: DomainData, - s: BackupSnapshot, - storage: BackupStorage, - digest: Map, string>, - verify: (docs: Doc[]) => Promise, - chunkSize: number -): Promise<{ modified: boolean, modifiedFiles: string[] }> { - let result: Doc[] = [] - const storageToRemove = new Set() - const validDocs = new Set>() - const modifiedFiles: string[] = [] - if (digest.size > 0) { - const sDigest = await loadDigest(ctx, storage, [s], domain) - const requiredDocs = new Map(Array.from(sDigest.entries()).filter(([it]) => digest.has(it))) - - if (requiredDocs.size > 0) { - ctx.info('updating', { domain, requiredDocs: requiredDocs.size }) - // We have required documents here. - for (const sf of d.storage ?? []) { - if (digest.size === 0) { - break - } - try { - const readStream = await storage.load(sf) - const ex = extract() - - ex.on('entry', (headers, stream, next) => { - const name = headers.name ?? '' - // We found blob data - const rdoc = name.substring(0, name.length - 5) as Ref - if (name.endsWith('.json') && requiredDocs.has(rdoc)) { - const chunks: Buffer[] = [] - const bname = name.substring(0, name.length - 5) - stream.on('data', (chunk) => { - chunks.push(chunk) - }) - stream.on('end', () => { - const bf = Buffer.concat(chunks as any) - let doc: Doc - try { - doc = JSON.parse(bf.toString()) as Doc - } catch (err) { - // Do not failure on this. - next() - return - } - - if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') { - // Skip blob - validDocs.add(bname as Ref) - } else { - ;(doc as any)['%hash%'] = digest.get(rdoc) - digest.delete(rdoc) - result.push(doc) - validDocs.add(bname as Ref) - - if (result.length > chunkSize) { - void verify(result) - .then(() => { - result = [] - next() - }) - .catch((err) => { - ctx.error('failed to verify', { err }) - next(err) - }) - } else { - next() - } - } - }) - } else { - next() - } - stream.resume() // just auto drain the stream - }) - - const unzip = createGunzip({ level: defaultLevel }) - - const endPromise = new Promise((resolve, reject) => { - ex.on('finish', () => { - resolve(null) - }) - - readStream.on('end', () => { - readStream.destroy() - }) - readStream.pipe(unzip).on('error', (err) => { - readStream.destroy() - storageToRemove.add(sf) - reject(err) - }) - unzip.pipe(ex) - }) - - await endPromise - if (result.length > 0) { - await verify(result) - } - } catch (err: any) { - storageToRemove.add(sf) - ctx.error('failed to processing', { storageFile: sf }) - } - } - } - } - let modified = false - if (storageToRemove.size > 0) { - modifiedFiles.push(...Array.from(storageToRemove)) - d.storage = (d.storage ?? []).filter((it) => !storageToRemove.has(it)) - for (const sf of storageToRemove) { - await storage.delete(sf) - } - modified = true - } - let smodifiedFiles: string[] = [] - ;({ modified, modifiedFiles: smodifiedFiles } = await updateDigest(d, ctx, storage, validDocs, modified, domain)) - modifiedFiles.push(...smodifiedFiles) - return { modified, modifiedFiles } -} - -function extendZero (value: number, count: number = 4): string { - let idxName = `${value}` - while (idxName.length < count) { - idxName = '0' + idxName - } - return idxName -} - -/** - * Compacting backup into just one snapshot. - * @public - */ -export async function compactBackup ( - ctx: MeasureContext, - storage: BackupStorage, - force: boolean = false, - opt?: { - blobLimit?: number - skipContentTypes?: string[] - }, - recalculateDigest: boolean = false -): Promise { - console.log('starting backup compaction') - - const tmpRoot = mkdtempSync('huly') - - try { - let backupInfo: BackupInfo - - // Version 0.6.2, format of digest file is changed to - - const infoFile = 'backup.json.gz' - - if (await storage.exists(infoFile)) { - backupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) - } else { - console.log('No backup found') - return - } - if (backupInfo.version !== '0.6.2') { - console.log('Invalid backup version') - return - } - - if (backupInfo.snapshots.length < 1 && !force) { - console.log('No need to compact') - return - } - - // Find compacting backup or create new one - const snapshot: BackupSnapshot = backupInfo.snapshots.find((it) => it.compacting === true) ?? { - date: Date.now(), - domains: {}, - compacting: true, - stIndex: 0 - } - - let backupIndex = `${(backupInfo.snapshotsIndex ?? backupInfo.snapshots.length) + 1}` - while (backupIndex.length < 6) { - backupIndex = '0' + backupIndex - } - - const domains: Domain[] = [] - for (const sn of backupInfo.snapshots) { - for (const d of Object.keys(sn.domains)) { - if (!domains.includes(d as Domain)) { - domains.push(d as Domain) - } - } - } - const snapshotsToClean = backupInfo.snapshots.filter((it) => it.compacting !== true).reverse() - backupInfo.snapshots.push(snapshot) - - const dirsToClean = new Set() - const filesToClean = new Set() - - const cleanFiles = async (): Promise => { - if (filesToClean.size > 0) { - for (const file of filesToClean) { - try { - await storage.delete(file) - } catch (err: any) { - // Ignore - } - } - filesToClean.clear() - } - } - - for (const domain of domains) { - console.log('compacting domain...', domain) - - const processedChanges: Snapshot = { - added: new Map(), - updated: new Map(), - removed: [] - } - - let changed = 0 - let snapshotIndex = 0 - const domainInfo: DomainData = { - snapshot: undefined, - snapshots: [], - storage: [], - added: 0, - updated: 0, - removed: 0 - } - - // Cumulative digest - // Documents modified in compacting snapshots in case of restart - const untouchedDigest = await loadDigest(ctx, storage, [snapshot], domain) - - // We need to load snapshots from removed ones and - const digest = await loadDigest(ctx, storage, snapshotsToClean, domain) - - // We remove all items we have in last part - Array.from(untouchedDigest.keys()).forEach((it) => digest.delete(it)) - - const digestAdded = new Map, string>() - - let _pack: Pack | undefined - let _packClose = async (): Promise => {} - let addedDocuments: () => number = () => 0 - - let processed = 0 - - let skipBlobs = 0 - let skipSize = 0 - - let lastSkipPrint = 0 - - const blobs = new Map() - - async function pushDocs (doc: Doc | undefined, size: number, blobData: Record, Buffer>): Promise { - if (doc == null) { - return - } - changed += 1 - // Chunk data into small pieces - // The size check 'addedDocuments() > dataBlobSize' is performed here, evaluating the archive's size - // *before* the documents in the current 'docs' batch are added. - // If the archive is, e.g., 49MB, this check passes. If the current 'docs' batch then adds 5MB, - // the archive becomes 54MB. This oversized archive is closed only on the *next* call to pushDocs. - const addedDocsValue = addedDocuments() - if (addedDocsValue > dataBlobSize && _pack !== undefined) { - await _packClose() - - if (changed > 0) { - snapshot.domains[domain] = domainInfo - domainInfo.added += processedChanges.added.size - domainInfo.updated += processedChanges.updated.size - domainInfo.removed += processedChanges.removed.length - - const snapshotFile = join(backupIndex, `${domain}-${snapshot.date}-${extendZero(snapshotIndex)}.snp.gz`) - snapshotIndex++ - domainInfo.snapshots = [...(domainInfo.snapshots ?? []), snapshotFile] - await writeChanges(storage, snapshotFile, processedChanges) - - processedChanges.added.clear() - processedChanges.removed = [] - processedChanges.updated.clear() - await storage.writeFile( - infoFile, - gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel }) - ) - - await cleanFiles() - } - } - if (_pack === undefined) { - _pack = pack() - snapshot.stIndex++ - const storageFile = join( - backupIndex, - `${domain}-data-${snapshot.date}-${extendZero(snapshot.stIndex)}.tar.gz` - ) - console.log('storing from domain', domain, storageFile) - domainInfo.storage = [...(domainInfo.storage ?? []), storageFile] - - const tmpFile = join(tmpRoot, basename(storageFile) + '.tmp') - const tempFile = createWriteStream(tmpFile) - // const dataStream = await storage.write(storageFile) - - const sizePass = new PassThrough() - let sz = 0 - sizePass._transform = (chunk, encoding, cb) => { - // No transformation, just pass through data - sz += chunk.length - sizePass.push(chunk) - cb() - } - - sizePass.pipe(tempFile) - - const storageZip = createGzip({ level: defaultLevel, memLevel: 9 }) - addedDocuments = () => sz - - _pack.pipe(storageZip) - storageZip.pipe(sizePass) - - _packClose = async () => { - ctx.info('finalize pack', { storageFile, size: sz }) - await new Promise((resolve) => { - tempFile.on('close', () => { - resolve() - }) - _pack?.finalize() - }) - // We need to upload file to storage - ctx.info('>>>> upload pack', { storageFile, size: sz }) - await storage.writeFile(storageFile, createReadStream(tmpFile)) - await rm(tmpFile) - - _pack = undefined - } - } - - // Move processed document to processedChanges - processedChanges.added.set(doc._id, digestAdded.get(doc._id) ?? '') - - if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') { - const blob = doc as Blob | BlobData - - const data = blobData[blob._id] - const descrJson = JSON.stringify(doc) - await new Promise((resolve, reject) => { - _pack?.entry({ name: doc._id + '.json' }, descrJson, function (err) { - if (err != null) reject(err) - resolve() - }) - }) - await new Promise((resolve, reject) => { - _pack?.entry({ name: doc._id }, data, function (err) { - if (err != null) reject(err) - resolve() - }) - }) - } else { - const data = JSON.stringify(doc) - await new Promise((resolve, reject) => { - _pack?.entry({ name: doc._id + '.json' }, data, function (err) { - if (err != null) reject(err) - resolve() - }) - }) - } - } - async function sendChunk (doc: Doc | undefined, len: number, blobData: Record, Buffer>): Promise { - if (doc !== undefined) { - if (domain === DOMAIN_BLOB) { - if (opt?.skipContentTypes !== undefined || (opt?.blobLimit ?? 0) > 0) { - // Check if we need to skip blob - const blob = doc as Blob - - const newSkipPrint = Math.round(skipSize / (1024 * 1024 * 100)) - if (newSkipPrint !== lastSkipPrint) { - lastSkipPrint = newSkipPrint - console.log('skipping blobs', skipBlobs, Math.round(skipSize / (1024 * 1024))) - } - const bsize = blob.size == null || Number.isNaN(blob.size) || !Number.isInteger(blob.size) ? 0 : blob.size - - if ( - opt?.skipContentTypes !== undefined && - opt?.skipContentTypes.some((it) => (blob.contentType ?? '').includes(it)) - ) { - skipBlobs++ - skipSize += bsize - digest.delete(doc._id) - return - } - if (opt?.blobLimit !== undefined && opt?.blobLimit > 0 && bsize > opt.blobLimit) { - skipBlobs++ - skipSize += bsize - digest.delete(doc._id) - return - } - } - } - const hash = digest.get(doc._id) - digest.delete(doc._id) - digestAdded.set(doc._id, hash ?? '') - await pushDocs(doc, len, blobData) - } - } - - for (const s of snapshotsToClean) { - const d = s.domains[domain] - - if (d !== undefined && digest.size > 0) { - ctx.info('checking-domain', { domain, name: s.date }) - const sDigest = await loadDigest(ctx, storage, [s], domain) - const requiredDocs = new Map(Array.from(sDigest.entries()).filter(([it]) => digest.has(it))) - if (requiredDocs.size > 0) { - console.log('updating', domain, requiredDocs.size) - // We have required documents here. - for (const sf of d.storage ?? []) { - if (digest.size === 0) { - break - } - try { - console.log('processing', sf, processed) - - const readStream = await storage.load(sf) - const ex = extract() - - ex.on('entry', (headers, stream, next) => { - const name = headers.name ?? '' - processed++ - // We found blob data - if (requiredDocs.has(name as Ref)) { - const chunks: Buffer[] = [] - stream.on('data', (chunk) => { - chunks.push(chunk) - }) - stream.on('end', () => { - const bf = Buffer.concat(chunks as any) - const d = blobs.get(name) - if (d === undefined) { - blobs.set(name, { doc: undefined, buffer: bf }) - next() - } else { - blobs.delete(name) - const doc = d.doc as Blob | undefined // d here is from the outer scope, known not to be undefined. - if (doc == null) { - const errorMsg = `CompactBackup: Metadata (doc) missing in blobs entry for data file ${name}` - ctx.error(errorMsg, { name }) - next() // Just skip - return - } - void sendChunk(doc, bf.length, { [doc._id]: bf }) - .finally(() => { - requiredDocs.delete(doc._id) - next() - }) - .catch((err) => { - next(err) - }) - } - }) - } else if ( - name.endsWith('.json') && - requiredDocs.has(name.substring(0, name.length - 5) as Ref) - ) { - const chunks: Buffer[] = [] - const bname = name.substring(0, name.length - 5) - stream.on('data', (chunk) => { - chunks.push(chunk) - }) - stream.on('end', () => { - const bf = Buffer.concat(chunks as any) - let doc: Doc - try { - doc = JSON.parse(bf.toString()) as Doc - } catch (err: any) { - console.error(err) - next() - return - } - if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') { - const d = blobs.get(bname) - if (d === undefined) { - blobs.set(bname, { doc, buffer: undefined }) - next() - } else { - blobs.delete(bname) - ;(doc as any)['%hash%'] = digest.get(doc._id) - void sendChunk(doc, bf.length, { [doc._id]: d?.buffer as Buffer }) - .finally(() => { - requiredDocs.delete(doc._id) - next() - }) - .catch((err) => { - ctx.error('failed to sendChunk', { err }) - next(err) - }) - } - } else { - ;(doc as any)['%hash%'] = digest.get(doc._id) - void sendChunk(doc, bf.length, {}) - .finally(() => { - requiredDocs.delete(doc._id) - next() - }) - .catch((err) => { - ctx.error('failed to sendChunk', { err }) - next(err) - }) - } - }) - } else { - next() - } - stream.resume() // just auto drain the stream - }) - - const unzip = createGunzip({ level: defaultLevel }) - const endPromise = new Promise((resolve, reject) => { - ex.on('finish', () => { - resolve(null) - }) - readStream.on('error', (err) => { - ctx.error('error during processing', { snapshot, err }) - reject(err) - }) - unzip.on('error', (err) => { - ctx.error('error during processing', { snapshot, err }) - reject(err) - }) - }) - - readStream.on('end', () => { - readStream.destroy() - }) - readStream.pipe(unzip) - unzip.pipe(ex) - - await endPromise - } catch (err: any) { - ctx.error('error processing', err) - } - } - } else { - console.log('domain had no changes', domain) - } - } - if (d !== undefined) { - for (const sf of d.storage ?? []) { - console.log('removing', sf) - dirsToClean.add(dirname(sf)) - filesToClean.add(sf) - } - for (const sf of d.snapshots ?? []) { - console.log('removing', sf) - dirsToClean.add(dirname(sf)) - filesToClean.add(sf) - } - if (d.snapshot !== undefined) { - dirsToClean.add(dirname(d.snapshot)) - filesToClean.add(d.snapshot) - } - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete s.domains[domain] - } - } - - if (changed > 0) { - domainInfo.added += processedChanges.added.size - domainInfo.updated += processedChanges.updated.size - domainInfo.removed += processedChanges.removed.length - - if (domainInfo.added + domainInfo.updated + domainInfo.removed > 0) { - snapshot.domains[domain] = domainInfo - - const snapshotFile = join(backupIndex, `${domain}-${snapshot.date}-${extendZero(snapshotIndex)}.snp.gz`) - snapshotIndex++ - domainInfo.snapshots = [...(domainInfo.snapshots ?? []), snapshotFile] - await writeChanges(storage, snapshotFile, processedChanges) - } - - processedChanges.added.clear() - processedChanges.removed = [] - processedChanges.updated.clear() - await _packClose() - } - // This will allow to retry in case of critical error. - await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) - } - snapshot.compacting = false - backupInfo.snapshots = [snapshot] - backupInfo.snapshotsIndex = (backupInfo.snapshotsIndex ?? backupInfo.snapshots.length) + 1 - await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) - - await cleanFiles() - for (const dir of dirsToClean) { - try { - await storage.deleteRecursive(dir) - } catch (err: any) { - // Ignore - } - } - const result: BackupResult = { - result: true, - dataSize: 0, - blobsSize: 0, - backupSize: 0 - } - if (recalculateDigest) { - await rebuildSizeInfo(storage, [], ctx, result, backupInfo, infoFile) - } - } catch (err: any) { - console.error(err) - } finally { - await rm(tmpRoot, { recursive: true }) - console.log('end compacting') - } -} - -export * from './service' -function migradeBlobData (blob: Blob, etag: string): string { - if (blob._class === 'core:class:BlobData') { - const bd = blob as unknown as BlobData - blob.contentType = blob.contentType ?? bd.type - blob.etag = etag - blob._class = core.class.Blob - delete (blob as any).type - const result = (blob as any).base64Data - delete (blob as any).base64Data - return result - } - return '' -} - -/** - * Will check backup integrity, and in case of some missing resources, will update digest files, so next backup will backup all missing parts. - * @public - */ -export async function checkBackupIntegrity (ctx: MeasureContext, storage: BackupStorage): Promise { - console.log('check backup integrity') - try { - let backupInfo: BackupInfo - - // Version 0.6.2, format of digest file is changed to - - const infoFile = 'backup.json.gz' - - if (await storage.exists(infoFile)) { - backupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) - } else { - console.log('No backup found') - return - } - if (backupInfo.version !== '0.6.2') { - console.log('Invalid backup version') - return - } - - const recheckSizes: string[] = [] - - const domains: Domain[] = [] - for (const sn of backupInfo.snapshots) { - for (const d of Object.keys(sn.domains)) { - if (!domains.includes(d as Domain)) { - domains.push(d as Domain) - } - } - } - let modified = false - - for (const domain of domains) { - console.log('checking domain...', domain) - const { modified: mm, modifiedFiles } = await verifyDigest(ctx, storage, backupInfo.snapshots, domain) - if (mm) { - recheckSizes.push(...modifiedFiles) - modified = true - } - } - if (backupInfo.migrations == null) { - backupInfo.migrations = {} - } - if (backupInfo.migrations.zeroCheckSize !== true) { - backupInfo.migrations.zeroCheckSize = true - modified = true - } - if (modified) { - await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) - } - - const bresult: BackupResult = { - backupSize: 0, - blobsSize: 0, - dataSize: 0, - result: true - } - await rebuildSizeInfo(storage, recheckSizes, ctx, bresult, backupInfo, infoFile) - } catch (err: any) { - console.error(err) - } finally { - console.log('end compacting') - } -} diff --git a/server/backup/src/index.ts b/server/backup/src/index.ts index b61a2c25e0..b2cd2bf630 100644 --- a/server/backup/src/index.ts +++ b/server/backup/src/index.ts @@ -13,4 +13,7 @@ // limitations under the License. // export * from './backup' +export * from './restore' export * from './service' +export * from './types' +export * from './utils' diff --git a/server/backup/src/restore.ts b/server/backup/src/restore.ts new file mode 100644 index 0000000000..dcefb07b4a --- /dev/null +++ b/server/backup/src/restore.ts @@ -0,0 +1,535 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// Copyright © 2021 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Analytics } from '@hcengineering/analytics' +import core, { + Doc, + Domain, + DOMAIN_BLOB, + MeasureContext, + RateLimiter, + Ref, + toIdMap, + TxProcessor, + type Blob, + type LowLevelStorage, + type TxCUD, + type WorkspaceIds +} from '@hcengineering/core' +import { BlobClient } from '@hcengineering/server-client' +import { BackupClientOps, createDummyStorageAdapter, type Pipeline } from '@hcengineering/server-core' +import { deepEqual } from 'fast-equals' +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { extract } from 'tar-stream' +import { createGunzip, gunzipSync } from 'zlib' +import { BackupStorage } from './storage' +import type { BackupInfo } from './types' +import { doTrimHash, loadDigest, migradeBlobData } from './utils' +export * from './storage' + +const dataUploadSize = 2 * 1024 * 1024 + +const defaultLevel = 9 + +/** + * @public + * Restore state of DB to specified point. + * + * Recheck mean we download and compare every document on our side and if found difference upload changed version to server. + */ +export async function restore ( + ctx: MeasureContext, + pipeline: Pipeline, + wsIds: WorkspaceIds, + storage: BackupStorage, + opt: { + date: number + merge?: boolean + parallel?: number + recheck?: boolean + include?: Set + skip?: Set + progress?: (progress: number) => Promise + cleanIndexState?: boolean + historyFile?: string + } +): Promise { + const infoFile = 'backup.json.gz' + const workspaceId = wsIds.uuid + if (!(await storage.exists(infoFile))) { + ctx.error('file not pressent', { file: infoFile }) + throw new Error(`${infoFile} should present to restore`) + } + const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + let snapshots = backupInfo.snapshots + if (opt.date !== -1) { + const bk = backupInfo.snapshots.findIndex((it) => it.date === opt.date) + if (bk === -1) { + ctx.error('could not restore to', { date: opt.date, file: infoFile, workspaceId }) + throw new Error(`${infoFile} could not restore to ${opt.date}. Snapshot is missing.`) + } + snapshots = backupInfo.snapshots.slice(0, bk + 1) + } else { + opt.date = snapshots[snapshots.length - 1].date + } + + if (backupInfo.domainHashes === undefined) { + backupInfo.domainHashes = {} + } + ctx.info('restore to ', { id: opt.date, date: new Date(opt.date).toDateString() }) + const rsnapshots = Array.from(snapshots).reverse() + + // Collect all possible domains + const domains = new Set() + for (const s of snapshots) { + Object.keys(s.domains).forEach((it) => domains.add(it as Domain)) + } + + const historyFile: Record = + opt.historyFile !== undefined && existsSync(opt.historyFile) + ? JSON.parse(readFileSync(opt.historyFile).toString()) + : {} + + const blobClient = new BlobClient(pipeline.context.storageAdapter ?? createDummyStorageAdapter(), wsIds) + console.log('connected') + + // We need to find empty domains and clean them. + const allDomains = pipeline.context.hierarchy.domains() + for (const d of allDomains) { + domains.add(d) + } + + // We do not backup elastic anymore + domains.delete('fulltext-blob' as Domain) + domains.delete('doc-index-state' as Domain) + + let uploadedMb = 0 + let uploaded = 0 + + let domainProgress = 0 + + const connection = pipeline.context.lowLevelStorage as LowLevelStorage + const ops = new BackupClientOps(connection) + + const printUploaded = (msg: string, size: number): void => { + if (size == null) { + return + } + uploaded += size + const newDownloadedMb = Math.round(uploaded / (1024 * 1024)) + const newId = Math.round(newDownloadedMb / 10) + if (uploadedMb !== newId) { + uploadedMb = newId + ctx.info('Uploaded', { + msg, + written: newDownloadedMb, + workspace: workspaceId + }) + } + } + + async function processDomain (c: Domain): Promise { + const dHash = await connection.getDomainHash(ctx, c) + if (backupInfo.domainHashes[c] === dHash) { + ctx.info('no changes in domain', { domain: c }) + return + } + const changeset = await loadDigest(ctx, storage, snapshots, c, opt.date) + // We need to load full changeset from server + const serverChangeset = new Map, string>() + + const oldUsed = process.memoryUsage().heapUsed + try { + global.gc?.() + } catch (err) {} + const mm = { old: oldUsed / (1024 * 1024), current: process.memoryUsage().heapUsed / (1024 * 1024) } + if (mm.old > mm.current + mm.current / 10) { + ctx.info('memory-stats', mm) + } + + let idx: number | undefined + let loaded = 0 + let el = 0 + let chunks = 0 + let dataSize = 0 + try { + while (true) { + if (opt.progress !== undefined) { + await opt.progress?.(domainProgress) + } + const st = Date.now() + const it = await ops.loadChunk(ctx, c, idx) + dataSize += it.size ?? 0 + chunks++ + + idx = it.idx + el += Date.now() - st + + for (const { id, hash } of it.docs) { + serverChangeset.set(id as Ref, hash) + loaded++ + } + + if (el > 2500) { + ctx.info('loaded from server', { domain: c, loaded, el, chunks, workspace: workspaceId }) + el = 0 + chunks = 0 + } + if (it.finished) { + break + } + } + } finally { + if (idx !== undefined) { + await ops.closeChunk(ctx, idx) + } + } + ctx.info('loaded', { + domain: c, + loaded, + workspace: workspaceId, + dataSize: Math.round((dataSize / (1024 * 1024)) * 100) / 100 + }) + ctx.info('\tcompare documents', { + size: changeset.size, + serverSize: serverChangeset.size, + workspace: workspaceId + }) + + // Let's find difference + const docsToAdd = new Map( + opt.recheck === true // If recheck we check all documents. + ? Array.from(changeset.entries()) + : Array.from(changeset.entries()).filter( + ([it]) => + !serverChangeset.has(it) || + (serverChangeset.has(it) && doTrimHash(serverChangeset.get(it)) !== doTrimHash(changeset.get(it))) + ) + ) + const docsToRemove = Array.from(serverChangeset.keys()).filter((it) => !changeset.has(it)) + + const docs: Doc[] = [] + const blobs = new Map() + let sendSize = 0 + let totalSend = 0 + async function sendChunk (doc: Doc | undefined, len: number): Promise { + if (opt.progress !== undefined) { + await opt.progress?.(domainProgress) + } + if (doc !== undefined) { + docsToAdd.delete(doc._id) + docs.push(doc) + } + sendSize = sendSize + len + + if (sendSize > dataUploadSize || (doc === undefined && docs.length > 0)) { + let docsToSend = docs + totalSend += docs.length + ctx.info('upload-' + c, { + docs: docs.length, + totalSend, + from: docsToAdd.size + totalSend, + sendSize, + workspace: workspaceId + }) + // Correct docs without space + for (const d of docs) { + if (d.space == null) { + d.space = core.space.Workspace + } + + if (TxProcessor.isExtendsCUD(d._class)) { + const tx = d as TxCUD + if (tx.objectSpace == null) { + tx.objectSpace = core.space.Workspace + } + } + } + + if (opt.recheck === true) { + // We need to download all documents and compare them. + const serverDocs = toIdMap( + await ops.loadDocs( + ctx, + c, + docs.map((it) => it._id) + ) + ) + docsToSend = docs.filter((doc) => { + const serverDoc = serverDocs.get(doc._id) + if (serverDoc !== undefined) { + const { '%hash%': _h1, ...dData } = doc as any + const { '%hash%': _h2, ...sData } = serverDoc as any + + return !deepEqual(dData, sData) + } + return true + }) + } + try { + await ops.upload(ctx, c, docsToSend) + } catch (err: any) { + ctx.error('error during upload', { err, docs: JSON.stringify(docs) }) + } + + docs.length = 0 + sendSize = 0 + } + printUploaded('upload', len) + } + + let processed = 0 + + const blobUploader = new RateLimiter(10) + + for (const s of rsnapshots) { + const d = s.domains[c] + + if (d !== undefined && docsToAdd.size > 0) { + const sDigest = await loadDigest(ctx, storage, [s], c) + const requiredDocs = new Map(Array.from(sDigest.entries()).filter(([it]) => docsToAdd.has(it))) + + let lastSendTime = Date.now() + async function sendBlob (blob: Blob, data: Buffer, next: (err?: any) => void): Promise { + await blobUploader.add(async () => { + next() + let needSend = true + if (opt.historyFile !== undefined) { + if (historyFile[blob._id] === blob.etag) { + needSend = false + } + } + + if (needSend) { + try { + await blobClient.upload(ctx, blob._id, blob.size, blob.contentType, data) + if (opt.historyFile !== undefined) { + historyFile[blob._id] = blob.etag + if (totalSend % 1000 === 0) { + writeFileSync(opt.historyFile, JSON.stringify(historyFile, undefined, 2)) + } + } + } catch (err: any) { + ctx.warn('failed to upload blob', { _id: blob._id, err, workspace: wsIds.uuid }) + next(err) + } + } + docsToAdd.delete(blob._id) + requiredDocs.delete(blob._id) + printUploaded('upload:' + blobUploader.processingQueue.size, data.length) + totalSend++ + if (lastSendTime < Date.now()) { + lastSendTime = Date.now() + 2500 + + ctx.info('upload ' + c, { + totalSend, + from: docsToAdd.size + totalSend, + sendSize, + workspace: workspaceId + }) + } + }) + } + + if (requiredDocs.size > 0) { + ctx.info('updating', { domain: c, requiredDocs: requiredDocs.size, workspace: workspaceId }) + // We have required documents here. + for (const sf of d.storage ?? []) { + if (docsToAdd.size === 0) { + break + } + ctx.info('processing', { storageFile: sf, processed, workspace: wsIds.url }) + try { + const readStream = await storage.load(sf) + const ex = extract() + + ex.on('entry', (headers, stream, next) => { + const name = headers.name ?? '' + processed++ + // We found blob data + if (requiredDocs.has(name as Ref)) { + const chunks: Buffer[] = [] + stream.on('data', (chunk) => { + chunks.push(chunk) + }) + stream.on('end', () => { + const bf = Buffer.concat(chunks as any) + const d = blobs.get(name) + if (d === undefined) { + blobs.set(name, { doc: undefined, buffer: bf }) + next() + } else { + blobs.delete(name) + const blob = d?.doc as Blob + ;(blob as any)['%hash%'] = changeset.get(blob._id) + let sz = blob.size + if (Number.isNaN(sz) || sz !== bf.length) { + sz = bf.length + } + void sendBlob(blob, bf, next).catch((err) => { + ctx.error('failed to send blob', { message: err.message }) + }) + } + }) + } else if (name.endsWith('.json') && requiredDocs.has(name.substring(0, name.length - 5) as Ref)) { + const chunks: Buffer[] = [] + const bname = name.substring(0, name.length - 5) + stream.on('data', (chunk) => { + chunks.push(chunk) + }) + stream.on('end', () => { + const bf = Buffer.concat(chunks as any) + let doc: Doc + try { + doc = JSON.parse(bf.toString()) as Doc + } catch (err) { + ctx.warn('failed to parse blob metadata', { name, workspace: wsIds.url, err }) + next() + return + } + + if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') { + const data = migradeBlobData(doc as Blob, changeset.get(doc._id) as string) + const d = blobs.get(bname) ?? (data !== '' ? Buffer.from(data, 'base64') : undefined) + if (d === undefined) { + blobs.set(bname, { doc, buffer: undefined }) + next() + } else { + blobs.delete(bname) + const blob = doc as Blob + const buff = d instanceof Buffer ? d : d.buffer + if (buff != null) { + void sendBlob(blob, d instanceof Buffer ? d : (d.buffer as Buffer), next).catch((err) => { + ctx.error('failed to send blob', { err }) + }) + } else { + next() + } + } + } else { + ;(doc as any)['%hash%'] = changeset.get(doc._id) + void sendChunk(doc, bf.length) + .finally(() => { + requiredDocs.delete(doc._id) + next() + }) + .catch((err) => { + ctx.error('failed to sendChunk', { err }) + next(err) + }) + } + }) + } else { + next() + } + stream.resume() // just auto drain the stream + }) + + await blobUploader.waitProcessing() + + const unzip = createGunzip({ level: defaultLevel }) + + const endPromise = new Promise((resolve, reject) => { + ex.on('finish', () => { + resolve(null) + }) + + readStream.on('end', () => { + readStream.destroy() + }) + readStream.pipe(unzip).on('error', (err) => { + readStream.destroy() + reject(err) + }) + unzip.pipe(ex) + }) + + await endPromise + } catch (err: any) { + ctx.error('failed to processing', { storageFile: sf, processed, workspace: wsIds.url }) + } + } + } + } + } + + await sendChunk(undefined, 0) + async function performCleanOfDomain (docsToRemove: Ref[], c: Domain): Promise { + ctx.info('cleanup', { toRemove: docsToRemove.length, workspace: workspaceId, domain: c }) + while (docsToRemove.length > 0) { + const part = docsToRemove.splice(0, 10000) + try { + await ops.clean(ctx, c, part) + } catch (err: any) { + ctx.error('failed to clean, will retry', { error: err.message, workspaceId }) + docsToRemove.push(...part) + } + } + } + if (c !== DOMAIN_BLOB) { + // Clean domain documents if not blob + if (docsToRemove.length > 0 && opt.merge !== true) { + await performCleanOfDomain(docsToRemove, c) + } + } + } + + const limiter = new RateLimiter(opt.parallel ?? 1) + + try { + let i = 0 + for (const c of domains) { + if (opt.progress !== undefined) { + await opt.progress?.(domainProgress) + } + if (opt.include !== undefined && !opt.include.has(c)) { + continue + } + if (opt.skip?.has(c) === true) { + continue + } + await limiter.add(async () => { + ctx.info('processing domain', { domain: c, workspaceId }) + let retry = 5 + let delay = 1 + while (retry > 0) { + retry-- + try { + await processDomain(c) + if (delay > 1) { + ctx.warn('retry-success', { retry, delay, workspaceId }) + } + break + } catch (err: any) { + ctx.error('failed to process domain', { err, domain: c, workspaceId }) + if (retry !== 0) { + ctx.warn('cool-down to retry', { delay, domain: c, workspaceId }) + await new Promise((resolve) => setTimeout(resolve, delay * 1000)) + delay++ + } + } + } + domainProgress = Math.round(i / domains.size) * 100 + i++ + }) + } + await limiter.waitProcessing() + } catch (err: any) { + Analytics.handleError(err) + return false + } + return true +} diff --git a/server/backup/src/service.ts b/server/backup/src/service.ts index 9d0490ee88..d8d6b3053d 100644 --- a/server/backup/src/service.ts +++ b/server/backup/src/service.ts @@ -14,28 +14,22 @@ // import { Analytics } from '@hcengineering/analytics' -import core, { - DOMAIN_TX, +import { groupByArray, - Hierarchy, isActiveMode, - ModelDb, RateLimiter, reduceCalls, - SortingOrder, systemAccountUuid, WorkspaceDataId, WorkspaceUuid, type BackupStatus, type Branding, type MeasureContext, - type Tx, type WorkspaceIds, type WorkspaceInfoWithStatus } from '@hcengineering/core' import { getAccountClient } from '@hcengineering/server-client' import { - wrapPipeline, type DbConfiguration, type Pipeline, type PipelineFactory, @@ -43,8 +37,9 @@ import { } from '@hcengineering/server-core' import { generateToken } from '@hcengineering/server-token' import { clearInterval } from 'node:timers' -import { backup, restore } from '.' import { createStorageBackupStorage } from './storage' +import { backup } from './backup' +import { restore } from './restore' export interface BackupConfig { AccountsURL: string @@ -71,7 +66,6 @@ class BackupWorker { readonly storageAdapter: StorageAdapter, readonly config: BackupConfig, readonly pipelineFactory: PipelineFactory, - readonly workspaceStorageAdapter: StorageAdapter, readonly getConfig: ( ctx: MeasureContext, workspace: WorkspaceIds, @@ -303,11 +297,23 @@ class BackupWorker { dataId: ws.dataId, url: ws.url } + pipeline = await this.pipelineFactory( + ctx, + wsIds, + { + broadcast: () => {}, + broadcastSessions: () => {} + }, + null + ) + if (pipeline === undefined) { + throw new Error('Pipeline is undefined, cannot proceed with backup') + } const result = await ctx.with( 'backup', {}, (ctx) => - backup(ctx, '', wsIds, storage, { + backup(ctx, pipeline as Pipeline, wsIds, storage, { skipDomains: this.skipDomains, force: true, timeout: this.config.Timeout * 1000, @@ -316,48 +322,6 @@ class BackupWorker { blobDownloadLimit: this.downloadLimit, skipBlobContentTypes: ['video/', 'audio/', 'image/'], fullVerify: this.fullCheck, - storageAdapter: this.workspaceStorageAdapter, - getLastTx: async (): Promise => { - const config = this.getConfig(ctx, wsIds, null, this.workspaceStorageAdapter) - const adapterConf = config.adapters[config.domains[DOMAIN_TX]] - const hierarchy = new Hierarchy() - const modelDb = new ModelDb(hierarchy) - const txAdapter = await adapterConf.factory( - ctx, - hierarchy, - adapterConf.url, - wsIds, - modelDb, - this.workspaceStorageAdapter - ) - try { - await txAdapter.init?.(ctx, {}) - - return ( - await txAdapter.rawFindAll( - DOMAIN_TX, - { objectSpace: { $ne: core.space.Model } }, - { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } - ) - ).shift() - } finally { - await txAdapter.close() - } - }, - getConnection: async () => { - if (pipeline === undefined) { - pipeline = await this.pipelineFactory( - ctx, - wsIds, - { - broadcast: () => {}, - broadcastSessions: () => {} - }, - null - ) - } - return wrapPipeline(ctx, pipeline, wsIds) - }, progress: (progress) => { return notify?.(progress) ?? Promise.resolve() } @@ -409,7 +373,6 @@ export function backupService ( storage: StorageAdapter, config: BackupConfig, pipelineFactory: PipelineFactory, - workspaceStorageAdapter: StorageAdapter, getConfig: ( ctx: MeasureContext, workspace: WorkspaceIds, @@ -419,7 +382,7 @@ export function backupService ( region: string, recheck?: boolean ): () => void { - const backupWorker = new BackupWorker(storage, config, pipelineFactory, workspaceStorageAdapter, getConfig, region) + const backupWorker = new BackupWorker(storage, config, pipelineFactory, getConfig, region) const shutdown = (): void => { void backupWorker.close() @@ -435,7 +398,6 @@ export async function doBackupWorkspace ( storage: StorageAdapter, config: BackupConfig, pipelineFactory: PipelineFactory, - workspaceStorageAdapter: StorageAdapter, getConfig: ( ctx: MeasureContext, workspace: WorkspaceIds, @@ -448,16 +410,7 @@ export async function doBackupWorkspace ( fullCheck: boolean = false, notify?: (progress: number) => Promise ): Promise { - const backupWorker = new BackupWorker( - storage, - config, - pipelineFactory, - workspaceStorageAdapter, - getConfig, - region, - skipDomains, - fullCheck - ) + const backupWorker = new BackupWorker(storage, config, pipelineFactory, getConfig, region, skipDomains, fullCheck) backupWorker.downloadLimit = downloadLimit const result = await backupWorker.doBackup(ctx, workspace, notify) await backupWorker.close() @@ -470,7 +423,6 @@ export async function doRestoreWorkspace ( backupAdapter: StorageAdapter, bucketName: string, pipelineFactory: PipelineFactory, - workspaceStorageAdapter: StorageAdapter, skipDomains: string[], cleanIndexState: boolean, notify?: (progress: number) => Promise @@ -482,32 +434,29 @@ export async function doRestoreWorkspace ( const ctx = rootCtx.newChild('doRestore', {}) let pipeline: Pipeline | undefined try { + pipeline = await pipelineFactory( + ctx, + wsIds, + { + broadcast: () => {}, + broadcastSessions: () => {} + }, + null + ) + if (pipeline === undefined) { + throw new Error('Pipeline is undefined, cannot proceed with restore') + } const restoreIds = { uuid: bucketName as WorkspaceUuid, dataId: bucketName as WorkspaceDataId, url: '' } const storage = await createStorageBackupStorage(ctx, backupAdapter, restoreIds, wsIds.dataId ?? wsIds.uuid) const result: boolean = await ctx.with( 'restore', {}, (ctx) => - restore(ctx, '', wsIds, storage, { + restore(ctx, pipeline as Pipeline, wsIds, storage, { date: -1, skip: new Set(skipDomains), recheck: false, // Do not need to recheck - storageAdapter: workspaceStorageAdapter, cleanIndexState, - getConnection: async () => { - if (pipeline === undefined) { - pipeline = await pipelineFactory( - ctx, - wsIds, - { - broadcast: () => {}, - broadcastSessions: () => {} - }, - null - ) - } - return wrapPipeline(ctx, pipeline, wsIds) - }, progress: (progress) => { return notify?.(progress) ?? Promise.resolve() } diff --git a/server/backup/src/types.ts b/server/backup/src/types.ts new file mode 100644 index 0000000000..bc1caddb93 --- /dev/null +++ b/server/backup/src/types.ts @@ -0,0 +1,97 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// Copyright © 2021 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Doc, Domain, Ref, type BackupStatus, type WorkspaceUuid } from '@hcengineering/core' +export * from './storage' + +/** + * Blob data from s3 storage + * @public + */ +export interface BlobData extends Doc { + name: string + size: number + type: string + provider?: string // If node defined, will be default one + base64Data: string // base64 encoded data +} + +/** + * @public + */ +export interface Snapshot { + added: Map, string> + updated: Map, string> + removed: Ref[] +} + +/** + * @public + */ +export interface SnapshotV6 { + added: Record, string> + updated: Record, string> + removed: Ref[] +} + +/** + * @public + */ +export interface DomainData { + snapshot?: string // 0.6 json snapshot + snapshots?: string[] + storage?: string[] + + // Some statistics + added: number + updated: number + removed: number +} + +/** + * @public + */ +export interface BackupSnapshot { + // _id => hash of added items. + domains: Record + date: number + + compacting?: boolean + stIndex: number // Snapshot index +} + +/** + * @public + */ +export interface BackupInfo { + workspace: WorkspaceUuid + version: string + snapshots: BackupSnapshot[] + snapshotsIndex?: number + lastTxId?: string + + // A hash of current domain transactions, so we could skip all other checks if same. + domainHashes: Record + + migrations: Record + dataSize?: number + blobsSize?: number + backupSize?: number +} + +export interface BackupResult extends Omit { + result: boolean +} diff --git a/server/backup/src/utils.ts b/server/backup/src/utils.ts new file mode 100644 index 0000000000..3cc3fb4c8e --- /dev/null +++ b/server/backup/src/utils.ts @@ -0,0 +1,1398 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// Copyright © 2021 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import core, { + Doc, + Domain, + DOMAIN_BLOB, + MeasureContext, + MeasureMetricsContext, + Ref, + type Blob +} from '@hcengineering/core' +import { + createReadStream, + createWriteStream, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + statSync, + writeFileSync +} from 'node:fs' +import { rm } from 'node:fs/promises' +import { basename, dirname } from 'node:path' +import { PassThrough, type Writable } from 'node:stream' +import { createGzip } from 'node:zlib' +import { join } from 'path' +import { extract, Pack, pack } from 'tar-stream' +import { createGunzip, gunzipSync, gzipSync } from 'zlib' +import { BackupStorage } from './storage' +import type { BackupInfo, BackupResult, BackupSnapshot, BlobData, DomainData, Snapshot, SnapshotV6 } from './types' +export * from './storage' + +const dataBlobSize = 250 * 1024 * 1024 + +const defaultLevel = 9 + +/** + * @public + */ +export async function backupList (storage: BackupStorage): Promise { + const infoFile = 'backup.json.gz' + + if (!(await storage.exists(infoFile))) { + throw new Error(`${infoFile} should present to restore`) + } + const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) + for (const s of backupInfo.snapshots) { + console.log('snapshot: id:', s.date, ' date:', new Date(s.date)) + } +} + +/** + * @public + */ +export async function backupRemoveLast (storage: BackupStorage, date: number): Promise { + const infoFile = 'backup.json.gz' + + if (!(await storage.exists(infoFile))) { + throw new Error(`${infoFile} should present to restore`) + } + const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) + const old = backupInfo.snapshots.length + backupInfo.snapshots = backupInfo.snapshots.filter((it) => it.date < date) + if (old !== backupInfo.snapshots.length) { + console.log('removed snapshots: id:', old - backupInfo.snapshots.length) + + await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) + } +} + +/** + * @public + */ +export async function backupSize (storage: BackupStorage): Promise { + const infoFile = 'backup.json.gz' + + if (!(await storage.exists(infoFile))) { + throw new Error(`${infoFile} should present to restore`) + } + let size = 0 + + const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) + const addFileSize = async (file: string | undefined | null): Promise => { + if (file != null && (await storage.exists(file))) { + try { + const fileSize = await storage.stat(file) + console.log(file, fileSize) + size += fileSize + } catch (err: any) { + console.error('failed to calculate size', { file, err }) + } + } + } + + // Let's calculate data size for backup + for (const sn of backupInfo.snapshots) { + for (const [, d] of Object.entries(sn.domains)) { + await addFileSize(d.snapshot) + for (const snp of d.snapshots ?? []) { + await addFileSize(snp) + } + for (const snp of d.storage ?? []) { + await addFileSize(snp) + } + } + } + await addFileSize(infoFile) + + console.log('Backup size', size / (1024 * 1024), 'Mb') +} + +/** + * @public + */ +export async function backupDownload (storage: BackupStorage, storeIn: string, skipDomains: Set): Promise { + const infoFile = 'backup.json.gz' + const sizeFile = 'backup.size.gz' + + if (!(await storage.exists(infoFile))) { + throw new Error(`${infoFile} should present to restore`) + } + let size = 0 + + const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) + + let sizeInfo: Record = {} + if (existsSync(join(storeIn, sizeFile))) { + console.log('Parse size file') + sizeInfo = JSON.parse(gunzipSync(new Uint8Array(readFileSync(join(storeIn, sizeFile)))).toString()) + } + console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) + + const downloadFile = async (file: string | undefined | null, force: boolean = false): Promise => { + console.log('Checking file', file) + if (file != null) { + const target = join(storeIn, file) + const dir = dirname(target) + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }) + } + + const serverSize: number | undefined = sizeInfo[file] ?? (await storage.stat(file)) + if (serverSize !== sizeInfo[file]) { + sizeInfo[file] = serverSize + writeFileSync( + join(storeIn, sizeFile), + gzipSync(JSON.stringify(sizeInfo, undefined, 2), { level: defaultLevel }) + ) + } + + if (!existsSync(target) || force || (serverSize !== undefined && serverSize !== statSync(target).size)) { + try { + console.log('downloading', file, serverSize) + const readStream = await storage.load(file) + const outp = createWriteStream(target) + + readStream.pipe(outp) + await new Promise((resolve, reject) => { + readStream.on('error', (err) => { + console.error('failed to download file', { file, err }) + reject(err) + }) + outp.on('error', (err) => { + console.error('failed to write file', { file, err }) + reject(err) + }) + outp.on('finish', () => { + readStream.destroy() + outp.close() + resolve() + }) + readStream.on('end', () => { + outp.end() + }) + }) + size += serverSize + } catch (err: any) { + console.error('failed to calculate size', { file, err }) + } + } + } + } + + await downloadFile(infoFile, true) + // Let's calculate data size for backup + for (const sn of backupInfo.snapshots) { + for (const [k, d] of Object.entries(sn.domains)) { + console.log('processing', sn.date, k) + if (skipDomains.has(k)) { + continue + } + await downloadFile(d.snapshot) + for (const snp of d.snapshots ?? []) { + await downloadFile(snp) + } + for (const snp of d.storage ?? []) { + await downloadFile(snp) + } + } + } + + console.log('Backup size', size / (1024 * 1024), 'Mb') +} + +/** + * @public + */ +export async function backupFind ( + storage: BackupStorage, + id: Ref, + showAll: boolean, + domain?: string +): Promise { + const infoFile = 'backup.json.gz' + + if (!(await storage.exists(infoFile))) { + throw new Error(`${infoFile} should present to restore`) + } + const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) + + const toolCtx = new MeasureMetricsContext('', {}) + + const snapshots = backupInfo.snapshots + const rnapshots = Array.from(backupInfo.snapshots).reverse() + + // Collect all possible domains + const domains = new Set() + for (const s of snapshots) { + Object.keys(s.domains).forEach((it) => domains.add(it as Domain)) + } + + for (const dd of domains) { + if (domain !== undefined && dd !== domain) { + continue + } + console.log('checking:', dd) + const sDigest = await loadDigest(toolCtx, storage, snapshots, dd) + if (sDigest.has(id)) { + console.log('we found file') + let found = false + for (const sn of rnapshots) { + const ssDigest = await loadDigest(toolCtx, storage, [sn], dd) + if (!ssDigest.has(id)) { + continue + } + const d = sn.domains[dd] + if (found && !showAll) { + break + } + for (const sf of d?.storage ?? []) { + if (found && !showAll) { + break + } + console.log('processing', sf) + const readStream = await storage.load(sf) + const ex = extract() + ex.on('entry', (headers, stream, next) => { + if (headers.name === id + '.json') { + console.log('file found in:', sf) + + const chunks: Buffer[] = [] + stream.on('data', (chunk) => { + chunks.push(chunk) + }) + stream.on('end', () => { + const bf = Buffer.concat(chunks as any) + console.log('>>>>>>>>>>>') + console.log(JSON.stringify(JSON.parse(bf.toString()), undefined, 2)) + console.log('>>>>>>>>>>>') + next() + }) + + found = true + } else { + stream.resume() // auto drain for non-matching entries + next() // continue to the next entry + } + }) + + const endPromise = new Promise((resolve) => { + ex.on('finish', () => { + resolve(null) + }) + }) + const unzip = createGunzip({ level: defaultLevel }) + + readStream.on('end', () => { + readStream.destroy() + }) + readStream.pipe(unzip) + unzip.pipe(ex) + + await endPromise + } + } + } + } +} + +export function extendZero (value: number, count: number = 4): string { + let idxName = `${value}` + while (idxName.length < count) { + idxName = '0' + idxName + } + return idxName +} + +/** + * Compacting backup into just one snapshot. + * @public + */ +export async function compactBackup ( + ctx: MeasureContext, + storage: BackupStorage, + force: boolean = false, + opt?: { + blobLimit?: number + skipContentTypes?: string[] + }, + recalculateDigest: boolean = false +): Promise { + console.log('starting backup compaction') + + const tmpRoot = mkdtempSync('huly') + + try { + let backupInfo: BackupInfo + + // Version 0.6.2, format of digest file is changed to + + const infoFile = 'backup.json.gz' + + if (await storage.exists(infoFile)) { + backupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + } else { + console.log('No backup found') + return + } + if (backupInfo.version !== '0.6.2') { + console.log('Invalid backup version') + return + } + + if (backupInfo.snapshots.length < 1 && !force) { + console.log('No need to compact') + return + } + + // Find compacting backup or create new one + const snapshot: BackupSnapshot = backupInfo.snapshots.find((it) => it.compacting === true) ?? { + date: Date.now(), + domains: {}, + compacting: true, + stIndex: 0 + } + + let backupIndex = `${(backupInfo.snapshotsIndex ?? backupInfo.snapshots.length) + 1}` + while (backupIndex.length < 6) { + backupIndex = '0' + backupIndex + } + + const domains: Domain[] = [] + for (const sn of backupInfo.snapshots) { + for (const d of Object.keys(sn.domains)) { + if (!domains.includes(d as Domain)) { + domains.push(d as Domain) + } + } + } + const snapshotsToClean = backupInfo.snapshots.filter((it) => it.compacting !== true).reverse() + backupInfo.snapshots.push(snapshot) + + const dirsToClean = new Set() + const filesToClean = new Set() + + const cleanFiles = async (): Promise => { + if (filesToClean.size > 0) { + for (const file of filesToClean) { + try { + await storage.delete(file) + } catch (err: any) { + // Ignore + } + } + filesToClean.clear() + } + } + + for (const domain of domains) { + console.log('compacting domain...', domain) + + const processedChanges: Snapshot = { + added: new Map(), + updated: new Map(), + removed: [] + } + + let changed = 0 + let snapshotIndex = 0 + const domainInfo: DomainData = { + snapshot: undefined, + snapshots: [], + storage: [], + added: 0, + updated: 0, + removed: 0 + } + + // Cumulative digest + // Documents modified in compacting snapshots in case of restart + const untouchedDigest = await loadDigest(ctx, storage, [snapshot], domain) + + // We need to load snapshots from removed ones and + const digest = await loadDigest(ctx, storage, snapshotsToClean, domain) + + // We remove all items we have in last part + Array.from(untouchedDigest.keys()).forEach((it) => digest.delete(it)) + + const digestAdded = new Map, string>() + + let _pack: Pack | undefined + let _packClose = async (): Promise => {} + let addedDocuments: () => number = () => 0 + + let processed = 0 + + let skipBlobs = 0 + let skipSize = 0 + + let lastSkipPrint = 0 + + const blobs = new Map() + + async function pushDocs (doc: Doc | undefined, size: number, blobData: Record, Buffer>): Promise { + if (doc == null) { + return + } + changed += 1 + // Chunk data into small pieces + // The size check 'addedDocuments() > dataBlobSize' is performed here, evaluating the archive's size + // *before* the documents in the current 'docs' batch are added. + // If the archive is, e.g., 49MB, this check passes. If the current 'docs' batch then adds 5MB, + // the archive becomes 54MB. This oversized archive is closed only on the *next* call to pushDocs. + const addedDocsValue = addedDocuments() + if (addedDocsValue > dataBlobSize && _pack !== undefined) { + await _packClose() + + if (changed > 0) { + snapshot.domains[domain] = domainInfo + domainInfo.added += processedChanges.added.size + domainInfo.updated += processedChanges.updated.size + domainInfo.removed += processedChanges.removed.length + + snapshotIndex++ + const snapshotFile = join(backupIndex, `${domain}-${snapshot.date}-${extendZero(snapshotIndex)}.snp.gz`) + domainInfo.snapshots = [...(domainInfo.snapshots ?? []), snapshotFile] + await writeChanges(storage, snapshotFile, processedChanges) + + processedChanges.added.clear() + processedChanges.removed = [] + processedChanges.updated.clear() + await storage.writeFile( + infoFile, + gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel }) + ) + + await cleanFiles() + } + } + if (_pack === undefined) { + _pack = pack() + snapshot.stIndex++ + const storageFile = join( + backupIndex, + `${domain}-data-${snapshot.date}-${extendZero(snapshot.stIndex)}.tar.gz` + ) + console.log('storing from domain', domain, storageFile) + domainInfo.storage = [...(domainInfo.storage ?? []), storageFile] + + const tmpFile = join(tmpRoot, basename(storageFile) + '.tmp') + const tempFile = createWriteStream(tmpFile) + // const dataStream = await storage.write(storageFile) + + const sizePass = new PassThrough() + let sz = 0 + sizePass._transform = (chunk, encoding, cb) => { + // No transformation, just pass through data + sz += chunk.length + sizePass.push(chunk) + cb() + } + + sizePass.pipe(tempFile) + + const storageZip = createGzip({ level: defaultLevel, memLevel: 9 }) + addedDocuments = () => sz + + _pack.pipe(storageZip) + storageZip.pipe(sizePass) + + _packClose = async () => { + ctx.info('finalize pack', { storageFile, size: sz }) + await new Promise((resolve) => { + tempFile.on('close', () => { + resolve() + }) + _pack?.finalize() + }) + // We need to upload file to storage + ctx.info('>>>> upload pack', { storageFile, size: sz }) + await storage.writeFile(storageFile, createReadStream(tmpFile)) + await rm(tmpFile) + + _pack = undefined + } + } + + // Move processed document to processedChanges + processedChanges.added.set(doc._id, digestAdded.get(doc._id) ?? '') + + if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') { + const blob = doc as Blob | BlobData + + const data = blobData[blob._id] + const descrJson = JSON.stringify(doc) + await new Promise((resolve, reject) => { + _pack?.entry({ name: doc._id + '.json' }, descrJson, function (err) { + if (err != null) reject(err) + resolve() + }) + }) + await new Promise((resolve, reject) => { + _pack?.entry({ name: doc._id }, data, function (err) { + if (err != null) reject(err) + resolve() + }) + }) + } else { + const data = JSON.stringify(doc) + await new Promise((resolve, reject) => { + _pack?.entry({ name: doc._id + '.json' }, data, function (err) { + if (err != null) reject(err) + resolve() + }) + }) + } + } + async function sendChunk (doc: Doc | undefined, len: number, blobData: Record, Buffer>): Promise { + if (doc !== undefined) { + if (domain === DOMAIN_BLOB) { + if (opt?.skipContentTypes !== undefined || (opt?.blobLimit ?? 0) > 0) { + // Check if we need to skip blob + const blob = doc as Blob + + const newSkipPrint = Math.round(skipSize / (1024 * 1024 * 100)) + if (newSkipPrint !== lastSkipPrint) { + lastSkipPrint = newSkipPrint + console.log('skipping blobs', skipBlobs, Math.round(skipSize / (1024 * 1024))) + } + const bsize = blob.size == null || Number.isNaN(blob.size) || !Number.isInteger(blob.size) ? 0 : blob.size + + if ( + opt?.skipContentTypes !== undefined && + opt?.skipContentTypes.some((it) => (blob.contentType ?? '').includes(it)) + ) { + skipBlobs++ + skipSize += bsize + digest.delete(doc._id) + return + } + if (opt?.blobLimit !== undefined && opt?.blobLimit > 0 && bsize > opt.blobLimit) { + skipBlobs++ + skipSize += bsize + digest.delete(doc._id) + return + } + } + } + const hash = digest.get(doc._id) + digest.delete(doc._id) + digestAdded.set(doc._id, hash ?? '') + await pushDocs(doc, len, blobData) + } + } + + for (const s of snapshotsToClean) { + const d = s.domains[domain] + + if (d !== undefined && digest.size > 0) { + ctx.info('checking-domain', { domain, name: s.date }) + const sDigest = await loadDigest(ctx, storage, [s], domain) + const requiredDocs = new Map(Array.from(sDigest.entries()).filter(([it]) => digest.has(it))) + if (requiredDocs.size > 0) { + console.log('updating', domain, requiredDocs.size) + // We have required documents here. + for (const sf of d.storage ?? []) { + if (digest.size === 0) { + break + } + try { + console.log('processing', sf, processed) + + const readStream = await storage.load(sf) + const ex = extract() + + ex.on('entry', (headers, stream, next) => { + const name = headers.name ?? '' + processed++ + // We found blob data + if (requiredDocs.has(name as Ref)) { + const chunks: Buffer[] = [] + stream.on('data', (chunk) => { + chunks.push(chunk) + }) + stream.on('end', () => { + const bf = Buffer.concat(chunks as any) + const d = blobs.get(name) + if (d === undefined) { + blobs.set(name, { doc: undefined, buffer: bf }) + next() + } else { + blobs.delete(name) + const doc = d.doc as Blob | undefined // d here is from the outer scope, known not to be undefined. + if (doc == null) { + const errorMsg = `CompactBackup: Metadata (doc) missing in blobs entry for data file ${name}` + ctx.error(errorMsg, { name }) + next() // Just skip + return + } + void sendChunk(doc, bf.length, { [doc._id]: bf }) + .finally(() => { + requiredDocs.delete(doc._id) + next() + }) + .catch((err) => { + next(err) + }) + } + }) + } else if ( + name.endsWith('.json') && + requiredDocs.has(name.substring(0, name.length - 5) as Ref) + ) { + const chunks: Buffer[] = [] + const bname = name.substring(0, name.length - 5) + stream.on('data', (chunk) => { + chunks.push(chunk) + }) + stream.on('end', () => { + const bf = Buffer.concat(chunks as any) + let doc: Doc + try { + doc = JSON.parse(bf.toString()) as Doc + } catch (err: any) { + console.error(err) + next() + return + } + if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') { + const d = blobs.get(bname) + if (d === undefined) { + blobs.set(bname, { doc, buffer: undefined }) + next() + } else { + blobs.delete(bname) + ;(doc as any)['%hash%'] = digest.get(doc._id) + void sendChunk(doc, bf.length, { [doc._id]: d?.buffer as Buffer }) + .finally(() => { + requiredDocs.delete(doc._id) + next() + }) + .catch((err) => { + ctx.error('failed to sendChunk', { err }) + next(err) + }) + } + } else { + ;(doc as any)['%hash%'] = digest.get(doc._id) + void sendChunk(doc, bf.length, {}) + .finally(() => { + requiredDocs.delete(doc._id) + next() + }) + .catch((err) => { + ctx.error('failed to sendChunk', { err }) + next(err) + }) + } + }) + } else { + next() + } + stream.resume() // just auto drain the stream + }) + + const unzip = createGunzip({ level: defaultLevel }) + const endPromise = new Promise((resolve, reject) => { + ex.on('finish', () => { + resolve(null) + }) + readStream.on('error', (err) => { + ctx.error('error during processing', { snapshot, err }) + reject(err) + }) + unzip.on('error', (err) => { + ctx.error('error during processing', { snapshot, err }) + reject(err) + }) + }) + + readStream.on('end', () => { + readStream.destroy() + }) + readStream.pipe(unzip) + unzip.pipe(ex) + + await endPromise + } catch (err: any) { + ctx.error('error processing', err) + } + } + } else { + console.log('domain had no changes', domain) + } + } + if (d !== undefined) { + for (const sf of d.storage ?? []) { + console.log('removing', sf) + dirsToClean.add(dirname(sf)) + filesToClean.add(sf) + } + for (const sf of d.snapshots ?? []) { + console.log('removing', sf) + dirsToClean.add(dirname(sf)) + filesToClean.add(sf) + } + if (d.snapshot !== undefined) { + dirsToClean.add(dirname(d.snapshot)) + filesToClean.add(d.snapshot) + } + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete s.domains[domain] + } + } + + if (changed > 0) { + domainInfo.added += processedChanges.added.size + domainInfo.updated += processedChanges.updated.size + domainInfo.removed += processedChanges.removed.length + + if (domainInfo.added + domainInfo.updated + domainInfo.removed > 0) { + snapshot.domains[domain] = domainInfo + + snapshotIndex++ + const snapshotFile = join(backupIndex, `${domain}-${snapshot.date}-${extendZero(snapshotIndex)}.snp.gz`) + domainInfo.snapshots = [...(domainInfo.snapshots ?? []), snapshotFile] + await writeChanges(storage, snapshotFile, processedChanges) + } + + processedChanges.added.clear() + processedChanges.removed = [] + processedChanges.updated.clear() + await _packClose() + } + // This will allow to retry in case of critical error. + await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) + } + snapshot.compacting = false + backupInfo.snapshots = [snapshot] + backupInfo.snapshotsIndex = (backupInfo.snapshotsIndex ?? backupInfo.snapshots.length) + 1 + await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) + + await cleanFiles() + for (const dir of dirsToClean) { + try { + await storage.deleteRecursive(dir) + } catch (err: any) { + // Ignore + } + } + const result: BackupResult = { + result: true, + dataSize: 0, + blobsSize: 0, + backupSize: 0 + } + if (recalculateDigest) { + await rebuildSizeInfo(storage, [], ctx, result, backupInfo, infoFile) + } + } catch (err: any) { + console.error(err) + } finally { + await rm(tmpRoot, { recursive: true }) + console.log('end compacting') + } +} + +export * from './service' +export function migradeBlobData (blob: Blob, etag: string): string { + if (blob._class === 'core:class:BlobData') { + const bd = blob as unknown as BlobData + blob.contentType = blob.contentType ?? bd.type + blob.etag = etag + blob._class = core.class.Blob + delete (blob as any).type + const result = (blob as any).base64Data + delete (blob as any).base64Data + return result + } + return '' +} + +/** + * Will check backup integrity, and in case of some missing resources, will update digest files, so next backup will backup all missing parts. + * @public + */ +export async function checkBackupIntegrity (ctx: MeasureContext, storage: BackupStorage): Promise { + console.log('check backup integrity') + try { + let backupInfo: BackupInfo + + // Version 0.6.2, format of digest file is changed to + + const infoFile = 'backup.json.gz' + + if (await storage.exists(infoFile)) { + backupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + } else { + console.log('No backup found') + return + } + if (backupInfo.version !== '0.6.2') { + console.log('Invalid backup version') + return + } + + const recheckSizes: string[] = [] + + const domains: Domain[] = [] + for (const sn of backupInfo.snapshots) { + for (const d of Object.keys(sn.domains)) { + if (!domains.includes(d as Domain)) { + domains.push(d as Domain) + } + } + } + let modified = false + + for (const domain of domains) { + console.log('checking domain...', domain) + const { modified: mm, modifiedFiles } = await verifyDigest(ctx, storage, backupInfo.snapshots, domain) + if (mm) { + recheckSizes.push(...modifiedFiles) + modified = true + } + } + if (backupInfo.migrations == null) { + backupInfo.migrations = {} + } + if (backupInfo.migrations.zeroCheckSize !== true) { + backupInfo.migrations.zeroCheckSize = true + modified = true + } + if (modified) { + await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })) + } + + const bresult: BackupResult = { + backupSize: 0, + blobsSize: 0, + dataSize: 0, + result: true + } + await rebuildSizeInfo(storage, recheckSizes, ctx, bresult, backupInfo, infoFile) + } catch (err: any) { + console.error(err) + } finally { + console.log('end compacting') + } +} + +export function doTrimHash (s: string | undefined): string | undefined { + if (s == null) { + return undefined + } + if (s.startsWith('"') && s.endsWith('"')) { + return s.slice(1, s.length - 1) + } + return s +} + +export async function loadDigest ( + ctx: MeasureContext, + storage: BackupStorage, + snapshots: BackupSnapshot[], + domain: Domain, + date?: number +): Promise, string>> { + const result = new Map, string>() + for (const s of snapshots) { + const d = s.domains[domain] + + // Load old JSON snapshot + if (d?.snapshot !== undefined) { + try { + const dChanges: SnapshotV6 = JSON.parse( + gunzipSync(new Uint8Array(await storage.loadFile(d.snapshot))).toString() + ) + for (const [k, v] of Object.entries(dChanges.added)) { + result.set(k as Ref, v) + } + for (const [k, v] of Object.entries(dChanges.updated)) { + result.set(k as Ref, v) + } + for (const d of dChanges.removed) { + result.delete(d) + } + } catch (err: any) { + ctx.warn('failed to load digest', { snapshot: d.snapshot }) + } + } + for (const snapshot of d?.snapshots ?? []) { + try { + const dataBlob = gunzipSync(new Uint8Array(await storage.loadFile(snapshot))) + .toString() + .split('\n') + const addedCount = parseInt(dataBlob.shift() ?? '0') + const added = dataBlob.splice(0, addedCount) + for (const it of added) { + const [k, v] = it.split(';') + result.set(k as Ref, v) + } + + const updatedCount = parseInt(dataBlob.shift() ?? '0') + const updated = dataBlob.splice(0, updatedCount) + for (const it of updated) { + const [k, v] = it.split(';') + result.set(k as Ref, v) + } + + const removedCount = parseInt(dataBlob.shift() ?? '0') + const removed = dataBlob.splice(0, removedCount) + for (const k of removed) { + result.delete(k as Ref) + } + } catch (err: any) { + ctx.warn('digest is broken', { domain, err: err.message, snapshot }) + } + } + // Stop if stop date is matched and provided + if (date !== undefined && date === s.date) { + break + } + } + // ctx.info('load-digest', { domain, snapshots: snapshots.length, documents: result.size }) + return result +} +export async function verifyDigest ( + ctx: MeasureContext, + storage: BackupStorage, + snapshots: BackupSnapshot[], + domain: Domain +): Promise<{ modified: boolean, modifiedFiles: string[] }> { + ctx = ctx.newChild('verify digest', { domain, count: snapshots.length }) + ctx.info('verify-digest', { domain, count: snapshots.length }) + let modified = false + const modifiedFiles: string[] = [] + for (const s of snapshots) { + const d = s.domains[domain] + if (d === undefined) { + continue + } + + const storageToRemove = new Set() + // We need to verify storage has all necessary resources + ctx.info('checking', { domain }) + // We have required documents here. + const validDocs = new Set>() + const zeroEntres = new Set>() + + for (const sf of d.storage ?? []) { + const blobs = new Map() + try { + ctx.info('checking storage', { sf }) + const readStream = await storage.load(sf) + const ex = extract() + + ex.on('entry', (headers, stream, next) => { + const name = headers.name ?? '' + // We found blob data + if (name.endsWith('.json')) { + const chunks: Buffer[] = [] + const bname = name.substring(0, name.length - 5) + stream.on('data', (chunk) => { + chunks.push(chunk) + }) + stream.on('end', () => { + try { + const bf = Buffer.concat(chunks as any) + const doc = JSON.parse(bf.toString()) as Doc + if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') { + const data = migradeBlobData(doc as Blob, '') + const d = blobs.get(bname) ?? (data !== '' ? Buffer.from(data, 'base64') : undefined) + if (d === undefined) { + blobs.set(bname, { doc, buffer: undefined }) + } else { + blobs.delete(bname) + } + } + validDocs.add(bname as Ref) + } catch (err: any) { + // If not a json, skip + } + next() + }) + } else { + if (headers.size === 0) { + zeroEntres.add(name as any) + } + next() + } + stream.resume() // just auto drain the stream + }) + + const unzip = createGunzip({ level: defaultLevel }) + const endPromise = new Promise((resolve, reject) => { + readStream.on('error', (err) => { + console.error(err) + reject(err) + }) + ex.on('finish', () => { + resolve(null) + }) + unzip.on('error', (err) => { + ctx.error('error during reading of', { sf, err }) + modified = true + storageToRemove.add(sf) + resolve(null) + }) + }) + + readStream.on('end', () => { + readStream.destroy() + }) + readStream.pipe(unzip) + unzip.pipe(ex) + + await endPromise + } catch (err: any) { + ctx.error('error during reading of', { sf, err }) + // In case of invalid archive, we need to + // We need to remove broken storage file + modified = true + storageToRemove.add(sf) + } + } + + // Clear zero files, they potentially wrong downloaded. + for (const zz of zeroEntres.values()) { + validDocs.delete(zz) + } + if (storageToRemove.size > 0) { + modified = true + d.storage = (d.storage ?? []).filter((it) => !storageToRemove.has(it)) + modifiedFiles.push(...Array.from(storageToRemove)) + for (const sf of storageToRemove) { + await storage.delete(sf) + } + } + let mfiles: string[] = [] + ;({ modified, modifiedFiles: mfiles } = await updateDigest(d, ctx, storage, validDocs, modified, domain)) + modifiedFiles.push(...mfiles) + } + ctx.end() + return { modified, modifiedFiles } +} + +export async function updateDigest ( + d: DomainData, + ctx: MeasureContext, + storage: BackupStorage, + validDocs: Set>, + modified: boolean, + domain: Domain +): Promise<{ modified: boolean, modifiedFiles: string[] }> { + const digestToRemove = new Set() + const modifiedFiles: string[] = [] + for (const snapshot of d?.snapshots ?? []) { + try { + ctx.info('checking', { snapshot }) + const changes: Snapshot = { + added: new Map(), + removed: [], + updated: new Map() + } + let lmodified = false + try { + const dataBlob = gunzipSync(new Uint8Array(await storage.loadFile(snapshot))) + .toString() + .split('\n') + const addedCount = parseInt(dataBlob.shift() ?? '0') + const added = dataBlob.splice(0, addedCount) + for (const it of added) { + const [k, v] = it.split(';') + if (validDocs.has(k as any)) { + changes.added.set(k as Ref, v) + } else { + lmodified = true + } + } + + const updatedCount = parseInt(dataBlob.shift() ?? '0') + const updated = dataBlob.splice(0, updatedCount) + for (const it of updated) { + const [k, v] = it.split(';') + if (validDocs.has(k as any)) { + changes.updated.set(k as Ref, v) + } else { + lmodified = true + } + } + + const removedCount = parseInt(dataBlob.shift() ?? '0') + const removed = dataBlob.splice(0, removedCount) + changes.removed = removed as Ref[] + if (addedCount === 0 && removedCount === 0 && updatedCount === 0) { + // Empty digest, need to clean + digestToRemove.add(snapshot) + lmodified = true + } + } catch (err: any) { + ctx.warn('failed during processing of snapshot file, it will be skipped', { snapshot }) + digestToRemove.add(snapshot) + modified = true + } + + if (lmodified) { + modified = true + if (digestToRemove.has(snapshot)) { + await storage.delete(snapshot) // No need for digest, lets' remove it + } else { + // Store changes without missing files + await writeChanges(storage, snapshot, changes) + } + } + } catch (err: any) { + digestToRemove.add(snapshot) + modifiedFiles.push(snapshot) + ctx.warn('digest is broken', { domain, err: err.message, snapshot }) + modified = true + } + } + d.snapshots = (d.snapshots ?? []).filter((it) => !digestToRemove.has(it)) + return { modified, modifiedFiles } +} + +async function write (chunk: any, stream: Writable): Promise { + let needDrain = false + await new Promise((resolve, reject) => { + needDrain = !stream.write(chunk, (err) => { + if (err != null) { + reject(err) + } else { + resolve(null) + } + }) + }) + if (needDrain) { + await new Promise((resolve) => stream.once('drain', resolve)) + } +} + +export async function writeChanges (storage: BackupStorage, snapshot: string, changes: Snapshot): Promise { + const snapshotWritable = await storage.write(snapshot) + const writable = createGzip({ level: defaultLevel }) + writable.pipe(snapshotWritable) + + // Write size + await write(`${changes.added.size}\n`, writable) + for (const [k, v] of changes.added.entries()) { + await write(`${k};${v}\n`, writable) + } + await write(`${changes.updated.size}\n`, writable) + for (const [k, v] of changes.updated.entries()) { + await write(`${k};${v}\n`, writable) + } + await write(`${changes.removed.length}\n`, writable) + for (const k of changes.removed) { + await write(`${k}\n`, writable) + } + writable.end() + await new Promise((resolve) => { + writable.flush(() => { + resolve(null) + }) + }) +} + +export async function verifyDocsFromSnapshot ( + ctx: MeasureContext, + domain: Domain, + d: DomainData, + s: BackupSnapshot, + storage: BackupStorage, + digest: Map, string>, + verify: (docs: Doc[]) => Promise, + chunkSize: number +): Promise<{ modified: boolean, modifiedFiles: string[] }> { + let result: Doc[] = [] + const storageToRemove = new Set() + const validDocs = new Set>() + const modifiedFiles: string[] = [] + if (digest.size > 0) { + const sDigest = await loadDigest(ctx, storage, [s], domain) + const requiredDocs = new Map(Array.from(sDigest.entries()).filter(([it]) => digest.has(it))) + + if (requiredDocs.size > 0) { + ctx.info('updating', { domain, requiredDocs: requiredDocs.size }) + // We have required documents here. + for (const sf of d.storage ?? []) { + if (digest.size === 0) { + break + } + try { + const readStream = await storage.load(sf) + const ex = extract() + + ex.on('entry', (headers, stream, next) => { + const name = headers.name ?? '' + // We found blob data + const rdoc = name.substring(0, name.length - 5) as Ref + if (name.endsWith('.json') && requiredDocs.has(rdoc)) { + const chunks: Buffer[] = [] + const bname = name.substring(0, name.length - 5) + stream.on('data', (chunk) => { + chunks.push(chunk) + }) + stream.on('end', () => { + const bf = Buffer.concat(chunks as any) + let doc: Doc + try { + doc = JSON.parse(bf.toString()) as Doc + } catch (err) { + // Do not failure on this. + next() + return + } + + if (doc._class === core.class.Blob || doc._class === 'core:class:BlobData') { + // Skip blob + validDocs.add(bname as Ref) + } else { + ;(doc as any)['%hash%'] = digest.get(rdoc) + digest.delete(rdoc) + result.push(doc) + validDocs.add(bname as Ref) + + if (result.length > chunkSize) { + void verify(result) + .then(() => { + result = [] + next() + }) + .catch((err) => { + ctx.error('failed to verify', { err }) + next(err) + }) + } else { + next() + } + } + }) + } else { + next() + } + stream.resume() // just auto drain the stream + }) + + const unzip = createGunzip({ level: defaultLevel }) + + const endPromise = new Promise((resolve, reject) => { + ex.on('finish', () => { + resolve(null) + }) + + readStream.on('end', () => { + readStream.destroy() + }) + readStream.pipe(unzip).on('error', (err) => { + readStream.destroy() + storageToRemove.add(sf) + reject(err) + }) + unzip.pipe(ex) + }) + + await endPromise + if (result.length > 0) { + await verify(result) + } + } catch (err: any) { + storageToRemove.add(sf) + ctx.error('failed to processing', { storageFile: sf }) + } + } + } + } + let modified = false + if (storageToRemove.size > 0) { + modifiedFiles.push(...Array.from(storageToRemove)) + d.storage = (d.storage ?? []).filter((it) => !storageToRemove.has(it)) + for (const sf of storageToRemove) { + await storage.delete(sf) + } + modified = true + } + let smodifiedFiles: string[] = [] + ;({ modified, modifiedFiles: smodifiedFiles } = await updateDigest(d, ctx, storage, validDocs, modified, domain)) + modifiedFiles.push(...smodifiedFiles) + return { modified, modifiedFiles } +} + +export async function rebuildSizeInfo ( + storage: BackupStorage, + recheckSizes: string[], + ctx: MeasureContext, + result: BackupResult, + backupInfo: BackupInfo, + infoFile: string +): Promise { + const sizeFile = 'backup.size.gz' + + let sizeInfo: Record = {} + + if (await storage.exists(sizeFile)) { + sizeInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(sizeFile))).toString()) + } + let processed = 0 + + for (const file of recheckSizes) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete sizeInfo[file] + } + + const addFileSize = async (file: string | undefined | null): Promise => { + if (file != null) { + try { + const sz = sizeInfo[file] + const fileSize = sz ?? (await storage.stat(file)) + if (sz === undefined) { + sizeInfo[file] = fileSize + processed++ + if (processed % 10 === 0) { + ctx.info('Calculate size processed', { processed, size: Math.round(result.backupSize / (1024 * 1024)) }) + } + } + result.backupSize += fileSize + } catch (err: any) { + ctx.error('failed to calculate size', { file, err }) + } + } + } + + // Let's calculate data size for backup + for (const sn of backupInfo.snapshots) { + for (const [, d] of Object.entries(sn.domains)) { + await addFileSize(d.snapshot) + for (const snp of d.snapshots ?? []) { + await addFileSize(snp) + } + for (const snp of d.storage ?? []) { + await addFileSize(snp) + } + } + } + await addFileSize(infoFile) + + await storage.writeFile(sizeFile, gzipSync(JSON.stringify(sizeInfo, undefined, 2), { level: defaultLevel })) +} diff --git a/server/client/src/blob.ts b/server/client/src/blob.ts index 5163412747..65171c8611 100644 --- a/server/client/src/blob.ts +++ b/server/client/src/blob.ts @@ -19,51 +19,18 @@ import { Buffer } from 'node:buffer' // Will use temporary file to store huge content into export class BlobClient { - transactorAPIUrl: string index: number constructor ( - readonly transactorUrl: string, - readonly token: string, - readonly workspace: WorkspaceIds, - readonly opt?: { - storageAdapter?: StorageAdapter - } + readonly storageAdapter: StorageAdapter, + readonly workspace: WorkspaceIds ) { this.index = 0 - let url = transactorUrl - if (url.endsWith('/')) { - url = url.slice(0, url.length - 1) - } - - this.transactorAPIUrl = url.replaceAll('wss://', 'https://').replace('ws://', 'http://') + '/api/v1/blob' } async checkFile (ctx: MeasureContext, name: string): Promise { - if (this.opt?.storageAdapter !== undefined) { - const obj = await this.opt?.storageAdapter.stat(ctx, this.workspace, name) - if (obj !== undefined) { - return true - } - } - for (let i = 0; i < 5; i++) { - try { - const response = await fetch(this.transactorAPIUrl + `?name=${encodeURIComponent(name)}`, { - headers: { - Authorization: 'Bearer ' + this.token, - Range: 'bytes=0-1' - } - }) - if (response.status === 404) { - return false - } - const buff = await response.arrayBuffer() - return buff.byteLength > 0 - } catch (err: any) { - if (i === 4) { - ctx.error('Failed to check file', { name, error: err }) - } - await new Promise((resolve) => setTimeout(resolve, 10)) - } + const obj = await this.storageAdapter.stat(ctx, this.workspace, name) + if (obj !== undefined) { + return true } return false } @@ -81,75 +48,22 @@ export class BlobClient { const chunkSize = 50 * 1024 * 1024 // Use ranges to iterave through file with retry if required. - while (written < size || size === -1) { + while (written < size) { let i = 0 - let response: Response | undefined for (; i < 5; i++) { try { - const st = Date.now() - let chunk: Buffer - - if (this.opt?.storageAdapter !== undefined) { - const chunks: Buffer[] = [] - const readable = await this.opt.storageAdapter.partial(ctx, this.workspace, name, written, chunkSize) - await new Promise((resolve) => { - readable.on('data', (chunk) => { - chunks.push(chunk) - }) - readable.on('end', () => { - readable.destroy() - resolve() - }) + const chunks: Buffer[] = [] + const readable = await this.storageAdapter.partial(ctx, this.workspace, name, written, chunkSize) + await new Promise((resolve) => { + readable.on('data', (chunk) => { + chunks.push(chunk) }) - chunk = Buffer.concat(chunks) - } else { - const header: Record = { - Authorization: 'Bearer ' + this.token - } - - if (!(size !== -1 && written === 0 && size < chunkSize)) { - header.Range = `bytes=${written}-${size === -1 ? written + chunkSize : Math.min(size - 1, written + chunkSize)}` - } - - response = await fetch(this.transactorAPIUrl + `?name=${encodeURIComponent(name)}`, { headers: header }) - if (header.Range != null) { - ctx.info('fetch part', { time: Date.now() - st, blobId: name, written, size }) - } - if (response.status === 403) { - i = 5 - // No file, so make it empty - throw new Error(`Unauthorized ${this.transactorAPIUrl}/${this.workspace.uuid}/${name}`) - } - if (response.status === 404) { - i = 5 - // No file, so make it empty - throw new Error(`No file for ${this.transactorAPIUrl}/${this.workspace.uuid}/${name}`) - } - if (response.status === 416) { - if (size === -1) { - size = parseInt((response.headers.get('content-range') ?? '').split('*/')[1]) - continue - } - - // No file, so make it empty - throw new Error(`No file for ${this.transactorAPIUrl}/${this.workspace.uuid}/${name}`) - } - chunk = Buffer.from(await response.arrayBuffer()) - - if (header.Range == null) { - size = chunk.length - } - // We need to parse - // 'Content-Range': `bytes ${start}-${end}/${size}` - // To determine if something is left - const range = response.headers.get('Content-Range') - if (range !== null) { - const [, total] = range.split(' ')[1].split('/') - if (total !== undefined) { - size = parseInt(total) - } - } - } + readable.on('end', () => { + readable.destroy() + resolve() + }) + }) + const chunk = Buffer.concat(chunks) await new Promise((resolve, reject) => { writable.write(chunk, (err) => { @@ -189,44 +103,6 @@ export class BlobClient { } async upload (ctx: MeasureContext, name: string, size: number, contentType: string, buffer: Buffer): Promise { - if (this.opt?.storageAdapter !== undefined) { - await this.opt.storageAdapter.put(ctx, this.workspace, name, buffer, contentType, size) - } else { - // TODO: We need to improve this logig, to allow restore of huge blobs - for (let i = 0; i < 5; i++) { - try { - const resp = await ( - await fetch( - this.transactorAPIUrl + - `?name=${encodeURIComponent(name)}&contentType=${encodeURIComponent(contentType)}&size=${size}`, - { - keepalive: true, - method: 'PUT', - headers: { - Authorization: 'Bearer ' + this.token, - 'Content-Type': contentType - }, - body: buffer - } - ) - ).text() - try { - const json = JSON.parse(resp) - if (json.error !== undefined) { - ctx.error('failed to upload file, error from server', { name, message: json.error }) - return - } - } catch (err) { - console.log(err) - } - break - } catch (err: any) { - if (i === 4) { - ctx.error('failed to upload file', { name, message: err.message, cause: err.cause?.message }) - throw err - } - } - } - } + await this.storageAdapter.put(ctx, this.workspace, name, buffer, contentType, size) } } diff --git a/server/core/src/storage.ts b/server/core/src/storage.ts index 2b62228a2c..05182ec50f 100644 --- a/server/core/src/storage.ts +++ b/server/core/src/storage.ts @@ -18,7 +18,7 @@ export function getBucketId (workspace: WorkspaceUuid): string { return workspace } -const chunkSize = 5000 +const chunkSize = 50000 /** * @public @@ -45,6 +45,7 @@ export class BackupClientOps { idx?: number ): Promise<{ idx: number + size?: number docs: DocInfo[] finished: boolean }> { diff --git a/server/core/src/types.ts b/server/core/src/types.ts index 5c3ae9cea5..d75b4ce05a 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -727,6 +727,8 @@ export interface SessionManager { forceClose: (wsId: WorkspaceUuid, ignoreSocket?: ConnectionSocket) => Promise + forceMaintenance: (ctx: MeasureContext, workspaceId: WorkspaceUuid) => Promise + closeWorkspaces: (ctx: MeasureContext) => Promise scheduleMaintenance: (timeMinutes: number, message?: string) => void diff --git a/server/server/src/sessionManager.ts b/server/server/src/sessionManager.ts index 08ba0c147d..f4d46b98bc 100644 --- a/server/server/src/sessionManager.ts +++ b/server/server/src/sessionManager.ts @@ -250,13 +250,18 @@ export class TSessionManager implements SessionManager { } // Wait some time for new client to appear before closing workspace. - if (workspace.sessions.size === 0 && workspace.closing === undefined && workspace.workspaceInitCompleted) { + if ( + !workspace.maintenance && + workspace.sessions.size === 0 && + workspace.closing === undefined && + workspace.workspaceInitCompleted + ) { workspace.softShutdown-- if (workspace.softShutdown <= 0) { this.ctx.warn('closing workspace, no users', { workspace: workspace.wsId.url, wsId, - upgrade: workspace.upgrade + upgrade: workspace.maintenance }) workspace.closing = this.performWorkspaceCloseCheck(workspace) } @@ -474,7 +479,7 @@ export class TSessionManager implements SessionManager { } if (token.extra?.model === 'upgrade') { - if (workspace.upgrade) { + if (workspace.maintenance) { ctx.warn('reconnect workspace in upgrade', { account: token.account, workspace: workspaceUuid, @@ -491,7 +496,7 @@ export class TSessionManager implements SessionManager { await this.switchToUpgradeSession(token, ctx.parent ?? ctx, workspace, ws) } } else { - if (workspace.upgrade) { + if (workspace.maintenance || this.maintenanceWorkspaces.has(workspace.wsId.uuid)) { ctx.warn('connect during upgrade', { account: token.account, workspace: workspace.wsId.url, @@ -512,6 +517,8 @@ export class TSessionManager implements SessionManager { socialIds: [] } + maintenanceWorkspaces = new Set() + async addSession ( ctx: MeasureContext, ws: ConnectionSocket, @@ -624,7 +631,7 @@ export class TSessionManager implements SessionManager { } // Mark as upgrade, to prevent any new clients to connect during close - workspace.upgrade = true + workspace.maintenance = true // If upgrade client is used. // Drop all existing clients await this.doCloseAll(workspace, 0, 'upgrade', ws) @@ -644,7 +651,7 @@ export class TSessionManager implements SessionManager { } doBroadcast (ws: Workspace, tx: Tx[], target?: AccountUuid | AccountUuid[], exclude?: AccountUuid[]): void { - if (ws.upgrade) { + if (ws.maintenance) { return } if (target !== undefined && !Array.isArray(target)) { @@ -725,7 +732,7 @@ export class TSessionManager implements SessionManager { }) return } - if (workspace?.upgrade ?? false) { + if (workspace?.maintenance ?? false) { return } @@ -789,7 +796,7 @@ export class TSessionManager implements SessionManager { workspaceIds, branding ) - workspace.upgrade = upgrade + workspace.maintenance = upgrade this.workspaces.set(token.workspace, workspace) return workspace @@ -895,7 +902,7 @@ export class TSessionManager implements SessionManager { const user = sessionRef.session.getUser() if (workspace !== undefined) { const another = Array.from(workspace.sessions.values()).findIndex((p) => p.session.getUser() === user) - if (another === -1 && !workspace.upgrade) { + if (another === -1 && !workspace.maintenance) { void workspace.with(async (pipeline) => { await pipeline.closeSession(ctx, sessionRef.session.sessionId) // await communicationApi.closeSession(sessionRef.session.sessionId) @@ -924,11 +931,22 @@ export class TSessionManager implements SessionManager { } } + async forceMaintenance (ctx: MeasureContext, workspaceId: WorkspaceUuid): Promise { + const workspace = this.workspaces.get(workspaceId) + this.maintenanceWorkspaces.add(workspaceId) + if (workspace !== undefined) { + workspace.maintenance = true + ctx.info('force-maintenance', { workspaceId }) + await this.doCloseAll(workspace, 99, 'force-close') + } + } + async forceClose (wsId: WorkspaceUuid, ignoreSocket?: ConnectionSocket): Promise { const ws = this.workspaces.get(wsId) + this.maintenanceWorkspaces.delete(wsId) if (ws !== undefined) { this.ctx.warn('force-close', { name: ws.wsId.url }) - ws.upgrade = true // We need to similare upgrade to refresh all clients. + ws.maintenance = true // We need to similare upgrade to refresh all clients. ws.closing = this.doCloseAll(ws, 99, 'force-close', ignoreSocket) this.workspaces.delete(wsId) await ws.closing @@ -1194,7 +1212,7 @@ export class TSessionManager implements SessionManager { // TODO: we chould allow this only for admin or system accounts let done = false const wsRef = this.workspaces.get(workspaceId) - if (wsRef?.upgrade ?? false) { + if (wsRef?.maintenance ?? false) { done = true this.ctx.warn('FORCE CLOSE', { workspace: workspaceId }) // In case of upgrade, we need to force close workspace not in interval handler diff --git a/server/server/src/workspace.ts b/server/server/src/workspace.ts index 458bd71fef..120b49b55f 100644 --- a/server/server/src/workspace.ts +++ b/server/server/src/workspace.ts @@ -29,7 +29,7 @@ export type WorkspacePipelineFactory = () => Promise */ export class Workspace { pipeline?: Pipeline | Promise - upgrade: boolean = false + maintenance: boolean = false closing?: Promise workspaceInitCompleted: boolean = false diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index 7feaa31a0d..aeb64b9097 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -32,7 +32,6 @@ import core, { platformNow, platformNowDiff, type Ref, - systemAccountUuid, type Tx, type TxOperations, type WithLookup, @@ -59,13 +58,14 @@ import toolPlugin from './plugin' import { MigrateClientImpl } from './upgrade' import { getMetadata, PlatformError, unknownError } from '@hcengineering/platform' -import { generateToken } from '@hcengineering/server-token' import fs from 'fs' import * as yaml from 'js-yaml' import path from 'path' +import { sendTransactorEvent } from './utils' export * from './connect' export * from './plugin' +export * from './utils' export { toolPlugin as default } export class FileModelLogger implements ModelLogger { @@ -384,16 +384,8 @@ export async function upgradeModel ( // We need to send reboot for workspace ctx.info('send force close', { workspace: wsIds, transactorUrl }) - const serverEndpoint = transactorUrl.replaceAll('wss://', 'https://').replace('ws://', 'http://') - const token = generateToken(systemAccountUuid, wsIds.uuid, { service: 'tool', admin: 'true' }) - try { - await fetch(serverEndpoint + `/api/v1/manage?token=${token}&operation=force-close`, { - method: 'PUT' - }) - } catch (err: any) { - // Ignore error if transactor is not yet ready - } + await sendTransactorEvent(wsIds.uuid, 'force-close') return model } diff --git a/server/tool/src/utils.ts b/server/tool/src/utils.ts new file mode 100644 index 0000000000..60fdfff228 --- /dev/null +++ b/server/tool/src/utils.ts @@ -0,0 +1,34 @@ +import { systemAccountUuid, type WorkspaceUuid } from '@hcengineering/core' +import { getTransactorEndpoint } from '@hcengineering/server-client' +import { generateToken } from '@hcengineering/server-token' + +export function getToolToken (workspace?: WorkspaceUuid): string { + return generateToken(systemAccountUuid, workspace, { service: 'tool', admin: 'true' }) +} + +export async function getWorkspaceTransactorEndpoint ( + workspace: WorkspaceUuid, + type: 'external' | 'internal' = 'external' +): Promise { + return await getTransactorEndpoint(getToolToken(workspace), type) +} + +export async function sendTransactorEvent ( + workspace: WorkspaceUuid, + operation: 'force-maintenance' | 'force-close', + type: 'external' | 'internal' = 'external' +): Promise { + const token = getToolToken(workspace) + const serverEndpoint = (await getTransactorEndpoint(token, type)) + .replaceAll('wss://', 'https://') + .replace('ws://', 'http://') + + try { + console.info('send transactor event', operation, 'to', serverEndpoint) + await fetch(serverEndpoint + `/api/v1/manage?token=${token}&operation=${operation}`, { + method: 'PUT' + }) + } catch (err: any) { + // Ignore error if transactor is not yet ready + } +} diff --git a/server/workspace-service/src/service.ts b/server/workspace-service/src/service.ts index fc672c7c75..7eda634038 100644 --- a/server/workspace-service/src/service.ts +++ b/server/workspace-service/src/service.ts @@ -616,7 +616,6 @@ export class WorkspaceWorker { KeepSnapshots: 7 * 12 }, pipelineFactory, - workspaceStorageAdapter, (ctx, workspace, branding, externalStorage) => { return getConfig(ctx, dbUrl, ctx, { externalStorage, @@ -702,7 +701,6 @@ export class WorkspaceWorker { opt.backup.backupStorage, opt.backup.bucketName, pipelineFactory, - workspaceStorageAdapter, [DOMAIN_BLOB], true, (_p: number) => { diff --git a/services/backup/backup-api-pod/src/server.ts b/services/backup/backup-api-pod/src/server.ts index 33d2f874dc..63ec6780b4 100644 --- a/services/backup/backup-api-pod/src/server.ts +++ b/services/backup/backup-api-pod/src/server.ts @@ -57,10 +57,12 @@ async function traverseBackup ( ctx: MeasureContext, storage: BackupStorage, snapshot: (date: number) => void, - addFile: (name: string, size?: number) => void + addFile: (name: string, size?: number) => void, + addBlobInfo: (info: Record) => void ): Promise { const infoFile = 'backup.json.gz' const sizeFile = 'backup.size.gz' + const blobInfoFile = 'blob-info.json.gz' const backupInfoFile = await storage.loadFile(infoFile) @@ -80,6 +82,16 @@ async function traverseBackup ( console.error('failed to load size file', err) } + let blobInfo: Record = {} + try { + const blobInfoData = await storage.loadFile(blobInfoFile) + if (blobInfoData != null) { + blobInfo = JSON.parse(gunzipSync(blobInfoData).toString()) + } + } catch (err: any) { + console.error('failed to load size file', err) + } + const addFileSize = async (file: string | undefined | null): Promise => { if (file != null) { addFile(file, sizeInfo?.[file]) @@ -101,6 +113,7 @@ async function traverseBackup ( } sizeInfo[infoFile] = backupInfoFile.length await addFileSize(infoFile) + addBlobInfo(blobInfo) return backupInfo } @@ -218,7 +231,8 @@ export async function createServer (ctx: MeasureContext, config: Config): Promis backupInfoHTML += `${name} Size: ${size ?? 'unknown'} etag
\n` - } + }, + (info) => {} ) } catch (err: any) { backupInfoHTML += `Error: ${err.message}` @@ -235,10 +249,12 @@ export async function createServer (ctx: MeasureContext, config: Config): Promis if (file === 'index.json') { const jsonData: { files: { name: string, size?: number }[] + extraBlobs: { name: string, size: number, contentType: string }[] error?: string info?: BackupInfo } = { - files: [] + files: [], + extraBlobs: [] } try { @@ -248,6 +264,11 @@ export async function createServer (ctx: MeasureContext, config: Config): Promis (snapshot) => {}, (name, size) => { jsonData.files.push({ name, size }) + }, + (info) => { + for (const [name, [contentType, size]] of Object.entries(info)) { + jsonData.extraBlobs.push({ name, size, contentType }) + } } ) } catch (err: any) { diff --git a/tests/tool-local.sh b/tests/tool-local.sh index 4df90262ef..7ac219d92d 100755 --- a/tests/tool-local.sh +++ b/tests/tool-local.sh @@ -1,7 +1,5 @@ export MODEL_VERSION=$(node ../common/scripts/show_version.js) -export MINIO_ACCESS_KEY=minioadmin -export MINIO_SECRET_KEY=minioadmin -export MINIO_ENDPOINT=localhost:9000 +export STORAGE_CONFIG="datalake|http://localhost:4030" export MONGO_URL=mongodb://localhost:27017 export DB_URL=postgresql://root@localhost:26257/defaultdb?sslmode=disable export ACCOUNT_DB_URL=postgresql://root@localhost:26257/defaultdb?sslmode=disable diff --git a/tests/tool-pg.sh b/tests/tool-pg.sh index efb9e1df6b..580a10c031 100755 --- a/tests/tool-pg.sh +++ b/tests/tool-pg.sh @@ -1,9 +1,7 @@ #!/usr/bin/env bash export MODEL_VERSION=$(node ../common/scripts/show_version.js) -export MINIO_ACCESS_KEY=minioadmin -export MINIO_SECRET_KEY=minioadmin -export MINIO_ENDPOINT=localhost:9002 +export STORAGE_CONFIG="datalake|http://localhost:4031" export ACCOUNTS_URL=http://localhost:3003 export TRANSACTOR_URL=ws://localhost:3334 export ACCOUNT_DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable diff --git a/tests/tool.sh b/tests/tool.sh index b10f652c84..e69ffec165 100755 --- a/tests/tool.sh +++ b/tests/tool.sh @@ -1,9 +1,7 @@ #!/usr/bin/env bash export MODEL_VERSION=$(node ../common/scripts/show_version.js) -export MINIO_ACCESS_KEY=minioadmin -export MINIO_SECRET_KEY=minioadmin -export MINIO_ENDPOINT=localhost:9002 +export STORAGE_CONFIG="datalake|http://localhost:4031" export ACCOUNTS_URL=http://localhost:3003 export TRANSACTOR_URL=ws://localhost:3334 export ACCOUNT_DB_URL=mongodb://localhost:27018