diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 986f9c2d33..b347af1b33 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -46,6 +46,7 @@ import { backupDownload, backupFind, checkBackupIntegrity, + checkWorkspaceBackup, compactBackup, createFileBackupStorage, createStorageBackupStorage, @@ -639,6 +640,73 @@ export function devTool ( await storageAdapter.close() }) + program + .command('backup-check-workspace [date]') + .description( + 'Check whether all data from a backup is present in the workspace. Read-only: no data is added, removed or changed.' + ) + .action(async (dirName: string, workspaceId: string, date: string | undefined) => { + await withAccountDatabase(async (db) => { + const { txes, dbUrl } = prepareTools() + const ws = await getWorkspace(db, workspaceId) + if (ws === null) { + throw new Error(`workspace ${workspaceId} not found`) + } + + const wsIds = { uuid: ws.uuid, dataId: ws.dataId, url: ws.url ?? '' } + const storage = await createFileBackupStorage(dirName) + const workspaceStorage: StorageAdapter = buildStorageFromConfig(storageConfigFromEnv()) + + let pipeline: Pipeline | undefined + try { + pipeline = await createBackupPipeline(toolCtx, dbUrl, txes, { + externalStorage: workspaceStorage, + usePassedCtx: true + })( + toolCtx, + { + uuid: ws.uuid, + url: ws.url ?? '', + dataId: ws.dataId + }, + createEmptyBroadcastOps(), + null + ) + if (pipeline === undefined) { + toolCtx.error('failed to check, pipeline is undefined', { workspaceId }) + process.exitCode = 1 + return + } + + const result = await checkWorkspaceBackup(toolCtx, pipeline, wsIds, storage, parseInt(date ?? '-1')) + + console.log('') + for (const d of result.domains) { + const ok = d.missing.length === 0 && d.modified.length === 0 + console.log( + `${ok ? 'OK ' : 'FAIL'} ${d.domain}: backup=${d.backupCount} workspace=${d.workspaceCount} missing=${d.missing.length} modified=${d.modified.length}` + ) + } + console.log( + `${result.blobs.ok ? 'OK ' : 'FAIL'} blobs (storage): total=${result.blobs.total} missing=${result.blobs.missing.length}` + ) + console.log('') + if (result.ok) { + console.log('OK: workspace contains all data from backup') + } else { + console.log('FAILED: workspace is missing data present in the backup') + process.exitCode = 1 + } + } catch (err: any) { + toolCtx.error('failed to check workspace against backup', { err, workspaceId }) + process.exitCode = 1 + } finally { + await pipeline?.close() + await workspaceStorage?.close() + } + }) + }) + program .command('validate-workspace ') .description('Validate a (restored) workspace: connect as system, check model, data counts and blob download') diff --git a/server/backup/src/__tests__/checkWorkspace.spec.ts b/server/backup/src/__tests__/checkWorkspace.spec.ts new file mode 100644 index 0000000000..be98044b19 --- /dev/null +++ b/server/backup/src/__tests__/checkWorkspace.spec.ts @@ -0,0 +1,83 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { compareDomainDigest, findMissingBlobs } from '../utils' + +describe('compareDomainDigest', () => { + it('reports nothing when workspace fully matches backup', () => { + const backup = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + const workspace = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] }) + }) + + it('reports documents present in backup but absent from workspace as missing', () => { + const backup = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + const workspace = new Map([['doc1', 'hash1']]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: ['doc2'], modified: [] }) + }) + + it('reports documents with a different hash as modified, not missing', () => { + const backup = new Map([['doc1', 'hash1']]) + const workspace = new Map([['doc1', 'hash1-changed']]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: ['doc1'] }) + }) + + it('ignores documents present in workspace but not in backup', () => { + const backup = new Map([['doc1', 'hash1']]) + const workspace = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] }) + }) + + it('treats quoted and unquoted equal hashes as the same (matches restore hash trimming)', () => { + const backup = new Map([['doc1', '"hash1"']]) + const workspace = new Map([['doc1', 'hash1']]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] }) + }) + + it('returns an empty result for an empty backup digest', () => { + const workspace = new Map([['doc1', 'hash1']]) + expect(compareDomainDigest(new Map(), workspace)).toEqual({ missing: [], modified: [] }) + }) +}) + +describe('findMissingBlobs', () => { + it('returns nothing when every backup blob exists in storage', () => { + expect(findMissingBlobs(['blob1', 'blob2'], new Set(['blob1', 'blob2', 'blob3']))).toEqual([]) + }) + + it('reports backup blobs absent from storage', () => { + expect(findMissingBlobs(['blob1', 'blob2'], new Set(['blob1']))).toEqual(['blob2']) + }) + + it('reports all backup blobs when storage is empty', () => { + expect(findMissingBlobs(['blob1', 'blob2'], new Set())).toEqual(['blob1', 'blob2']) + }) + + it('returns nothing for an empty list of backup blobs', () => { + expect(findMissingBlobs([], new Set(['blob1']))).toEqual([]) + }) +}) diff --git a/server/backup/src/check.ts b/server/backup/src/check.ts new file mode 100644 index 0000000000..9bb7edccb9 --- /dev/null +++ b/server/backup/src/check.ts @@ -0,0 +1,254 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + Doc, + Domain, + DOMAIN_BLOB, + MeasureContext, + Ref, + type Blob, + type LowLevelStorage, + type WorkspaceIds +} from '@hcengineering/core' +import { BackupClientOps, createDummyStorageAdapter, type Pipeline } from '@hcengineering/server-core' +import { gunzipSync } from 'zlib' +import { BackupStorage } from './storage' +import type { BackupDocId, BackupInfo, BackupSnapshot } from './types' +import { compareDomainDigest, findMissingBlobs, isAccountDomain, loadDigest } from './utils' +export * from './storage' + +/** + * @public + */ +export interface DomainCheckResult { + domain: Domain + backupCount: number + workspaceCount: number + missing: BackupDocId[] + modified: BackupDocId[] +} + +/** + * @public + */ +export interface BlobCheckResult { + total: number + missing: Ref[] + ok: boolean +} + +/** + * @public + */ +export interface WorkspaceCheckResult { + date: number + domains: DomainCheckResult[] + blobs: BlobCheckResult + ok: boolean +} + +async function resolveSnapshots ( + storage: BackupStorage, + date: number +): Promise<{ backupInfo: BackupInfo, snapshots: BackupSnapshot[], date: number }> { + const infoFile = 'backup.json.gz' + if (!(await storage.exists(infoFile))) { + throw new Error(`${infoFile} should present to check`) + } + const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + + let snapshots = backupInfo.snapshots + if (date !== -1) { + const bk = backupInfo.snapshots.findIndex((it) => it.date === date) + if (bk === -1) { + throw new Error(`${infoFile} has no snapshot at ${date}`) + } + snapshots = backupInfo.snapshots.slice(0, bk + 1) + } else { + date = snapshots[snapshots.length - 1]?.date ?? -1 + } + return { backupInfo, snapshots, date } +} + +/** + * Checks whether all documents recorded in a backup are present, and unchanged, in the given + * workspace's document domains, and whether every backed-up blob's content exists in blob + * storage (see {@link checkWorkspaceBlobs}). + * + * This is read-only: nothing is uploaded, removed, or otherwise modified in either the workspace + * or the backup. It is meant as a diagnostic to run before trusting a backup (or after a restore) + * — to find out if the workspace is missing data the backup has, without acting on it. + * + * Account domains (person/socialId) are skipped, since they live in the account database rather + * than in the workspace's own domains and can't be checked against `pipeline.context.lowLevelStorage`. + * + * @param date optional snapshot date to check against, defaults to the latest snapshot (-1). + * @public + */ +export async function checkWorkspaceBackup ( + ctx: MeasureContext, + pipeline: Pipeline, + wsIds: WorkspaceIds, + storage: BackupStorage, + date: number = -1 +): Promise { + const resolved = await resolveSnapshots(storage, date) + const snapshots = resolved.snapshots + date = resolved.date + + ctx.info('checking workspace against backup', { workspace: wsIds.uuid, date }) + + const domains = new Set() + for (const s of snapshots) { + Object.keys(s.domains).forEach((it) => domains.add(it as Domain)) + } + + const connection = pipeline.context.lowLevelStorage as LowLevelStorage + const ops = new BackupClientOps(connection) + + const results: DomainCheckResult[] = [] + + for (const domain of domains) { + if (isAccountDomain(domain)) { + continue + } + + ctx.info('checking domain', { domain }) + const backupDigest = (await loadDigest(ctx, storage, snapshots, domain, date)) as Map, string> + + const workspaceDigest = new Map, string>() + let idx: number | undefined + try { + while (true) { + const it = await ops.loadChunk(ctx, domain, idx) + idx = it.idx + for (const { id, hash } of it.docs) { + workspaceDigest.set(id as Ref, hash) + } + if (it.finished) { + break + } + } + } finally { + if (idx !== undefined) { + await ops.closeChunk(ctx, idx) + } + } + + const { missing, modified } = compareDomainDigest(backupDigest, workspaceDigest) + + const result: DomainCheckResult = { + domain, + backupCount: backupDigest.size, + workspaceCount: workspaceDigest.size, + missing, + modified + } + results.push(result) + + if (missing.length > 0 || modified.length > 0) { + ctx.warn('backup data not fully present in workspace', { + domain, + backupCount: result.backupCount, + workspaceCount: result.workspaceCount, + missing: missing.length, + modified: modified.length, + sampleMissing: missing.slice(0, 10), + sampleModified: modified.slice(0, 10) + }) + } else { + ctx.info('domain ok', { domain, count: result.backupCount }) + } + } + + const blobs = await checkWorkspaceBlobs(ctx, pipeline, wsIds, storage, date, snapshots) + + const ok = results.every((it) => it.missing.length === 0 && it.modified.length === 0) && blobs.ok + + ctx.info('check complete', { + workspace: wsIds.uuid, + ok, + domains: results.length, + missing: results.reduce((sum, it) => sum + it.missing.length, 0), + modified: results.reduce((sum, it) => sum + it.modified.length, 0), + missingBlobs: blobs.missing.length + }) + + return { date, domains: results, blobs, ok } +} + +/** + * Checks whether every blob recorded in a backup actually has its content present in the + * workspace's blob storage (S3/minio/datalake), as opposed to just a metadata record in + * `DOMAIN_BLOB`. + * + * Read-only: only lists and stats existing blobs, never uploads or removes anything. + * + * @param date optional snapshot date to check against, defaults to the latest snapshot (-1). + * @param snapshots pre-resolved snapshots, to avoid re-reading `backup.json.gz` when called from + * {@link checkWorkspaceBackup}. If omitted, it's resolved from `storage`/`date`. + * @public + */ +export async function checkWorkspaceBlobs ( + ctx: MeasureContext, + pipeline: Pipeline, + wsIds: WorkspaceIds, + storage: BackupStorage, + date: number = -1, + snapshots?: BackupSnapshot[] +): Promise { + if (snapshots === undefined) { + const resolved = await resolveSnapshots(storage, date) + snapshots = resolved.snapshots + date = resolved.date + } + + ctx.info('checking blobs against backup', { workspace: wsIds.uuid, date }) + + const backupDigest = await loadDigest(ctx, storage, snapshots, DOMAIN_BLOB, date) + + const storageAdapter = pipeline.context.storageAdapter ?? createDummyStorageAdapter() + const existingBlobIds = new Set() + const iterator = await storageAdapter.listStream(ctx, wsIds) + try { + while (true) { + const batch = await iterator.next() + if (batch.length === 0) { + break + } + for (const b of batch) { + existingBlobIds.add(b._id) + } + } + } finally { + await iterator.close() + } + + const missing = findMissingBlobs(backupDigest.keys(), existingBlobIds) as Ref[] + const ok = missing.length === 0 + + if (ok) { + ctx.info('blobs ok', { total: backupDigest.size }) + } else { + ctx.warn('backup blobs missing from storage', { + total: backupDigest.size, + missing: missing.length, + sampleMissing: missing.slice(0, 10) + }) + } + + return { total: backupDigest.size, missing, ok } +} diff --git a/server/backup/src/index.ts b/server/backup/src/index.ts index b2cd2bf630..43ea32b91a 100644 --- a/server/backup/src/index.ts +++ b/server/backup/src/index.ts @@ -13,6 +13,7 @@ // limitations under the License. // export * from './backup' +export * from './check' export * from './restore' export * from './service' export * from './types' diff --git a/server/backup/src/utils.ts b/server/backup/src/utils.ts index 5316d82521..149c5136cc 100644 --- a/server/backup/src/utils.ts +++ b/server/backup/src/utils.ts @@ -941,6 +941,51 @@ export function doTrimHash (s: string | undefined): string | undefined { return s } +/** + * Compares a per-domain digest reconstructed from a backup with a digest read from a live + * workspace and reports the difference from the backup's point of view. + * + * - `missing` — documents present in the backup but absent from the workspace. + * - `modified` — documents present in both, but with a different content hash (the workspace + * version diverged from the backed-up one). + * + * Documents present in the workspace but not in the backup are intentionally not reported here: + * this check only answers "is everything from the backup present in the workspace", not the + * reverse. + * @public + */ +export function compareDomainDigest ( + backupDigest: Map, + workspaceDigest: Map +): { missing: BackupDocId[], modified: BackupDocId[] } { + const missing: BackupDocId[] = [] + const modified: BackupDocId[] = [] + for (const [id, hash] of backupDigest) { + const workspaceHash = workspaceDigest.get(id) + if (workspaceHash === undefined) { + missing.push(id) + } else if (doTrimHash(workspaceHash) !== doTrimHash(hash)) { + modified.push(id) + } + } + return { missing, modified } +} + +/** + * Finds blob ids that are recorded in a backup but do not exist in the workspace's blob storage + * (e.g. S3/minio/datalake), as opposed to just the blob metadata record in a domain. + * @public + */ +export function findMissingBlobs (backupBlobIds: Iterable, existingBlobIds: Set): BackupDocId[] { + const missing: BackupDocId[] = [] + for (const id of backupBlobIds) { + if (!existingBlobIds.has(id as string)) { + missing.push(id) + } + } + return missing +} + export async function loadDigest ( ctx: MeasureContext, storage: BackupStorage,