mirror of
https://github.com/hcengineering/platform.git
synced 2026-08-17 18:05:42 +02:00
Add script to verify all backup data is present in a workspace (#10999)
* Add script to verify all backup data is present in a workspace Signed-off-by: Artyom Savchenko <armisav@gmail.com> * Verify blobs exist Signed-off-by: Artyom Savchenko <armisav@gmail.com> * Fix formatting Signed-off-by: Artyom Savchenko <armisav@gmail.com> --------- Signed-off-by: Artyom Savchenko <armisav@gmail.com>
This commit is contained in:
Artyom Savchenko
@@ -46,6 +46,7 @@ import {
|
||||
backupDownload,
|
||||
backupFind,
|
||||
checkBackupIntegrity,
|
||||
checkWorkspaceBackup,
|
||||
compactBackup,
|
||||
createFileBackupStorage,
|
||||
createStorageBackupStorage,
|
||||
@@ -639,6 +640,73 @@ export function devTool (
|
||||
await storageAdapter.close()
|
||||
})
|
||||
|
||||
program
|
||||
.command('backup-check-workspace <dirName> <workspace> [date]')
|
||||
.description(
|
||||
'Check whether all data from a backup is present in the workspace. Read-only: no data is added, removed or changed.'
|
||||
)
|
||||
.action(async (dirName: string, workspaceId: string, date: string | undefined) => {
|
||||
await withAccountDatabase(async (db) => {
|
||||
const { txes, dbUrl } = prepareTools()
|
||||
const ws = await getWorkspace(db, workspaceId)
|
||||
if (ws === null) {
|
||||
throw new Error(`workspace ${workspaceId} not found`)
|
||||
}
|
||||
|
||||
const wsIds = { uuid: ws.uuid, dataId: ws.dataId, url: ws.url ?? '' }
|
||||
const storage = await createFileBackupStorage(dirName)
|
||||
const workspaceStorage: StorageAdapter = buildStorageFromConfig(storageConfigFromEnv())
|
||||
|
||||
let pipeline: Pipeline | undefined
|
||||
try {
|
||||
pipeline = await createBackupPipeline(toolCtx, dbUrl, txes, {
|
||||
externalStorage: workspaceStorage,
|
||||
usePassedCtx: true
|
||||
})(
|
||||
toolCtx,
|
||||
{
|
||||
uuid: ws.uuid,
|
||||
url: ws.url ?? '',
|
||||
dataId: ws.dataId
|
||||
},
|
||||
createEmptyBroadcastOps(),
|
||||
null
|
||||
)
|
||||
if (pipeline === undefined) {
|
||||
toolCtx.error('failed to check, pipeline is undefined', { workspaceId })
|
||||
process.exitCode = 1
|
||||
return
|
||||
}
|
||||
|
||||
const result = await checkWorkspaceBackup(toolCtx, pipeline, wsIds, storage, parseInt(date ?? '-1'))
|
||||
|
||||
console.log('')
|
||||
for (const d of result.domains) {
|
||||
const ok = d.missing.length === 0 && d.modified.length === 0
|
||||
console.log(
|
||||
`${ok ? 'OK ' : 'FAIL'} ${d.domain}: backup=${d.backupCount} workspace=${d.workspaceCount} missing=${d.missing.length} modified=${d.modified.length}`
|
||||
)
|
||||
}
|
||||
console.log(
|
||||
`${result.blobs.ok ? 'OK ' : 'FAIL'} blobs (storage): total=${result.blobs.total} missing=${result.blobs.missing.length}`
|
||||
)
|
||||
console.log('')
|
||||
if (result.ok) {
|
||||
console.log('OK: workspace contains all data from backup')
|
||||
} else {
|
||||
console.log('FAILED: workspace is missing data present in the backup')
|
||||
process.exitCode = 1
|
||||
}
|
||||
} catch (err: any) {
|
||||
toolCtx.error('failed to check workspace against backup', { err, workspaceId })
|
||||
process.exitCode = 1
|
||||
} finally {
|
||||
await pipeline?.close()
|
||||
await workspaceStorage?.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
program
|
||||
.command('validate-workspace <workspace>')
|
||||
.description('Validate a (restored) workspace: connect as system, check model, data counts and blob download')
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
//
|
||||
// Copyright © 2026 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { compareDomainDigest, findMissingBlobs } from '../utils'
|
||||
|
||||
describe('compareDomainDigest', () => {
|
||||
it('reports nothing when workspace fully matches backup', () => {
|
||||
const backup = new Map([
|
||||
['doc1', 'hash1'],
|
||||
['doc2', 'hash2']
|
||||
])
|
||||
const workspace = new Map([
|
||||
['doc1', 'hash1'],
|
||||
['doc2', 'hash2']
|
||||
])
|
||||
expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] })
|
||||
})
|
||||
|
||||
it('reports documents present in backup but absent from workspace as missing', () => {
|
||||
const backup = new Map([
|
||||
['doc1', 'hash1'],
|
||||
['doc2', 'hash2']
|
||||
])
|
||||
const workspace = new Map([['doc1', 'hash1']])
|
||||
expect(compareDomainDigest(backup, workspace)).toEqual({ missing: ['doc2'], modified: [] })
|
||||
})
|
||||
|
||||
it('reports documents with a different hash as modified, not missing', () => {
|
||||
const backup = new Map([['doc1', 'hash1']])
|
||||
const workspace = new Map([['doc1', 'hash1-changed']])
|
||||
expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: ['doc1'] })
|
||||
})
|
||||
|
||||
it('ignores documents present in workspace but not in backup', () => {
|
||||
const backup = new Map([['doc1', 'hash1']])
|
||||
const workspace = new Map([
|
||||
['doc1', 'hash1'],
|
||||
['doc2', 'hash2']
|
||||
])
|
||||
expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] })
|
||||
})
|
||||
|
||||
it('treats quoted and unquoted equal hashes as the same (matches restore hash trimming)', () => {
|
||||
const backup = new Map([['doc1', '"hash1"']])
|
||||
const workspace = new Map([['doc1', 'hash1']])
|
||||
expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] })
|
||||
})
|
||||
|
||||
it('returns an empty result for an empty backup digest', () => {
|
||||
const workspace = new Map([['doc1', 'hash1']])
|
||||
expect(compareDomainDigest(new Map(), workspace)).toEqual({ missing: [], modified: [] })
|
||||
})
|
||||
})
|
||||
|
||||
describe('findMissingBlobs', () => {
|
||||
it('returns nothing when every backup blob exists in storage', () => {
|
||||
expect(findMissingBlobs(['blob1', 'blob2'], new Set(['blob1', 'blob2', 'blob3']))).toEqual([])
|
||||
})
|
||||
|
||||
it('reports backup blobs absent from storage', () => {
|
||||
expect(findMissingBlobs(['blob1', 'blob2'], new Set(['blob1']))).toEqual(['blob2'])
|
||||
})
|
||||
|
||||
it('reports all backup blobs when storage is empty', () => {
|
||||
expect(findMissingBlobs(['blob1', 'blob2'], new Set())).toEqual(['blob1', 'blob2'])
|
||||
})
|
||||
|
||||
it('returns nothing for an empty list of backup blobs', () => {
|
||||
expect(findMissingBlobs([], new Set(['blob1']))).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,254 @@
|
||||
//
|
||||
// Copyright © 2026 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import {
|
||||
Doc,
|
||||
Domain,
|
||||
DOMAIN_BLOB,
|
||||
MeasureContext,
|
||||
Ref,
|
||||
type Blob,
|
||||
type LowLevelStorage,
|
||||
type WorkspaceIds
|
||||
} from '@hcengineering/core'
|
||||
import { BackupClientOps, createDummyStorageAdapter, type Pipeline } from '@hcengineering/server-core'
|
||||
import { gunzipSync } from 'zlib'
|
||||
import { BackupStorage } from './storage'
|
||||
import type { BackupDocId, BackupInfo, BackupSnapshot } from './types'
|
||||
import { compareDomainDigest, findMissingBlobs, isAccountDomain, loadDigest } from './utils'
|
||||
export * from './storage'
|
||||
|
||||
/**
|
||||
* @public
|
||||
*/
|
||||
export interface DomainCheckResult {
|
||||
domain: Domain
|
||||
backupCount: number
|
||||
workspaceCount: number
|
||||
missing: BackupDocId[]
|
||||
modified: BackupDocId[]
|
||||
}
|
||||
|
||||
/**
|
||||
* @public
|
||||
*/
|
||||
export interface BlobCheckResult {
|
||||
total: number
|
||||
missing: Ref<Blob>[]
|
||||
ok: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* @public
|
||||
*/
|
||||
export interface WorkspaceCheckResult {
|
||||
date: number
|
||||
domains: DomainCheckResult[]
|
||||
blobs: BlobCheckResult
|
||||
ok: boolean
|
||||
}
|
||||
|
||||
async function resolveSnapshots (
|
||||
storage: BackupStorage,
|
||||
date: number
|
||||
): Promise<{ backupInfo: BackupInfo, snapshots: BackupSnapshot[], date: number }> {
|
||||
const infoFile = 'backup.json.gz'
|
||||
if (!(await storage.exists(infoFile))) {
|
||||
throw new Error(`${infoFile} should present to check`)
|
||||
}
|
||||
const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString())
|
||||
|
||||
let snapshots = backupInfo.snapshots
|
||||
if (date !== -1) {
|
||||
const bk = backupInfo.snapshots.findIndex((it) => it.date === date)
|
||||
if (bk === -1) {
|
||||
throw new Error(`${infoFile} has no snapshot at ${date}`)
|
||||
}
|
||||
snapshots = backupInfo.snapshots.slice(0, bk + 1)
|
||||
} else {
|
||||
date = snapshots[snapshots.length - 1]?.date ?? -1
|
||||
}
|
||||
return { backupInfo, snapshots, date }
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether all documents recorded in a backup are present, and unchanged, in the given
|
||||
* workspace's document domains, and whether every backed-up blob's content exists in blob
|
||||
* storage (see {@link checkWorkspaceBlobs}).
|
||||
*
|
||||
* This is read-only: nothing is uploaded, removed, or otherwise modified in either the workspace
|
||||
* or the backup. It is meant as a diagnostic to run before trusting a backup (or after a restore)
|
||||
* — to find out if the workspace is missing data the backup has, without acting on it.
|
||||
*
|
||||
* Account domains (person/socialId) are skipped, since they live in the account database rather
|
||||
* than in the workspace's own domains and can't be checked against `pipeline.context.lowLevelStorage`.
|
||||
*
|
||||
* @param date optional snapshot date to check against, defaults to the latest snapshot (-1).
|
||||
* @public
|
||||
*/
|
||||
export async function checkWorkspaceBackup (
|
||||
ctx: MeasureContext,
|
||||
pipeline: Pipeline,
|
||||
wsIds: WorkspaceIds,
|
||||
storage: BackupStorage,
|
||||
date: number = -1
|
||||
): Promise<WorkspaceCheckResult> {
|
||||
const resolved = await resolveSnapshots(storage, date)
|
||||
const snapshots = resolved.snapshots
|
||||
date = resolved.date
|
||||
|
||||
ctx.info('checking workspace against backup', { workspace: wsIds.uuid, date })
|
||||
|
||||
const domains = new Set<Domain>()
|
||||
for (const s of snapshots) {
|
||||
Object.keys(s.domains).forEach((it) => domains.add(it as Domain))
|
||||
}
|
||||
|
||||
const connection = pipeline.context.lowLevelStorage as LowLevelStorage
|
||||
const ops = new BackupClientOps(connection)
|
||||
|
||||
const results: DomainCheckResult[] = []
|
||||
|
||||
for (const domain of domains) {
|
||||
if (isAccountDomain(domain)) {
|
||||
continue
|
||||
}
|
||||
|
||||
ctx.info('checking domain', { domain })
|
||||
const backupDigest = (await loadDigest(ctx, storage, snapshots, domain, date)) as Map<Ref<Doc>, string>
|
||||
|
||||
const workspaceDigest = new Map<Ref<Doc>, string>()
|
||||
let idx: number | undefined
|
||||
try {
|
||||
while (true) {
|
||||
const it = await ops.loadChunk(ctx, domain, idx)
|
||||
idx = it.idx
|
||||
for (const { id, hash } of it.docs) {
|
||||
workspaceDigest.set(id as Ref<Doc>, hash)
|
||||
}
|
||||
if (it.finished) {
|
||||
break
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (idx !== undefined) {
|
||||
await ops.closeChunk(ctx, idx)
|
||||
}
|
||||
}
|
||||
|
||||
const { missing, modified } = compareDomainDigest(backupDigest, workspaceDigest)
|
||||
|
||||
const result: DomainCheckResult = {
|
||||
domain,
|
||||
backupCount: backupDigest.size,
|
||||
workspaceCount: workspaceDigest.size,
|
||||
missing,
|
||||
modified
|
||||
}
|
||||
results.push(result)
|
||||
|
||||
if (missing.length > 0 || modified.length > 0) {
|
||||
ctx.warn('backup data not fully present in workspace', {
|
||||
domain,
|
||||
backupCount: result.backupCount,
|
||||
workspaceCount: result.workspaceCount,
|
||||
missing: missing.length,
|
||||
modified: modified.length,
|
||||
sampleMissing: missing.slice(0, 10),
|
||||
sampleModified: modified.slice(0, 10)
|
||||
})
|
||||
} else {
|
||||
ctx.info('domain ok', { domain, count: result.backupCount })
|
||||
}
|
||||
}
|
||||
|
||||
const blobs = await checkWorkspaceBlobs(ctx, pipeline, wsIds, storage, date, snapshots)
|
||||
|
||||
const ok = results.every((it) => it.missing.length === 0 && it.modified.length === 0) && blobs.ok
|
||||
|
||||
ctx.info('check complete', {
|
||||
workspace: wsIds.uuid,
|
||||
ok,
|
||||
domains: results.length,
|
||||
missing: results.reduce((sum, it) => sum + it.missing.length, 0),
|
||||
modified: results.reduce((sum, it) => sum + it.modified.length, 0),
|
||||
missingBlobs: blobs.missing.length
|
||||
})
|
||||
|
||||
return { date, domains: results, blobs, ok }
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether every blob recorded in a backup actually has its content present in the
|
||||
* workspace's blob storage (S3/minio/datalake), as opposed to just a metadata record in
|
||||
* `DOMAIN_BLOB`.
|
||||
*
|
||||
* Read-only: only lists and stats existing blobs, never uploads or removes anything.
|
||||
*
|
||||
* @param date optional snapshot date to check against, defaults to the latest snapshot (-1).
|
||||
* @param snapshots pre-resolved snapshots, to avoid re-reading `backup.json.gz` when called from
|
||||
* {@link checkWorkspaceBackup}. If omitted, it's resolved from `storage`/`date`.
|
||||
* @public
|
||||
*/
|
||||
export async function checkWorkspaceBlobs (
|
||||
ctx: MeasureContext,
|
||||
pipeline: Pipeline,
|
||||
wsIds: WorkspaceIds,
|
||||
storage: BackupStorage,
|
||||
date: number = -1,
|
||||
snapshots?: BackupSnapshot[]
|
||||
): Promise<BlobCheckResult> {
|
||||
if (snapshots === undefined) {
|
||||
const resolved = await resolveSnapshots(storage, date)
|
||||
snapshots = resolved.snapshots
|
||||
date = resolved.date
|
||||
}
|
||||
|
||||
ctx.info('checking blobs against backup', { workspace: wsIds.uuid, date })
|
||||
|
||||
const backupDigest = await loadDigest(ctx, storage, snapshots, DOMAIN_BLOB, date)
|
||||
|
||||
const storageAdapter = pipeline.context.storageAdapter ?? createDummyStorageAdapter()
|
||||
const existingBlobIds = new Set<string>()
|
||||
const iterator = await storageAdapter.listStream(ctx, wsIds)
|
||||
try {
|
||||
while (true) {
|
||||
const batch = await iterator.next()
|
||||
if (batch.length === 0) {
|
||||
break
|
||||
}
|
||||
for (const b of batch) {
|
||||
existingBlobIds.add(b._id)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await iterator.close()
|
||||
}
|
||||
|
||||
const missing = findMissingBlobs(backupDigest.keys(), existingBlobIds) as Ref<Blob>[]
|
||||
const ok = missing.length === 0
|
||||
|
||||
if (ok) {
|
||||
ctx.info('blobs ok', { total: backupDigest.size })
|
||||
} else {
|
||||
ctx.warn('backup blobs missing from storage', {
|
||||
total: backupDigest.size,
|
||||
missing: missing.length,
|
||||
sampleMissing: missing.slice(0, 10)
|
||||
})
|
||||
}
|
||||
|
||||
return { total: backupDigest.size, missing, ok }
|
||||
}
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
//
|
||||
export * from './backup'
|
||||
export * from './check'
|
||||
export * from './restore'
|
||||
export * from './service'
|
||||
export * from './types'
|
||||
|
||||
@@ -941,6 +941,51 @@ export function doTrimHash (s: string | undefined): string | undefined {
|
||||
return s
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares a per-domain digest reconstructed from a backup with a digest read from a live
|
||||
* workspace and reports the difference from the backup's point of view.
|
||||
*
|
||||
* - `missing` — documents present in the backup but absent from the workspace.
|
||||
* - `modified` — documents present in both, but with a different content hash (the workspace
|
||||
* version diverged from the backed-up one).
|
||||
*
|
||||
* Documents present in the workspace but not in the backup are intentionally not reported here:
|
||||
* this check only answers "is everything from the backup present in the workspace", not the
|
||||
* reverse.
|
||||
* @public
|
||||
*/
|
||||
export function compareDomainDigest (
|
||||
backupDigest: Map<BackupDocId, string>,
|
||||
workspaceDigest: Map<BackupDocId, string>
|
||||
): { missing: BackupDocId[], modified: BackupDocId[] } {
|
||||
const missing: BackupDocId[] = []
|
||||
const modified: BackupDocId[] = []
|
||||
for (const [id, hash] of backupDigest) {
|
||||
const workspaceHash = workspaceDigest.get(id)
|
||||
if (workspaceHash === undefined) {
|
||||
missing.push(id)
|
||||
} else if (doTrimHash(workspaceHash) !== doTrimHash(hash)) {
|
||||
modified.push(id)
|
||||
}
|
||||
}
|
||||
return { missing, modified }
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds blob ids that are recorded in a backup but do not exist in the workspace's blob storage
|
||||
* (e.g. S3/minio/datalake), as opposed to just the blob metadata record in a domain.
|
||||
* @public
|
||||
*/
|
||||
export function findMissingBlobs (backupBlobIds: Iterable<BackupDocId>, existingBlobIds: Set<string>): BackupDocId[] {
|
||||
const missing: BackupDocId[] = []
|
||||
for (const id of backupBlobIds) {
|
||||
if (!existingBlobIds.has(id as string)) {
|
||||
missing.push(id)
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
export async function loadDigest (
|
||||
ctx: MeasureContext,
|
||||
storage: BackupStorage,
|
||||
|
||||
Reference in New Issue
Block a user