diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 1d8720c9c0..b347af1b33 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -46,6 +46,7 @@ import { backupDownload, backupFind, checkBackupIntegrity, + checkWorkspaceBackup, compactBackup, createFileBackupStorage, createStorageBackupStorage, @@ -114,7 +115,6 @@ import { type StorageAdapter } from '@hcengineering/server-core' import { getAccountDBUrl, getKvsUrl, getMongoDBUrl } from './__start' -// import { fillGithubUsers, fixAccountEmails, renameAccount } from './account' import { changeConfiguration } from './configuration' import { performCalendarAccountMigrations } from './calendar' @@ -264,77 +264,6 @@ export function devTool ( }) }) - // program - // .command('reset-account ') - // .description('create user and corresponding account in master database') - // .option('-p, --password ', 'new user password') - // .action(async (email: string, cmd) => { - // await withAccountDatabase(async (db) => { - // console.log(`update account ${email} ${cmd.first as string} ${cmd.last as string}...`) - // await replacePassword(db, email, cmd.password) - // }) - // }) - - // program - // .command('reset-email ') - // .description('rename account in accounts and all workspaces') - // .action(async (email: string, newEmail: string, cmd) => { - // await withAccountDatabase(async (db) => { - // console.log(`update account ${email} to ${newEmail}`) - // await renameAccount(toolCtx, db, accountsUrl, email, newEmail) - // }) - // }) - - // program - // .command('fix-email ') - // .description('fix email in all workspaces to be proper one') - // .action(async (email: string, newEmail: string, cmd) => { - // await withAccountDatabase(async (db) => { - // console.log(`update account ${email} to ${newEmail}`) - // await fixAccountEmails(toolCtx, db, accountsUrl, email, newEmail) - // }) - // }) - - // program - // .command('compact-db-mongo') - // .description('compact all db collections') - // .option('-w, --workspace ', 'A selected "workspace" only', '') - // .action(async (cmd: { workspace: string }) => { - // const dbUrl = getMongoDBUrl() - // await withAccountDatabase(async (db) => { - // console.log('compacting db ...') - // let gtotal: number = 0 - // const client = getMongoClient(dbUrl) - // const _client = await client.getClient() - // try { - // const workspaces = await listWorkspacesPure(db) - // for (const workspace of workspaces) { - // if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - // continue - // } - // let total: number = 0 - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - // const collections = wsDb.listCollections() - // while (true) { - // const collInfo = await collections.next() - // if (collInfo === null) { - // break - // } - // const result = await wsDb.command({ compact: collInfo.name }) - // total += result.bytesFreed - // } - // gtotal += total - // console.log('total feed for db', workspace.workspaceName, Math.round(total / (1024 * 1024))) - // } - // console.log('global total feed', Math.round(gtotal / (1024 * 1024))) - // } catch (err: any) { - // console.error(err) - // } finally { - // client.close() - // } - // }) - // }) - program .command('assign-workspace ') .description('assign workspace') @@ -358,16 +287,6 @@ export function devTool ( }) }) - // program - // .command('show-user ') - // .description('show user') - // .action(async (email) => { - // await withAccountDatabase(async (db) => { - // const info = await getAccount(db, email) - // console.log(info) - // }) - // }) - program .command('create-workspace ') .description('create workspace') @@ -458,16 +377,6 @@ export function devTool ( }) }) - // program - // .command('set-user-admin ') - // .description('set user role') - // .action(async (email: string, role: string) => { - // console.log(`set user ${email} admin...`) - // await withAccountDatabase(async (db) => { - // await setAccountAdmin(db, email, role === 'true') - // }) - // }) - async function doUpgrade ( toolCtx: MeasureMetricsContext, workspace: WorkspaceUuid, @@ -531,405 +440,6 @@ export function devTool ( await doUpgrade(toolCtx, workspace, cmd.force, cmd.indexes) }) - // program - // .command('upgrade') - // .description('upgrade') - // .option('-l|--logs ', 'Default logs folder', './logs') - // .option('-i|--ignore [ignore]', 'Ignore workspaces', '') - // .option('-r|--region [region]', 'Region of workspaces', '') - // .option( - // '-c|--console', - // 'Display all information into console(default will create logs folder with {workspace}.log files', - // false - // ) - // .option('-f|--force [force]', 'Force update', false) - // .action(async (cmd: { logs: string, force: boolean, console: boolean, ignore: string, region: string }) => { - // const { version, txes, migrateOperations } = prepareTools() - // await withAccountDatabase(async (db) => { - // const workspaces = (await listWorkspacesRaw(db, cmd.region)).filter((ws) => !cmd.ignore.includes(ws.workspace)) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - // const measureCtx = new MeasureMetricsContext('upgrade', {}) - - // for (const ws of workspaces) { - // console.warn('UPGRADING', ws.workspaceName) - // const logger = cmd.console - // ? consoleModelLogger - // : new FileModelLogger(path.join(cmd.logs, `${ws.workspace}.log`)) - - // try { - // await upgradeWorkspace( - // measureCtx, - // version, - // txes, - // migrateOperations, - // ws, - // logger, - // async () => {}, - // cmd.force, - // false, - // true - // ) - - // await updateWorkspace(db, ws, { - // mode: 'active', - // progress: 100, - // version, - // attempts: 0 - // }) - // } catch (err: any) { - // toolCtx.error('failed to upgrade', { err, workspace: ws.workspace, workspaceName: ws.workspaceName }) - // continue - // } - // } - // console.log('upgrade done') - // }) - // }) - - // program - // .command('list-unused-workspaces') - // .description('list unused workspaces. Without it will only mark them disabled') - // .option('-t|--timeout [timeout]', 'Timeout in days', '60') - // .action(async (cmd: { disable: boolean, exclude: string, timeout: string }) => { - // await withAccountDatabase(async (db) => { - // const workspaces = new Map((await listWorkspacesPure(db)).map((p) => [p._id.toString(), p])) - - // const accounts = await listAccounts(db) - - // const _timeout = parseInt(cmd.timeout) ?? 7 - - // let used = 0 - // let unused = 0 - - // for (const a of accounts) { - // const authored = a.workspaces - // .map((it) => workspaces.get(it.toString())) - // .filter((it) => it !== undefined && it.createdBy?.trim() === a.email?.trim()) as Workspace[] - // authored.sort((a, b) => b.lastVisit - a.lastVisit) - // if (authored.length > 0) { - // const lastLoginDays = Math.floor((Date.now() - a.lastVisit) / 1000 / 3600 / 24) - // toolCtx.info(a.email, { - // workspaces: a.workspaces.length, - // firstName: a.first, - // lastName: a.last, - // lastLoginDays - // }) - // for (const ws of authored) { - // const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - - // if (lastVisitDays > _timeout) { - // unused++ - // toolCtx.warn(' --- unused', { - // url: ws.workspaceUrl, - // id: ws.workspace, - // lastVisitDays - // }) - // } else { - // used++ - // toolCtx.warn(' +++ used', { - // url: ws.workspaceUrl, - // id: ws.workspace, - // createdBy: ws.createdBy, - // lastVisitDays - // }) - // } - // } - // } - // } - - // console.log('Used: ', used, 'Unused: ', unused) - // }) - // }) - - // program - // .command('archive-workspaces') - // .description('Archive and delete non visited workspaces...') - // .option('-r|--remove [remove]', 'Pass to remove all data', false) - // .option('--region [region]', 'Pass to remove all data', '') - // .option('-t|--timeout [timeout]', 'Timeout in days', '60') - // .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '') - // .action( - // async (cmd: { - // disable: boolean - // exclude: string - // timeout: string - // remove: boolean - // workspace: string - // region: string - // }) => { - // const { dbUrl, txes } = prepareTools() - // await withAccountDatabase(async (db) => { - // const workspaces = (await listWorkspacesPure(db)) - // .sort((a, b) => a.lastVisit - b.lastVisit) - // .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) - - // const _timeout = parseInt(cmd.timeout) ?? 7 - - // let unused = 0 - // for (const ws of workspaces) { - // const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - - // if (lastVisitDays > _timeout && isActiveMode(ws.mode)) { - // unused++ - // toolCtx.warn('--- unused', { - // url: ws.workspaceUrl, - // id: ws.workspace, - // lastVisitDays, - // mode: ws.mode - // }) - // try { - // await backupWorkspace( - // toolCtx, - // ws, - // (dbUrl, storageAdapter) => { - // const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { - // externalStorage: storageAdapter, - // usePassedCtx: true - // }) - // return factory - // }, - // (ctx, dbUrls, workspace, branding, externalStorage) => { - // return getConfig(ctx, dbUrls, ctx, { - // externalStorage, - // disableTriggers: true - // }) - // }, - // cmd.region, - // 5000, // 5 gigabytes per blob - // async (storage, workspaceStorage) => { - // if (cmd.remove) { - // await updateArchiveInfo(toolCtx, db, ws.workspace, true) - // const files = await workspaceStorage.listStream(toolCtx, { name: ws.workspace }) - - // while (true) { - // const docs = await files.next() - // if (docs.length === 0) { - // break - // } - // await workspaceStorage.remove( - // toolCtx, - // { name: ws.workspace }, - // docs.map((it) => it._id) - // ) - // } - - // program - // .command('backup-all') - // .description('Backup all workspaces...') - // .option('--region [region]', 'Force backup of selected workspace', '') - // .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '') - // .action(async (cmd: { workspace: string, region: string }) => { - // const { txes } = prepareTools() - // await withAccountDatabase(async (db) => { - // const workspaces = (await listWorkspacesPure(db)) - // .sort((a, b) => a.lastVisit - b.lastVisit) - // .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) - - // let processed = 0 - - // // We need to update workspaces with missing workspaceUrl - // for (const ws of workspaces) { - // try { - // if ( - // await backupWorkspace( - // toolCtx, - // ws, - // (dbUrl, storageAdapter) => { - // const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { - // externalStorage: storageAdapter, - // usePassedCtx: true - // }) - // return factory - // }, - // (ctx, dbUrls, workspace, branding, externalStorage) => { - // return getConfig(ctx, dbUrls, ctx, { - // externalStorage, - // disableTriggers: true - // }) - // }, - // cmd.region, - // 100, - // ) - // ) { - // processed++ - // } - // } catch (err: any) { - // toolCtx.error('Failed to backup workspace', { workspace: ws.workspace }) - // } - // } - // console.log('Processed workspaces', processed) - // }) - // }) - - // program - // .command('drop-workspace ') - // .description('drop workspace') - // .option('--full [full]', 'Force remove all data', false) - // .action(async (workspace, cmd: { full: boolean }) => { - // const { dbUrl } = prepareTools() - - // await withStorage(async (storageAdapter) => { - // await withAccountDatabase(async (db) => { - // const ws = await getWorkspaceById(db, workspace) - // if (ws === null) { - // console.log('no workspace exists') - // return - // } - // if (cmd.full) { - // await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace, storageAdapter) - // } else { - // await dropWorkspace(toolCtx, db, null, workspace) - // } - // }) - // }) - // }) - - // program - // .command('drop-workspace-by-email ') - // .description('drop workspace') - // .option('--full [full]', 'Force remove all data', false) - // .action(async (email, cmd: { full: boolean }) => { - // const { dbUrl } = prepareTools() - // await withStorage(async (storageAdapter) => { - // await withAccountDatabase(async (db) => { - // for (const workspace of await listWorkspacesByAccount(db, email)) { - // if (cmd.full) { - // await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace.workspace, storageAdapter) - // } else { - // await dropWorkspace(toolCtx, db, null, workspace.workspace) - // } - // } - // }) - // }) - // }) - // program - // .command('list-workspace-by-email ') - // .description('drop workspace') - // .option('--full [full]', 'Force remove all data', false) - // .action(async (email, cmd: { full: boolean }) => { - // await withAccountDatabase(async (db) => { - // for (const workspace of await listWorkspacesByAccount(db, email)) { - // console.log(workspace.workspace, workspace.workspaceUrl, workspace.workspaceName) - // } - // }) - // }) - - // program - // .command('drop-workspace-last-visit') - // .description('drop old workspaces') - // .action(async (cmd: any) => { - // const { dbUrl } = prepareTools() - - // await withStorage(async (storageAdapter) => { - // await withAccountDatabase(async (db) => { - // const workspacesJSON = await listWorkspacesPure(db) - // for (const ws of workspacesJSON) { - // const lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - // if (lastVisit > 60) { - // await dropWorkspaceFull(toolCtx, db, dbUrl, null, ws.workspace, storageAdapter) - // } - // } - // }) - // }) - // }) - - // program - // .command('list-workspaces') - // .description('List workspaces') - // .option('-e|--expired [expired]', 'Show only expired', false) - // .action(async (cmd: { expired: boolean }) => { - // const { version } = prepareTools() - // await withAccountDatabase(async (db) => { - // const workspacesJSON = await listWorkspacesPure(db) - // for (const ws of workspacesJSON) { - // let lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - // if (cmd.expired && lastVisit <= 7) { - // continue - // } - // console.log( - // colorConstants.colorBlue + - // '####################################################################################################' + - // colorConstants.reset - // ) - // console.log('id:', colorConstants.colorWhiteCyan + ws.workspace + colorConstants.reset) - // console.log('url:', ws.workspaceUrl, 'name:', ws.workspaceName) - // console.log( - // 'version:', - // ws.version !== undefined ? versionToString(ws.version) : 'not-set', - // !deepEqual(ws.version, version) ? `upgrade to ${versionToString(version)} is required` : '' - // ) - // console.log('disabled:', ws.disabled) - // console.log('mode:', ws.mode) - // console.log('created by:', ws.createdBy) - // console.log('members:', (ws.accounts ?? []).length) - // if (Number.isNaN(lastVisit)) { - // lastVisit = 365 - // } - // if (lastVisit > 30) { - // console.log(colorConstants.colorRed + `last visit: ${lastVisit} days ago` + colorConstants.reset) - // } else if (lastVisit > 7) { - // console.log(colorConstants.colorRedYellow + `last visit: ${lastVisit} days ago` + colorConstants.reset) - // } else { - // console.log('last visit:', lastVisit, 'days ago') - // } - // } - - // console.log('latest model version:', JSON.stringify(version)) - // }) - // }) - - // program.command('fix-person-accounts-mongo').action(async () => { - // const { version } = prepareTools() - // const mongodbUri = getMongoDBUrl() - // await withAccountDatabase(async (db) => { - // const ws = await listWorkspacesPure(db) - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - // try { - // for (const w of ws) { - // const wsDb = getWorkspaceMongoDB(_client, { name: w.workspace }) - // await wsDb.collection('tx').updateMany( - // { - // objectClass: contact.class.PersonAccount, - // objectSpace: null - // }, - // { $set: { objectSpace: core.space.Model } } - // ) - // } - // } finally { - // client.close() - // } - - // console.log('latest model version:', JSON.stringify(version)) - // }) - // }) - - // program - // .command('show-accounts') - // .description('Show accounts') - // .action(async () => { - // await withAccountDatabase(async (db) => { - // const workspaces = await listWorkspacesPure(db) - // const accounts = await listAccounts(db) - // for (const a of accounts) { - // const wss = a.workspaces.map((it) => it.toString()) - // console.info( - // a.email, - // a.confirmed, - // workspaces.filter((it) => wss.includes(it._id.toString())).map((it) => it.workspaceUrl ?? it.workspace) - // ) - // } - // }) - // }) - - // program - // .command('drop-account ') - // .description('drop account') - // .action(async (email: string, cmd) => { - // await withAccountDatabase(async (db) => { - // await dropAccount(toolCtx, db, null, email) - // }) - // }) - program .command('backup ') .description('dump workspace transactions, blobs and accounts') @@ -1130,6 +640,73 @@ export function devTool ( await storageAdapter.close() }) + program + .command('backup-check-workspace [date]') + .description( + 'Check whether all data from a backup is present in the workspace. Read-only: no data is added, removed or changed.' + ) + .action(async (dirName: string, workspaceId: string, date: string | undefined) => { + await withAccountDatabase(async (db) => { + const { txes, dbUrl } = prepareTools() + const ws = await getWorkspace(db, workspaceId) + if (ws === null) { + throw new Error(`workspace ${workspaceId} not found`) + } + + const wsIds = { uuid: ws.uuid, dataId: ws.dataId, url: ws.url ?? '' } + const storage = await createFileBackupStorage(dirName) + const workspaceStorage: StorageAdapter = buildStorageFromConfig(storageConfigFromEnv()) + + let pipeline: Pipeline | undefined + try { + pipeline = await createBackupPipeline(toolCtx, dbUrl, txes, { + externalStorage: workspaceStorage, + usePassedCtx: true + })( + toolCtx, + { + uuid: ws.uuid, + url: ws.url ?? '', + dataId: ws.dataId + }, + createEmptyBroadcastOps(), + null + ) + if (pipeline === undefined) { + toolCtx.error('failed to check, pipeline is undefined', { workspaceId }) + process.exitCode = 1 + return + } + + const result = await checkWorkspaceBackup(toolCtx, pipeline, wsIds, storage, parseInt(date ?? '-1')) + + console.log('') + for (const d of result.domains) { + const ok = d.missing.length === 0 && d.modified.length === 0 + console.log( + `${ok ? 'OK ' : 'FAIL'} ${d.domain}: backup=${d.backupCount} workspace=${d.workspaceCount} missing=${d.missing.length} modified=${d.modified.length}` + ) + } + console.log( + `${result.blobs.ok ? 'OK ' : 'FAIL'} blobs (storage): total=${result.blobs.total} missing=${result.blobs.missing.length}` + ) + console.log('') + if (result.ok) { + console.log('OK: workspace contains all data from backup') + } else { + console.log('FAILED: workspace is missing data present in the backup') + process.exitCode = 1 + } + } catch (err: any) { + toolCtx.error('failed to check workspace against backup', { err, workspaceId }) + process.exitCode = 1 + } finally { + await pipeline?.close() + await workspaceStorage?.close() + } + }) + }) + program .command('validate-workspace ') .description('Validate a (restored) workspace: connect as system, check model, data counts and blob download') @@ -1416,85 +993,6 @@ export function devTool ( } ) - // program - // .command('backup-list ') - // .description('list snaphost ids for backup') - // .action(async (dirName: string, cmd) => { - // const storage = await createFileBackupStorage(dirName) - // await backupList(storage) - // }) - - // program - // .command('backup-s3 ') - // .description('dump workspace transactions and minio resources') - // .action(async (bucketName: string, dirName: string, workspace: string, cmd) => { - // await withStorage(async (adapter) => { - // const storage = await createStorageBackupStorage(toolCtx, adapter, getWorkspaceId(bucketName), dirName) - // const wsid = getWorkspaceId(workspace) - // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - // await backup(toolCtx, endpoint, wsIds, storage) - // }) - // }) - - // program - // .command('backup-s3-clean ') - // .description('dump workspace transactions and minio resources') - // .action(async (bucketName: string, days: string, cmd) => { - // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - - // const daysInterval = Date.now() - parseInt(days) * 24 * 60 * 60 * 1000 - // try { - // const token = generateToken(systemAccountUuid, undefined, { service: 'tool' }) - // const accountClient = getAccountClient(token) - // const workspaces = (await accountClient.listWorkspaces(null, 'active')).filter((it) => { - // const lastBackup = it.backupInfo?.lastBackup ?? 0 - // if (lastBackup > daysInterval) { - // // No backup required, interval not elapsed - // return true - // } - - // if (it.lastVisit == null) { - // return false - // } - - // return false - // }) - // workspaces.sort((a, b) => { - // return (b.backupInfo?.backupSize ?? 0) - (a.backupInfo?.backupSize ?? 0) - // }) - - // for (const ws of workspaces) { - // const storage = await createStorageBackupStorage( - // toolCtx, - // storageAdapter, - // getWorkspaceId(bucketName), - // ws.workspace - // ) - // await backupRemoveLast(storage, daysInterval) - // const accountClient = getAccountClient(token) - // await accountClient.updateBackupInfo({ - // backups: ws.backupInfo?.backups ?? 0, - // backupSize: ws.backupInfo?.backupSize ?? 0, - // blobsSize: ws.backupInfo?.blobsSize ?? 0, - // dataSize: ws.backupInfo?.dataSize ?? 0, - // lastBackup: daysInterval - // }) - // } - // } finally { - // await storageAdapter.close() - // } - // }) - - // program - // .command('backup-clean ') - // .description('dump workspace transactions and minio resources') - // .action(async (dirName: string, days: string, cmd) => { - // const daysInterval = Date.now() - parseInt(days) * 24 * 60 * 60 * 1000 - // const storage = await createFileBackupStorage(dirName) - // await backupRemoveLast(storage, daysInterval) - // }) - program .command('backup-s3-compact ') .description('Compact a given backup to just one snapshot') @@ -1525,68 +1023,6 @@ export function devTool ( } await storageAdapter.close() }) - // program - // .command('backup-s3-check ') - // .description('Compact a given backup to just one snapshot') - // .action(async (bucketName: string, dirName: string, cmd: any) => { - // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - // try { - // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - // await checkBackupIntegrity(toolCtx, storage) - // } catch (err: any) { - // toolCtx.error('failed to size backup', { err }) - // } - // await storageAdapter.close() - // }) - - // program - // .command('backup-s3-restore [date]') - // .description('dump workspace transactions and minio resources') - // .action(async (bucketName: string, dirName: string, workspace: string, date, cmd) => { - // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - // try { - // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - // const wsid = getWorkspaceId(workspace) - // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - // await restore(toolCtx, endpoint, wsid, storage, { - // date: parseInt(date ?? '-1') - // }) - // } catch (err: any) { - // toolCtx.error('failed to size backup', { err }) - // } - // await storageAdapter.close() - // }) - // program - // .command('backup-s3-list ') - // .description('list snaphost ids for backup') - // .action(async (bucketName: string, dirName: string, cmd) => { - // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - // try { - // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - // await backupList(storage) - // } catch (err: any) { - // toolCtx.error('failed to size backup', { err }) - // } - // await storageAdapter.close() - // }) - - // program - // .command('backup-s3-size ') - // .description('list snaphost ids for backup') - // .action(async (bucketName: string, dirName: string, cmd) => { - // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - // try { - // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - // await backupSize(storage) - // } catch (err: any) { - // toolCtx.error('failed to size backup', { err }) - // } - // await storageAdapter.close() - // }) program .command('backup-s3-download ') @@ -1606,309 +1042,6 @@ export function devTool ( await storageAdapter.close() }) - // program - // .command('copy-s3-datalake') - // .description('copy files from s3 to datalake') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('-c, --concurrency ', 'Number of files being processed concurrently', '10') - // .option('-s, --skip ', 'Number of workspaces to skip', '0') - // .option('-e, --existing', 'Copy existing blobs', false) - // .action(async (cmd: { workspace: string, concurrency: string, existing: boolean, skip: string }) => { - // const params = { - // concurrency: parseInt(cmd.concurrency), - // existing: cmd.existing - // } - // const skip = parseInt(cmd.skip) - - // const storageConfig = storageConfigFromEnv(process.env.STORAGE) - - // const storages = storageConfig.storages.filter((p) => p.kind === S3_CONFIG_KIND) as S3Config[] - // if (storages.length === 0) { - // throw new Error('S3 storage config is required') - // } - - // const datalakeConfig = storageConfig.storages.find((p) => p.kind === DATALAKE_CONFIG_KIND) - // if (datalakeConfig === undefined) { - // throw new Error('Datalake storage config is required') - // } - - // toolCtx.info('using datalake', { datalake: datalakeConfig }) - - // let workspaces: Workspace[] = [] - // await withAccountDatabase(async (db) => { - // workspaces = await listWorkspacesPure(db) - // workspaces = workspaces - // .filter((p) => isActiveMode(p.mode) || isArchivingMode(p.mode)) - // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // // .sort((a, b) => b.lastVisit - a.lastVisit) - // .sort((a, b) => { - // if (a.backupInfo !== undefined && b.backupInfo !== undefined) { - // return b.backupInfo.blobsSize - a.backupInfo.blobsSize - // } else if (b.backupInfo !== undefined) { - // return 1 - // } else if (a.backupInfo !== undefined) { - // return -1 - // } else { - // return b.lastVisit - a.lastVisit - // } - // }) - // }) - - // const count = workspaces.length - // console.log('found workspaces', count) - - // let index = 0 - // for (const workspace of workspaces) { - // index++ - // if (index <= skip) { - // toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) - // continue - // } - - // toolCtx.info('processing workspace', { - // workspace: workspace.workspace, - // index, - // count, - // blobsSize: workspace.backupInfo?.blobsSize ?? 0 - // }) - // const workspaceId = getWorkspaceId(workspace.workspace) - // const token = generateToken(systemAccountEmail, workspaceId) - // const datalake = createDatalakeClient(datalakeConfig as DatalakeConfig, token) - - // for (const config of storages) { - // const storage = new S3Service(config) - // await copyToDatalake(toolCtx, workspaceId, config, storage, datalake, params) - // } - // } - // }) - - // program - // .command('restore-wiki-content-mongo') - // .description('restore wiki document contents') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('-d, --dryrun', 'Dry run', false) - // .action(async (cmd: { workspace: string, dryrun: boolean }) => { - // const params = { - // dryRun: cmd.dryrun - // } - - // const { version } = prepareTools() - - // let workspaces: Workspace[] = [] - // await withAccountDatabase(async (db) => { - // workspaces = await listWorkspacesPure(db) - // workspaces = workspaces - // .filter((p) => isActiveMode(p.mode)) - // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // .sort((a, b) => b.lastVisit - a.lastVisit) - // }) - - // console.log('found workspaces', workspaces.length) - - // await withStorage(async (storageAdapter) => { - // const mongodbUri = getMongoDBUrl() - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - - // try { - // const count = workspaces.length - // let index = 0 - // for (const workspace of workspaces) { - // index++ - - // toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) - // if (workspace.version === undefined || !deepEqual(workspace.version, version)) { - // console.log(`upgrade to ${versionToString(version)} is required`) - // continue - // } - - // const workspaceId = getWorkspaceId(workspace.workspace) - // const workspaceDataId = workspace.dataId ?? workspace.uuid - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - // await restoreWikiContentMongo(toolCtx, wsDb, workspaceDataId, storageAdapter, params) - // } - // } finally { - // client.close() - // } - // }) - // }) - - // program - // .command('restore-controlled-content-mongo') - // .description('restore controlled document contents') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('-d, --dryrun', 'Dry run', false) - // .option('-f, --force', 'Force update', false) - // .action(async (cmd: { workspace: string, dryrun: boolean, force: boolean }) => { - // const params = { - // dryRun: cmd.dryrun - // } - - // const { version } = prepareTools() - - // let workspaces: Workspace[] = [] - // await withAccountDatabase(async (db) => { - // workspaces = await listWorkspacesPure(db) - // workspaces = workspaces - // .filter((p) => p.mode !== 'archived') - // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // .sort((a, b) => b.lastVisit - a.lastVisit) - // }) - - // console.log('found workspaces', workspaces.length) - - // await withStorage(async (storageAdapter) => { - // await withAccountDatabase(async (db) => { - // const mongodbUri = getMongoDBUrl() - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - - // try { - // const count = workspaces.length - // let index = 0 - // for (const workspace of workspaces) { - // index++ - - // toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) - - // if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) { - // console.log(`upgrade to ${versionToString(version)} is required`) - // continue - // } - - // const workspaceId = getWorkspaceId(workspace.workspace) - // const workspaceDataId = workspace.dataId ?? workspace.uuid - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - // await restoreControlledDocContentMongo(toolCtx, wsDb, workspaceDataId, storageAdapter, params) - // } - // } finally { - // client.close() - // } - // }) - // }) - // }) - - // program - // .command('restore-markup-ref-mongo') - // .description('restore markup document content refs') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('-f, --force', 'Force update', false) - // .action(async (cmd: { workspace: string, force: boolean }) => { - // const { txes, version } = prepareTools() - - // const { hierarchy } = await buildModel(toolCtx, txes) - - // let workspaces: Workspace[] = [] - // await withAccountDatabase(async (db) => { - // workspaces = await listWorkspacesPure(db) - // workspaces = workspaces - // .filter((p) => isActiveMode(p.mode)) - // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // .sort((a, b) => b.lastVisit - a.lastVisit) - // }) - - // console.log('found workspaces', workspaces.length) - - // await withStorage(async (storageAdapter) => { - // const mongodbUri = getMongoDBUrl() - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - - // try { - // const count = workspaces.length - // let index = 0 - // for (const workspace of workspaces) { - // index++ - - // toolCtx.info('processing workspace', { - // workspace: workspace.workspace, - // version: workspace.version, - // index, - // count - // }) - - // if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) { - // console.log(`upgrade to ${versionToString(version)} is required`) - // continue - // } - - // const workspaceId = getWorkspaceId(workspace.workspace) - // const workspaceDataId = workspace.dataId ?? workspace.uuid - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - // await restoreMarkupRefsMongo(toolCtx, wsDb, workspaceDataId, hierarchy, storageAdapter) - // } - // } finally { - // client.close() - // } - // }) - // }) - - // program - // .command('confirm-email ') - // .description('confirm user email') - // .action(async (email: string, cmd) => { - // await withAccountDatabase(async (db) => { - // const account = await getAccount(db, email) - // if (account?.confirmed === true) { - // console.log(`Already confirmed:${email}`) - // } else { - // await confirmEmail(db, email) - // } - // }) - // }) - - // program - // .command('diff-workspace ') - // .description('restore workspace transactions and minio resources from previous dump.') - // .action(async (workspace: string, cmd) => { - // const { dbUrl, txes } = prepareTools() - // await diffWorkspace(dbUrl, workspace, txes) - // }) - - // program - // .command('clear-telegram-history ') - // .description('clear telegram history') - // .option('-w, --workspace ', 'target workspace') - // .action(async (workspace: string, cmd) => { - // const { dbUrl } = prepareTools() - // await withStorage(async (adapter) => { - // const telegramDB = process.env.TELEGRAM_DATABASE - // if (telegramDB === undefined) { - // console.error('please provide TELEGRAM_DATABASE.') - // process.exit(1) - // } - - // console.log(`clearing ${workspace} history:`) - // await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(workspace), telegramDB, adapter) - // }) - // }) - - // program - // .command('clear-telegram-all-history') - // .description('clear telegram history') - // .action(async (cmd) => { - // const { dbUrl } = prepareTools() - // await withStorage(async (adapter) => { - // await withAccountDatabase(async (db) => { - // const telegramDB = process.env.TELEGRAM_DATABASE - // if (telegramDB === undefined) { - // console.error('please provide TELEGRAM_DATABASE.') - // process.exit(1) - // } - - // const workspaces = await listWorkspacesPure(db) - - // for (const w of workspaces) { - // console.log(`clearing ${w.workspace} history:`) - // await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(w.workspace), telegramDB, adapter) - // } - // }) - // }) - // }) - program .command('generate-token ') .description('generate token') @@ -1980,248 +1113,6 @@ export function devTool ( console.log(decodeToken(token)) }) - // program - // .command('clean-workspace ') - // .description('clean workspace') - // .option('--recruit', 'Clean recruit', false) - // .option('--tracker', 'Clean tracker', false) - // .option('--removedTx', 'Clean removed transactions', false) - // .action(async (workspace: string, cmd: { recruit: boolean, tracker: boolean, removedTx: boolean }) => { - // const { dbUrl } = prepareTools() - // await withStorage(async (adapter) => { - // const wsid = getWorkspaceId(workspace) - // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - // await cleanWorkspace(toolCtx, dbUrl, wsid, adapter, endpoint, cmd) - // }) - // }) - // program - // .command('clean-empty-buckets') - // .option('--prefix [prefix]', 'Prefix', '') - // .action(async (cmd: { prefix: string }) => { - // await withStorage(async (adapter) => { - // const buckets = await adapter.listBuckets(toolCtx) - // for (const ws of buckets) { - // if (ws.name.startsWith(cmd.prefix)) { - // console.log('Checking', ws.name) - // const l = await ws.list() - // const docs = await l.next() - // if (docs.length === 0) { - // await l.close() - // // No data, we could delete it. - // console.log('Clean bucket', ws.name) - // await ws.delete() - // } else { - // await l.close() - // } - // } - // } - // }) - // }) - // program - // .command('upload-file ') - // .action(async (workspace: string, local: string, remote: string, contentType: string, cmd: any) => { - // const wsId: WorkspaceId = { - // name: workspace - // } - // const token = generateToken(systemAccountEmail, wsId) - // const endpoint = await getTransactorEndpoint(token) - // const blobClient = new BlobClient(endpoint, token, wsId) - // const buffer = readFileSync(local) - // await blobClient.upload(toolCtx, remote, buffer.length, contentType, buffer) - // }) - - // program - // .command('download-file ') - // .action(async (workspace: string, remote: string, local: string, cmd: any) => { - // const wsId: WorkspaceId = { - // name: workspace - // } - // const token = generateToken(systemAccountEmail, wsId) - // const endpoint = await getTransactorEndpoint(token) - // const blobClient = new BlobClient(endpoint, token, wsId) - // const wrstream = createWriteStream(local) - // await blobClient.writeTo(toolCtx, remote, -1, { - // write: (buffer, cb) => { - // wrstream.write(buffer, cb) - // }, - // end: (cb) => { - // wrstream.end(cb) - // } - // }) - // }) - - // program - // .command('move-files') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('-m, --move ', 'When set to true, the files will be moved, otherwise copied', 'false') - // .option('-bl, --blobLimit ', 'A blob size limit in megabytes (default 50mb)', '999999') - // .option('-c, --concurrency ', 'Number of files being processed concurrently', '10') - // .option('--disabled', 'Include disabled workspaces', false) - // .action( - // async (cmd: { workspace: string, move: string, blobLimit: string, concurrency: string, disabled: boolean }) => { - // const params = { - // concurrency: parseInt(cmd.concurrency), - // move: cmd.move === 'true' - // } - - // await withAccountDatabase(async (db) => { - // await withStorage(async (adapter) => { - // try { - // const exAdapter = adapter as StorageAdapterEx - // if (exAdapter.adapters === undefined || exAdapter.adapters.length < 2) { - // throw new Error('bad storage config, at least two storage providers are required') - // } - - // console.log('moving files to storage provider', exAdapter.adapters[0].name) - - // let index = 1 - // const workspaces = await listWorkspacesPure(db) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - - // const rateLimit = new RateLimiter(10) - // for (const workspace of workspaces) { - // if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - // continue - // } - // if (!isActiveMode(workspace.mode)) { - // console.log('ignore non active workspace', workspace.workspace, workspace.mode) - // continue - // } - // if (workspace.disabled === true && !cmd.disabled) { - // console.log('ignore disabled workspace', workspace.workspace) - // continue - // } - - // await rateLimit.exec(async () => { - // console.log('start', workspace.workspace, index, '/', workspaces.length) - // await moveFiles(toolCtx, getWorkspaceId(workspace.workspace), exAdapter, params) - // console.log('done', workspace.workspace) - // index += 1 - // }) - // } - // await rateLimit.waitProcessing() - // } catch (err: any) { - // console.error(err) - // } - // }) - // }) - // } - // ) - - // program - // .command('show-lost-files-mongo') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('--disabled', 'Include disabled workspaces', false) - // .option('--all', 'Show all files', false) - // .action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => { - // await withAccountDatabase(async (db) => { - // await withStorage(async (adapter) => { - // const mongodbUri = getMongoDBUrl() - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - // try { - // let index = 1 - // const workspaces = await listWorkspacesPure(db) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - - // for (const workspace of workspaces) { - // if (!isActiveMode(workspace.mode)) { - // console.log('ignore non active workspace', workspace.workspace, workspace.mode) - // continue - // } - // if (workspace.disabled === true && !cmd.disabled) { - // console.log('ignore disabled workspace', workspace.workspace) - // continue - // } - - // if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - // continue - // } - - // try { - // console.log('start', workspace.workspace, index, '/', workspaces.length) - // const workspaceId = getWorkspaceId(workspace.workspace) - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - // await showLostFiles(toolCtx, workspaceId, wsDb, adapter, { showAll: cmd.all }) - // console.log('done', workspace.workspace) - // } catch (err) { - // console.error(err) - // } - - // index += 1 - // } - // } catch (err: any) { - // console.error(err) - // } finally { - // client.close() - // } - // }) - // }) - // }) - - // program.command('fix-bw-workspace ').action(async (workspace: string) => { - // await withStorage(async (adapter) => { - // await fixMinioBW(toolCtx, getWorkspaceId(workspace), adapter) - // }) - // }) - - // program - // .command('clean-removed-transactions ') - // .description('clean removed transactions') - // .action(async (workspace: string, cmd: any) => { - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await cleanRemovedTransactions(wsid, endpoint) - // }) - - // program - // .command('clean-archived-spaces ') - // .description('clean archived spaces') - // .action(async (workspace: string, cmd: any) => { - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await cleanArchivedSpaces(wsid, endpoint) - // }) - - // program - // .command('chunter-fix-comments ') - // .description('chunter-fix-comments') - // .action(async (workspace: string, cmd: any) => { - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await fixCommentDoubleIdCreate(wsid, endpoint) - // }) - - // program - // .command('mixin-show-foreign-attributes ') - // .description('mixin-show-foreign-attributes') - // .option('--mixin ', 'Mixin class', '') - // .option('--property ', 'Property name', '') - // .option('--detail ', 'Show details', false) - // .action(async (workspace: string, cmd: { detail: boolean, mixin: string, property: string }) => { - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await showMixinForeignAttributes(wsid, endpoint, cmd) - // }) - - // program - // .command('mixin-fix-foreign-attributes-mongo ') - // .description('mixin-fix-foreign-attributes') - // .option('--mixin ', 'Mixin class', '') - // .option('--property ', 'Property name', '') - // .action(async (workspace: string, cmd: { mixin: string, property: string }) => { - // const mongodbUri = getMongoDBUrl() - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // FIXME: add dataId - // await fixMixinForeignAttributes(mongodbUri, wsid, endpoint, cmd) - // }) - program .command('configure ') .description('clean archived spaces') @@ -2240,152 +1131,6 @@ export function devTool ( }) }) - // program - // .command('configure-all') - // .description('configure all spaces') - // .option('--enable ', 'Enable plugin configuration', '') - // .option('--disable ', 'Disable plugin configuration', '') - // .option('--list', 'List plugin states', false) - // .action(async (cmd: { enable: string, disable: string, list: boolean }) => { - // await withAccountDatabase(async (db) => { - // console.log('configure all workspaces') - // console.log(JSON.stringify(cmd)) - // const workspaces = await listWorkspacesRaw(db) - // for (const ws of workspaces) { - // console.log('configure', ws.workspaceName ?? ws.workspace) - // const wsid = getWorkspaceId(ws.workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await changeConfiguration(wsid, endpoint, cmd) - // } - // }) - // }) - - // program - // .command('optimize-model ') - // .description('optimize model') - // .action(async (workspace: string, cmd: { enable: string, disable: string, list: boolean }) => { - // console.log(JSON.stringify(cmd)) - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await optimizeModel(wsid, endpoint) - // }) - - // program - // .command('benchmark') - // .description('benchmark') - // .option('--from ', 'Min client count', '10') - // .option('--steps ', 'Step with client count', '10') - // .option('--sleep ', 'Random Delay max between operations', '0') - // .option('--binary ', 'Use binary data transfer', false) - // .option('--compression ', 'Use protocol compression', false) - // .option('--write ', 'Perform write operations', false) - // .option('--workspaces ', 'Workspaces to test on, comma separated', '') - // .option('--mode ', 'A benchmark mode. Supported values: `find-all`, `connect-only` ', 'find-all') - // .action( - // async (cmd: { - // from: string - // steps: string - // sleep: string - // workspaces: string - // binary: string - // compression: string - // write: string - // mode: 'find-all' | 'connect-only' - // }) => { - // await withAccountDatabase(async (db) => { - // console.log(JSON.stringify(cmd)) - // if (!['find-all', 'connect-only'].includes(cmd.mode)) { - // console.log('wrong mode') - // return - // } - - // const allWorkspacesPure = Array.from(await listWorkspacesPure(db)) - // const allWorkspaces = new Map(allWorkspacesPure.map((it) => [it.workspace, it])) - - // let workspaces = cmd.workspaces - // .split(',') - // .map((it) => it.trim()) - // .filter((it) => it.length > 0) - // .map((it) => getWorkspaceId(it)) - - // if (cmd.workspaces.length === 0) { - // workspaces = allWorkspacesPure.map((it) => getWorkspaceId(it.workspace)) - // } - // const accounts = new Map(Array.from(await listAccounts(db)).map((it) => [it._id.toString(), it.email])) - - // const accountWorkspaces = new Map() - // for (const ws of workspaces) { - // const wsInfo = allWorkspaces.get(ws.name) - // if (wsInfo !== undefined) { - // accountWorkspaces.set( - // ws.name, - // wsInfo.accounts.map((it) => accounts.get(it.toString()) as string) - // ) - // } - // } - // await benchmark(workspaces, accountWorkspaces, accountsUrl, { - // steps: parseInt(cmd.steps), - // from: parseInt(cmd.from), - // sleep: parseInt(cmd.sleep), - // binary: cmd.binary === 'true', - // compression: cmd.compression === 'true', - // write: cmd.write === 'true', - // mode: cmd.mode - // }) - // }) - // } - // ) - // program - // .command('benchmarkWorker') - // .description('benchmarkWorker') - // .action(async (cmd: any) => { - // console.log(JSON.stringify(cmd)) - // benchmarkWorker() - // }) - - // program - // .command('stress ') - // .description('stress benchmark') - // .option('--mode ', 'A benchmark mode. Supported values: `wrong`, `connect-disconnect` ', 'wrong') - // .action(async (transactor: string, cmd: { mode: StressBenchmarkMode }) => { - // await stressBenchmark(transactor, cmd.mode) - // }) - - // program - // .command('fix-skills-mongo ') - // .description('fix skills for workspace') - // .action(async (workspace: string, step: string) => { - // const mongodbUri = getMongoDBUrl() - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await fixSkills(mongodbUri, wsid, endpoint, step) - // }) - - // program - // .command('restore-ats-types-mongo ') - // .description('Restore recruiting task types for workspace') - // .action(async (workspace: string) => { - // const mongodbUri = getMongoDBUrl() - // console.log('Restoring recruiting task types in workspace ', workspace, '...') - // const wsid = getWorkspaceId(workspace) - // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - // await restoreRecruitingTaskTypes(mongodbUri, wsid, endpoint) - // }) - - // program - // .command('restore-ats-types-2-mongo ') - // .description('Restore recruiting task types for workspace 2') - // .action(async (workspace: string) => { - // const mongodbUri = getMongoDBUrl() - // console.log('Restoring recruiting task types in workspace ', workspace, '...') - // const wsid = getWorkspaceId(workspace) - // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - // await restoreHrTaskTypesFromUpdates(mongodbUri, wsid, endpoint) - // }) - program .command('change-field ') .description('change field value for the object') @@ -2471,149 +1216,6 @@ export function devTool ( console.log('done') }) - // program - // .command('remove-duplicates-ids-mongo ') - // .description('remove duplicates ids for futue migration') - // .action(async (workspaces: string) => { - // const mongodbUri = getMongoDBUrl() - // await withStorage(async (adapter) => { - // await removeDuplicateIds(toolCtx, mongodbUri, adapter, accountsUrl, workspaces) - // }) - // }) - - // program.command('move-to-pg ').action(async (region: string) => { - // const { dbUrl } = prepareTools() - // const mongodbUri = getMongoDBUrl() - - // await withAccountDatabase(async (db) => { - // const workspaces = await listWorkspacesRaw(db) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - // await moveFromMongoToPG( - // db, - // mongodbUri, - // dbUrl, - // workspaces.filter((p) => p.region !== region), - // region - // ) - // }) - // }) - - // program - // .command('move-workspace-to-pg ') - // .option('-i, --include ', 'A list of ; separated domain names to include during backup', '*') - // .option('-f|--force [force]', 'Force update', false) - // .action( - // async ( - // workspace: string, - // region: string, - // cmd: { - // include: string - // force: boolean - // } - // ) => { - // const { dbUrl } = prepareTools() - // const mongodbUri = getMongoDBUrl() - - // await withAccountDatabase(async (db) => { - // const ws = await getWorkspace(db, workspace) - // if (ws === null) { - // throw new Error(`workspace ${workspace} not found`) - // } - // await updateField(ws.uuid, await getWorkspaceTransactorEndpoint(ws.uuid), cmd) - // }) - // } - // ) - - // program - // .command('recreate-elastic-indexes-mongo ') - // .description('reindex workspace to elastic') - // .action(async (workspace: string) => { - // const mongodbUri = getMongoDBUrl() - // const wsid = getWorkspaceId(workspace) - // await recreateElastic(mongodbUri, wsid) - // }) - - // program - // .command('recreate-all-elastic-indexes-mongo') - // .description('reindex elastic') - // .action(async () => { - // const { dbUrl } = prepareTools() - // const mongodbUri = getMongoDBUrl() - - // await withAccountDatabase(async (db) => { - // const workspaces = await listWorkspacesRaw(db) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - // for (const workspace of workspaces) { - // const wsid = getWorkspaceId(workspace.workspace) - // await recreateElastic(mongodbUri ?? dbUrl, wsid) - // } - // }) - // }) - - // program - // .command('remove-duplicates-ids-mongo ') - // .description('remove duplicates ids for futue migration') - // .action(async (workspaces: string) => { - // const mongodbUri = getMongoDBUrl() - // await withStorage(async (adapter) => { - // await removeDuplicateIds(toolCtx, mongodbUri, adapter, accountsUrl, workspaces) - // }) - // }) - - // program.command('move-to-pg ').action(async (region: string) => { - // const { dbUrl } = prepareTools() - // const mongodbUri = getMongoDBUrl() - - // await withAccountDatabase(async (db) => { - // const workspaces = await listWorkspacesRaw(db) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - // await moveFromMongoToPG( - // db, - // mongodbUri, - // dbUrl, - // workspaces.filter((p) => p.region !== region), - // region - // ) - // }) - // }) - - // program - // .command('move-workspace-to-pg ') - // .option('-i, --include ', 'A list of ; separated domain names to include during backup', '*') - // .option('-f|--force [force]', 'Force update', false) - // .action( - // async ( - // workspace: string, - // region: string, - // cmd: { - // include: string - // force: boolean - // } - // ) => { - // const { dbUrl } = prepareTools() - // const mongodbUri = getMongoDBUrl() - - // await withAccountDatabase(async (db) => { - // const workspaceInfo = await getWorkspaceById(db, workspace) - // if (workspaceInfo === null) { - // throw new Error(`workspace ${workspace} not found`) - // } - // if (workspaceInfo.region === region && !cmd.force) { - // throw new Error(`workspace ${workspace} is already migrated`) - // } - // await moveWorkspaceFromMongoToPG( - // db, - // mongodbUri, - // dbUrl, - // workspaceInfo, - // region, - // cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())), - // cmd.force - // ) - // }) - // } - // ) - program.command('move-account-db-to-pg').action(async () => { const { dbUrl } = prepareTools() const mongodbUri = getMongoDBUrl() @@ -2718,123 +1320,6 @@ export function devTool ( }, dbUrl) }) - // program - // .command('perfomance') - // .option('-p, --parallel', '', false) - // .action(async (cmd: { parallel: boolean }) => { - // const { txes, version, migrateOperations } = prepareTools() - // await withAccountDatabase(async (db) => { - // const email = generateId() - // const ws = generateId() - // const wsid = getWorkspaceId(ws) - // const start = new Date() - // const measureCtx = new MeasureMetricsContext('create-workspace', {}) - // const wsInfo = await createWorkspaceRecord(measureCtx, db, null, email, ws, ws) - - // // update the record so it's not taken by one of the workers for the next 60 seconds - // await updateWorkspace(db, wsInfo, { - // mode: 'creating', - // progress: 0, - // lastProcessingTime: Date.now() + 1000 * 60 - // }) - - // await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations, undefined, true) - - // await updateWorkspace(db, wsInfo, { - // mode: 'active', - // progress: 100, - // disabled: false, - // version - // }) - // await createAcc(toolCtx, db, null, email, '1234', '', '', true) - // await assignAccountToWs(toolCtx, db, null, email, ws, AccountRole.User) - // console.log('Workspace created in', new Date().getTime() - start.getTime(), 'ms') - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token, 'external') - // await generateWorkspaceData(endpoint, ws, cmd.parallel, email) - // await testFindAll(endpoint, ws, email) - // await dropWorkspace(toolCtx, db, null, ws) - // }) - // }) - - // program - // .command('reset-ws-attempts ') - // .description('Reset workspace creation/upgrade attempts counter') - // .action(async (workspace) => { - // await withAccountDatabase(async (db) => { - // const info = await getWorkspaceById(db, workspace) - // if (info === null) { - // throw new Error(`workspace ${workspace} not found`) - // } - - // await updateWorkspace(db, info, { - // attempts: 0 - // }) - - // console.log('Attempts counter for workspace', workspace, 'has been reset') - // }) - // }) - - // program - // .command('add-controlled-doc-rank-mongo') - // .description('add rank to controlled documents') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .action(async (cmd: { workspace: string }) => { - // const { version } = prepareTools() - - // let workspaces: Workspace[] = [] - // await withAccountDatabase(async (db) => { - // workspaces = await listWorkspacesPure(db) - // workspaces = workspaces - // .filter((p) => isActiveMode(p.mode)) - // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // .sort((a, b) => b.lastVisit - a.lastVisit) - // }) - - // console.log('found workspaces', workspaces.length) - - // const mongodbUri = getMongoDBUrl() - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - - // try { - // const count = workspaces.length - // let index = 0 - // for (const workspace of workspaces) { - // index++ - - // toolCtx.info('processing workspace', { - // workspace: workspace.workspace, - // version: workspace.version, - // index, - // count - // }) - - // if (workspace.version === undefined || !deepEqual(workspace.version, version)) { - // console.log(`upgrade to ${versionToString(version)} is required`) - // continue - // } - // const workspaceId = getWorkspaceId(workspace.workspace) - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - // await addControlledDocumentRank(toolCtx, wsDb, workspaceId) - // } - // } finally { - // client.close() - // } - // }) - - // Not needed anymore? - // program - // .command('fill-github-users') - // .option('-t, --token ', 'Github token to increase the limit of requests to GitHub') - // .description('adds github username info to all accounts') - // .action(async (cmd: { token?: string }) => { - // await withAccountDatabase(async (db) => { - // await fillGithubUsers(toolCtx, db, cmd.token) - // }) - // }) - program .command('migrate-github-account') .option('--db ', 'Github DB', '%github') diff --git a/models/process/src/functions.ts b/models/process/src/functions.ts index 11d982c12c..2ed2985176 100644 --- a/models/process/src/functions.ts +++ b/models/process/src/functions.ts @@ -185,6 +185,19 @@ export function defineFunctions (builder: Builder): void { process.function.All ) + builder.createDoc( + process.class.ProcessFunction, + core.space.Model, + { + of: core.class.ArrOf, + category: 'array', + label: process.string.AllMatchValue, + type: 'reduce', + editor: process.transformEditor.FilterEditor + }, + process.function.AllMatchValue + ) + builder.createDoc( process.class.ProcessFunction, core.space.Model, diff --git a/models/server-process/src/index.ts b/models/server-process/src/index.ts index a1179e8160..eb040fa7c1 100644 --- a/models/server-process/src/index.ts +++ b/models/server-process/src/index.ts @@ -388,6 +388,10 @@ export function createModel (builder: Builder): void { func: serverProcess.transform.FirstMatchValue }) + builder.mixin(process.function.AllMatchValue, process.class.ProcessFunction, serverProcess.mixin.FuncImpl, { + func: serverProcess.transform.AllMatchValue + }) + builder.mixin(process.function.Filter, process.class.ProcessFunction, serverProcess.mixin.FuncImpl, { func: serverProcess.transform.Filter }) diff --git a/plugins/process-assets/lang/cs.json b/plugins/process-assets/lang/cs.json index 33c159f0d3..715f5be61d 100644 --- a/plugins/process-assets/lang/cs.json +++ b/plugins/process-assets/lang/cs.json @@ -117,6 +117,7 @@ "ExecutionInitiator": "Iniciátor provedení", "ExecutionStarted": "Provedení spuštěno", "Filter": "Filtr", + "AllMatchValue": "Všechny odpovídající hodnoty", "FirstMatchValue": "První odpovídající hodnota", "ConfigLabel": "Konfigurace procesů", "ConfigDescription": "Definujte procesy pro automatizaci pracovních postupů a obchodních procesů.", @@ -197,4 +198,4 @@ "TooDeepTransitionRecursion": "Příliš hluboká rekurze přechodů", "ToDoAlreadyCompleted": "Akční položka již byla dokončena" } -} \ No newline at end of file +} diff --git a/plugins/process-assets/lang/de.json b/plugins/process-assets/lang/de.json index 33f6d9512c..d7c4ba82e3 100644 --- a/plugins/process-assets/lang/de.json +++ b/plugins/process-assets/lang/de.json @@ -117,6 +117,7 @@ "ExecutionInitiator": "Ausführungsinitiator", "ExecutionStarted": "Ausführung gestartet", "Filter": "Filter", + "AllMatchValue": "Alle passenden Werte", "FirstMatchValue": "Erster passender Wert", "ConfigLabel": "Prozesskonfiguration", "ConfigDescription": "Definieren Sie Prozesse, um Workflows und Geschäftsprozesse zu automatisieren.", diff --git a/plugins/process-assets/lang/en.json b/plugins/process-assets/lang/en.json index bccf7c8eeb..4c22b97054 100644 --- a/plugins/process-assets/lang/en.json +++ b/plugins/process-assets/lang/en.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Execution initiator", "ExecutionStarted": "Execution started", "Filter": "Filter", + "AllMatchValue": "All matching values", "FirstMatchValue": "First match value", "ConfigLabel": "Process configuration", "ConfigDescription": "Define processes to automate workflows and business processes.", diff --git a/plugins/process-assets/lang/es.json b/plugins/process-assets/lang/es.json index 1e378ef541..efc1eb50b0 100644 --- a/plugins/process-assets/lang/es.json +++ b/plugins/process-assets/lang/es.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Iniciador de Ejecución", "ExecutionStarted": "Ejecución Iniciada", "Filter": "Filtro", + "AllMatchValue": "Todos los valores coincidentes", "FirstMatchValue": "Primer valor coincidente", "ConfigLabel": "Configuración de procesos", "ConfigDescription": "Defina procesos para automatizar flujos de trabajo y procesos comerciales.", diff --git a/plugins/process-assets/lang/fr.json b/plugins/process-assets/lang/fr.json index 23ca3bacd7..54c238bdfb 100644 --- a/plugins/process-assets/lang/fr.json +++ b/plugins/process-assets/lang/fr.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Initiateur d'exécution", "ExecutionStarted": "Exécution démarrée", "Filter": "Filtrer", + "AllMatchValue": "Toutes les valeurs correspondantes", "FirstMatchValue": "Première valeur correspondante", "ConfigLabel": "Configuration des processus", "ConfigDescription": "Définissez des processus pour automatiser les flux de travail et les processus métier.", diff --git a/plugins/process-assets/lang/it.json b/plugins/process-assets/lang/it.json index ba760e1993..065536fa15 100644 --- a/plugins/process-assets/lang/it.json +++ b/plugins/process-assets/lang/it.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Iniziatore dell'esecuzione", "ExecutionStarted": "Esecuzione avviata", "Filter": "Filtro", + "AllMatchValue": "Tutti i valori corrispondenti", "FirstMatchValue": "Primo valore corrispondente", "ConfigLabel": "Configurazione del processo", "ConfigDescription": "Definire i processi per automatizzare i flussi di lavoro e i processi aziendali.", diff --git a/plugins/process-assets/lang/ja.json b/plugins/process-assets/lang/ja.json index 5764530c01..f73657ee90 100644 --- a/plugins/process-assets/lang/ja.json +++ b/plugins/process-assets/lang/ja.json @@ -123,6 +123,7 @@ "ExecutionInitiator": "実行の発起人", "ExecutionStarted": "実行が開始されました", "Filter": "フィルター", + "AllMatchValue": "一致するすべての値", "FirstMatchValue": "最初の一致する値", "ConfigLabel": "プロセス構成", "ConfigDescription": "ワークフローやビジネスプロセスを自動化するためのプロセスを定義します。", diff --git a/plugins/process-assets/lang/ko.json b/plugins/process-assets/lang/ko.json index b85b181871..c5143204b3 100644 --- a/plugins/process-assets/lang/ko.json +++ b/plugins/process-assets/lang/ko.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "실행 시작자", "ExecutionStarted": "실행 시작됨", "Filter": "필터", + "AllMatchValue": "모든 일치 값", "FirstMatchValue": "첫 일치 값", "ConfigLabel": "프로세스 구성", "ConfigDescription": "워크플로와 비즈니스 프로세스를 자동화할 프로세스를 정의합니다.", diff --git a/plugins/process-assets/lang/pl.json b/plugins/process-assets/lang/pl.json index 1c2916fd17..50a71e204d 100644 --- a/plugins/process-assets/lang/pl.json +++ b/plugins/process-assets/lang/pl.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Inicjator wykonania", "ExecutionStarted": "Wykonanie rozpoczęte", "Filter": "Filtr", + "AllMatchValue": "Wszystkie pasujące wartości", "FirstMatchValue": "Pierwsza pasująca wartość", "ConfigLabel": "Konfiguracja procesu", "ConfigDescription": "Definiuj procesy do automatyzacji organizacji zadań i procesów biznesowych.", diff --git a/plugins/process-assets/lang/pt-br.json b/plugins/process-assets/lang/pt-br.json index 9f50306818..470219d7c2 100644 --- a/plugins/process-assets/lang/pt-br.json +++ b/plugins/process-assets/lang/pt-br.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Iniciador de Execução", "ExecutionStarted": "Execução Iniciada", "Filter": "Filtrar", + "AllMatchValue": "Todos os valores correspondentes", "FirstMatchValue": "Primeiro Valor Correspondente", "ConfigLabel": "Configuração de processos", "ConfigDescription": "Defina processos para automatizar fluxos de trabalho e processos de negócios.", diff --git a/plugins/process-assets/lang/pt.json b/plugins/process-assets/lang/pt.json index de5c2dd6c2..f660e43e63 100644 --- a/plugins/process-assets/lang/pt.json +++ b/plugins/process-assets/lang/pt.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Iniciador de Execução", "ExecutionStarted": "Execução Iniciada", "Filter": "Filtrar", + "AllMatchValue": "Todos os valores correspondentes", "FirstMatchValue": "Primeiro Valor Correspondente", "ConfigLabel": "Configuração de processos", "ConfigDescription": "Defina processos para automatizar fluxos de trabalho e processos de negócios.", diff --git a/plugins/process-assets/lang/ru.json b/plugins/process-assets/lang/ru.json index 446fca4f13..1a80cf32ce 100644 --- a/plugins/process-assets/lang/ru.json +++ b/plugins/process-assets/lang/ru.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Инициатор выполнения", "ExecutionStarted": "Выполнение начато", "Filter": "Фильтр", + "AllMatchValue": "Все подходящие значения", "FirstMatchValue": "Первое подходящее значение", "ConfigLabel": "Конфигурация процессов", "ConfigDescription": "Определяйте процессы для автоматизации рабочих процессов и бизнес-процессов.", diff --git a/plugins/process-assets/lang/tr.json b/plugins/process-assets/lang/tr.json index 437e325a76..bc2181f246 100644 --- a/plugins/process-assets/lang/tr.json +++ b/plugins/process-assets/lang/tr.json @@ -121,6 +121,7 @@ "ProcessFinished": "\"{process}\" süreci \"{state}\" durumunda tamamlandı", "NewProcessToDo": "Yeni süreç Eylem öğesi", "Filter": "Filtre", + "AllMatchValue": "Tüm eşleşen değerler", "FirstMatchValue": "İlk eşleşen değer", "ConfigLabel": "Süreç yapılandırması", "ConfigDescription": "İş akışlarını ve iş süreçlerini otomatikleştirmek için süreçler tanımlayın.", diff --git a/plugins/process-assets/lang/zh.json b/plugins/process-assets/lang/zh.json index 9c245cbbc8..3d6faf0406 100644 --- a/plugins/process-assets/lang/zh.json +++ b/plugins/process-assets/lang/zh.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "执行发起者", "ExecutionStarted": "执行已启动", "Filter": "过滤", + "AllMatchValue": "所有匹配值", "FirstMatchValue": "第一个匹配值", "ConfigLabel": "流程配置", "ConfigDescription": "定义流程以自动化工作流和业务流程。", diff --git a/plugins/process-resources/src/plugin.ts b/plugins/process-resources/src/plugin.ts index 70d4c32262..d512dcc172 100644 --- a/plugins/process-resources/src/plugin.ts +++ b/plugins/process-resources/src/plugin.ts @@ -157,6 +157,7 @@ export default mergeIds(processId, process, { Split: '' as IntlString, Cut: '' as IntlString, Filter: '' as IntlString, + AllMatchValue: '' as IntlString, FirstMatchValue: '' as IntlString, FirstValue: '' as IntlString, LastValue: '' as IntlString, diff --git a/plugins/process/src/index.ts b/plugins/process/src/index.ts index 52015fef0a..9568ea8602 100644 --- a/plugins/process/src/index.ts +++ b/plugins/process/src/index.ts @@ -353,6 +353,7 @@ export default plugin(processId, { OnEvent: '' as Asset }, function: { + AllMatchValue: '' as Ref, FirstMatchValue: '' as Ref, Filter: '' as Ref, FirstValue: '' as Ref, diff --git a/pods/preview/src/__tests__/middleware.test.ts b/pods/preview/src/__tests__/middleware.test.ts new file mode 100644 index 0000000000..e8687da2f1 --- /dev/null +++ b/pods/preview/src/__tests__/middleware.test.ts @@ -0,0 +1,140 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type AccountUuid, systemAccountUuid, type WorkspaceUuid } from '@hcengineering/core' +import { extractToken } from '@hcengineering/server-client' +import { type Token } from '@hcengineering/server-token' +import { type NextFunction, type Response } from 'express' + +import { HttpError } from '../error' +import { type RequestWithAuth, withAuthorization, withBlob } from '../middleware' + +jest.mock('@hcengineering/server-client', () => ({ + extractToken: jest.fn() +})) + +const extractTokenMock = extractToken as jest.MockedFunction + +const workspaceA = '00000000-0000-4000-8000-00000000000a' as WorkspaceUuid +const workspaceB = '00000000-0000-4000-8000-00000000000b' as WorkspaceUuid +const account = '00000000-0000-4000-8000-0000000000ac' as AccountUuid + +function makeToken (token: Partial): Token { + const result: Token = { account, workspace: workspaceA, extra: {} } + return { ...result, ...token } +} + +function makeRequest (workspace: string, name: string, token?: Token): RequestWithAuth { + return { headers: {}, params: { workspace, name }, token } as unknown as RequestWithAuth +} + +const res = {} as unknown as Response + +describe('withAuthorization', () => { + beforeEach(() => { + extractTokenMock.mockReset() + }) + + it('rejects requests without a token', () => { + extractTokenMock.mockReturnValue(undefined) + const next = jest.fn() as unknown as NextFunction + + withAuthorization(makeRequest(workspaceA, 'blob'), res, next) + + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 401 })) + }) + + it('rejects guest and readonly tokens', () => { + for (const extra of [{ guest: 'true' }, { readonly: 'true' }]) { + extractTokenMock.mockReturnValue(makeToken({ extra })) + const next = jest.fn() as unknown as NextFunction + + withAuthorization(makeRequest(workspaceA, 'blob'), res, next) + + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 401 })) + } + }) + + it('attaches a valid token to the request', () => { + const token = makeToken({}) + extractTokenMock.mockReturnValue(token) + const req = makeRequest(workspaceA, 'blob') + const next = jest.fn() as unknown as NextFunction + + withAuthorization(req, res, next) + + expect(req.token).toBe(token) + expect(next).toHaveBeenCalledWith() + }) +}) + +describe('withBlob', () => { + it('rejects a missing workspace', () => { + const next = jest.fn() as unknown as NextFunction + withBlob(makeRequest('', 'blob'), res, next) + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 400 })) + }) + + it('accepts a non-uuid workspace id when the token matches it', () => { + const workspace = 'not-a-uuid' + const next = jest.fn() as unknown as NextFunction + + withBlob(makeRequest(workspace, 'blob', makeToken({ workspace: workspace as WorkspaceUuid })), res, next) + + expect(next).toHaveBeenCalledWith() + }) + + it('rejects a missing blob name', () => { + const next = jest.fn() as unknown as NextFunction + withBlob(makeRequest(workspaceA, ''), res, next) + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 400 })) + }) + + it('rejects a token scoped to another workspace', () => { + const next = jest.fn() as unknown as NextFunction + + withBlob(makeRequest(workspaceB, 'blob', makeToken({ workspace: workspaceA })), res, next) + + expect(next).toHaveBeenCalledWith(expect.any(HttpError)) + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 401 })) + }) + + it('allows a token scoped to the requested workspace', () => { + const next = jest.fn() as unknown as NextFunction + + withBlob(makeRequest(workspaceA, 'blob', makeToken({ workspace: workspaceA })), res, next) + + expect(next).toHaveBeenCalledWith() + }) + + it('allows the system account and admins to access any workspace', () => { + for (const token of [ + makeToken({ account: systemAccountUuid, workspace: workspaceA }), + makeToken({ workspace: workspaceA, extra: { admin: 'true' } }) + ]) { + const next = jest.fn() as unknown as NextFunction + withBlob(makeRequest(workspaceB, 'blob', token), res, next) + expect(next).toHaveBeenCalledWith() + } + }) + + it('rejects requests with no token attached', () => { + const next = jest.fn() as unknown as NextFunction + + withBlob(makeRequest(workspaceA, 'blob'), res, next) + + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 401 })) + }) +}) diff --git a/pods/preview/src/middleware.ts b/pods/preview/src/middleware.ts index 0d8b0e0187..017a053a1c 100644 --- a/pods/preview/src/middleware.ts +++ b/pods/preview/src/middleware.ts @@ -66,6 +66,40 @@ export const withAuthorization = (req: RequestWithAuth, res: Response, next: Nex } } +/** + * Validates blob route params and ensures the caller's token grants access to + * the workspace taken from the URL. Must run after `withAuthorization`, which + * guarantees a token is present. + */ +export const withBlob = (req: RequestWithAuth, res: Response, next: NextFunction): void => { + try { + const workspace = req.params.workspace + const name = req.params.name + + if (workspace === undefined || workspace === '') { + throw new HttpError(400, 'Missing workspace') + } + if (name === undefined || name === '') { + throw new HttpError(400, 'Missing blob name') + } + + const token = req.token + if (token == null) { + throw new HttpError(401, 'Unauthorized') + } + + const hasWorkspaceAccess = + (token.workspace as string) === workspace || token.account === systemAccountUuid || token.extra?.admin === 'true' + if (!hasWorkspaceAccess) { + throw new HttpError(401, 'Unauthorized') + } + + next() + } catch (err: any) { + next(err) + } +} + export interface ErrorHandlerOptions { ctx: MeasureContext } diff --git a/pods/preview/src/server.ts b/pods/preview/src/server.ts index 18026a42f1..32fa7d67f0 100644 --- a/pods/preview/src/server.ts +++ b/pods/preview/src/server.ts @@ -29,7 +29,7 @@ import { pipeline } from 'stream/promises' import { createCache } from './cache' import { type Config } from './config' -import { type RequestWithAuth, errorHandler, keepAlive } from './middleware' +import { type RequestWithAuth, errorHandler, keepAlive, withAuthorization, withBlob } from './middleware' import { createPreviewService, ThumbnailParams } from './service' import { TemporaryDir } from './tempdir' @@ -174,6 +174,8 @@ export async function createServer (ctx: MeasureContext, config: Config): Promis app.get( '/metadata/:workspace/:name', + withAuthorization, + withBlob, wrapRequest(ctx, 'getMetadata', async (ctx, req, res) => { const workspace = req.params.workspace as WorkspaceUuid const name = req.params.name @@ -185,6 +187,8 @@ export async function createServer (ctx: MeasureContext, config: Config): Promis app.get( '/image/:transform/:workspace/:name', + withAuthorization, + withBlob, wrapRequest(ctx, 'getThumbnail', async (ctx, req, res) => { const workspace = req.params.workspace as WorkspaceUuid const name = req.params.name diff --git a/server-plugins/process-resources/src/index.ts b/server-plugins/process-resources/src/index.ts index 2f3ca49fbd..4791dec4d0 100644 --- a/server-plugins/process-resources/src/index.ts +++ b/server-plugins/process-resources/src/index.ts @@ -85,6 +85,7 @@ import { FieldChangedRollback, ToDoCancellRollback, ToDoCloseRollback } from './ import { Absolute, Add, + AllMatchValue, All, Append, Ceil, @@ -819,6 +820,7 @@ export default async () => ({ EmptyValue, ExecutionInitiator, ExecutionStarted, + AllMatchValue, FirstMatchValue, Filter, StringFromNumber, diff --git a/server-plugins/process-resources/src/transform.ts b/server-plugins/process-resources/src/transform.ts index 66e412efce..d863c7b53c 100644 --- a/server-plugins/process-resources/src/transform.ts +++ b/server-plugins/process-resources/src/transform.ts @@ -63,6 +63,26 @@ export async function FirstMatchValue ( } } +export async function AllMatchValue ( + value: any[], + props: Record, + control: ProcessControl +): Promise { + if (value == null) { + return + } + if (!Array.isArray(value)) return value + const { _class, ...otherProps } = props + if (_class == null) return + if (value.length === 0) return + if (typeof value[0] === 'string') { + const docs = await control.client.findAll(_class, { _id: { $in: value } }) + return matchQuery(docs, otherProps, core.class.Doc, control.client.getHierarchy(), true).map((p) => p._id) + } else if (typeof value[0] === 'object') { + return matchQuery(value, otherProps, core.class.Doc, control.client.getHierarchy(), true) + } +} + // #endregion // #region Array diff --git a/server-plugins/process/src/index.ts b/server-plugins/process/src/index.ts index b3198c791c..d5b4f8c25b 100644 --- a/server-plugins/process/src/index.ts +++ b/server-plugins/process/src/index.ts @@ -115,6 +115,7 @@ export default plugin(serverProcessId, { EmptyValue: '' as Resource, EmptyArray: '' as Resource, Filter: '' as Resource, + AllMatchValue: '' as Resource, FirstMatchValue: '' as Resource, ExecutionInitiator: '' as Resource, ExecutionStarted: '' as Resource, diff --git a/server/account/src/__tests__/operations.test.ts b/server/account/src/__tests__/operations.test.ts index 686f9c6e33..55ad76337e 100644 --- a/server/account/src/__tests__/operations.test.ts +++ b/server/account/src/__tests__/operations.test.ts @@ -53,7 +53,9 @@ import { createAccessLink, getSubscriptions, leaveWorkspace, - checkJoin + checkJoin, + mergeSpecifiedPersons, + canMergeSpecifiedPersons } from '../operations' import { accountPlugin } from '../plugin' @@ -3183,3 +3185,287 @@ describe('getSubscriptions', () => { await expect(getSubscriptions(mockCtx, mockDb, mockBranding, 'test-token', {})).rejects.toThrow(PlatformError) }) }) + +describe('merge specified persons', () => { + const mockCtx = { + error: jest.fn(), + info: jest.fn(), + warn: jest.fn() + } as unknown as MeasureContext + + const mockBranding = null + const workspaceUuid = 'caller-workspace-uuid' as WorkspaceUuid + const callerUuid = 'caller-account-uuid' as AccountUuid + const primaryPerson = 'primary-person-uuid' as PersonUuid + const secondaryPerson = 'secondary-person-uuid' as PersonUuid + const params = { primaryPerson, secondaryPerson } + + let mockDb: any + + beforeEach(() => { + jest.clearAllMocks() + jest.restoreAllMocks() + + mockDb = { + account: { + findOne: jest.fn().mockResolvedValue(null) + }, + person: { + findOne: jest.fn().mockImplementation(async ({ uuid }: { uuid: PersonUuid }) => ({ uuid })) + }, + socialId: { + find: jest.fn().mockResolvedValue([]) + }, + getWorkspaceRole: jest.fn().mockResolvedValue(null) + } + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: callerUuid, + workspace: workspaceUuid, + extra: {} + }) + }) + + // The caller maintains the workspace, and neither merged person belongs to another one. + const asWorkspaceMaintainer = (): void => { + mockDb.getWorkspaceRole.mockImplementation(async (account: AccountUuid) => + account === callerUuid ? AccountRole.Maintainer : null + ) + } + + describe('mergeSpecifiedPersons', () => { + test('should throw BadRequest for empty params', async () => { + await expect( + mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', { + primaryPerson: '' as PersonUuid, + secondaryPerson + }) + ).rejects.toThrow(PlatformError) + + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + + test('should throw Forbidden for a token without workspace', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ account: callerUuid, extra: {} }) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect(mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).rejects.toThrow( + PlatformError + ) + + expect(spy).not.toHaveBeenCalled() + // Pins the workspace guard itself rather than the role lookup that follows it. + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + + test('should throw Forbidden when caller is below Maintainer', async () => { + mockDb.getWorkspaceRole.mockResolvedValue(AccountRole.User) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect(mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).rejects.toThrow( + PlatformError + ) + + expect(spy).not.toHaveBeenCalled() + }) + + test('should throw Forbidden when the secondary person is an account of another workspace', async () => { + asWorkspaceMaintainer() + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === secondaryPerson ? { uuid } : null + ) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect(mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).rejects.toThrow( + PlatformError + ) + + expect(spy).not.toHaveBeenCalled() + }) + + test('should throw Forbidden when the primary person is an account of another workspace', async () => { + asWorkspaceMaintainer() + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === primaryPerson ? { uuid } : null + ) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect(mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).rejects.toThrow( + PlatformError + ) + + expect(spy).not.toHaveBeenCalled() + }) + + test('should merge workspace contacts without accounts for a Maintainer', async () => { + asWorkspaceMaintainer() + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(spy).toHaveBeenCalledWith(mockDb, primaryPerson, secondaryPerson) + }) + + test('should merge a contact into a member of the caller workspace', async () => { + mockDb.getWorkspaceRole.mockImplementation(async (account: AccountUuid) => + account === secondaryPerson ? null : AccountRole.Maintainer + ) + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === primaryPerson ? { uuid } : null + ) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(spy).toHaveBeenCalledWith(mockDb, primaryPerson, secondaryPerson) + }) + + test('should throw Forbidden when a login capable social id would move onto a foreign account', async () => { + // A maintainer minting a person that carries their own email and merging it into a + // co-member would hand them that member's account through password recovery. + mockDb.getWorkspaceRole.mockImplementation(async (account: AccountUuid) => + account === secondaryPerson ? null : AccountRole.Maintainer + ) + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === primaryPerson ? { uuid } : null + ) + mockDb.socialId.find.mockResolvedValue([{ _id: 'attacker-email', type: SocialIdType.EMAIL }]) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect(mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).rejects.toThrow( + PlatformError + ) + + expect(spy).not.toHaveBeenCalled() + }) + + test('should allow a login capable social id to move onto the caller own account', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: primaryPerson as unknown as AccountUuid, + workspace: workspaceUuid, + extra: {} + }) + mockDb.getWorkspaceRole.mockImplementation(async (account: AccountUuid) => + account === secondaryPerson ? null : AccountRole.Maintainer + ) + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === primaryPerson ? { uuid } : null + ) + mockDb.socialId.find.mockResolvedValue([{ _id: 'own-email', type: SocialIdType.EMAIL }]) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(spy).toHaveBeenCalledWith(mockDb, primaryPerson, secondaryPerson) + }) + + test('should throw Forbidden when merging the platform guest account', async () => { + asWorkspaceMaintainer() + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect( + mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', { + primaryPerson: readOnlyGuestAccountUuid as PersonUuid, + secondaryPerson + }) + ).rejects.toThrow(PlatformError) + + expect(spy).not.toHaveBeenCalled() + }) + + test('should merge for an allowed service token', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: systemAccountUuid, + extra: { service: 'tool' } + }) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(spy).toHaveBeenCalledWith(mockDb, primaryPerson, secondaryPerson) + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + + test('should merge for a global admin token', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: callerUuid, + extra: { admin: 'true' } + }) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(spy).toHaveBeenCalledWith(mockDb, primaryPerson, secondaryPerson) + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + }) + + describe('canMergeSpecifiedPersons', () => { + beforeEach(() => { + asWorkspaceMaintainer() + }) + + // The merge dialog awaits this predicate without a catch, so refusals must be answered, + // not thrown: a rejection leaves it spinning on a disabled Save button forever. + test('should return false without looking persons up when caller does not maintain a workspace', async () => { + mockDb.getWorkspaceRole.mockResolvedValue(null) + + expect(await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).toBe(false) + expect(mockDb.person.findOne).not.toHaveBeenCalled() + }) + + test('should return false when a person is an account of another workspace', async () => { + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === secondaryPerson ? { uuid } : null + ) + + expect(await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).toBe(false) + expect(mockDb.person.findOne).not.toHaveBeenCalled() + }) + + test('should return false for equal persons without authorizing or looking them up', async () => { + const result = await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', { + primaryPerson, + secondaryPerson: primaryPerson + }) + + expect(result).toBe(false) + expect(mockDb.person.findOne).not.toHaveBeenCalled() + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + + test('should return true for a Maintainer when secondary has no verified social ids', async () => { + const result = await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(result).toBe(true) + expect(mockDb.socialId.find).toHaveBeenCalledWith({ personUuid: secondaryPerson, verifiedOn: { $ne: null } }) + }) + + test('should return false when secondary person has verified social ids', async () => { + mockDb.socialId.find.mockResolvedValue([{ _id: 'verified-social-id' }]) + + const result = await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(result).toBe(false) + }) + + test('should allow an allowed service token', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: systemAccountUuid, + extra: { service: 'tool' } + }) + + expect(await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).toBe(true) + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + + test('should allow a global admin token', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: callerUuid, + extra: { admin: 'true' } + }) + + expect(await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).toBe(true) + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + }) +}) diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index 3f2f3d5921..6d88d46fe7 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -37,7 +37,13 @@ import { type IntegrationKind } from '@hcengineering/core' import platform, { getMetadata, PlatformError, Severity, Status, translate } from '@hcengineering/platform' -import { decodeToken, decodeTokenVerbose, generateToken, type PermissionsGrant } from '@hcengineering/server-token' +import { + decodeToken, + decodeTokenVerbose, + generateToken, + type PermissionsGrant, + type Token +} from '@hcengineering/server-token' import { isAdminEmail } from './admin' import { accountPlugin } from './plugin' @@ -2877,6 +2883,79 @@ export async function deleteAccount ( }) } +// Social ids that resolve to an account on their own, and therefore hand over the ability to +// authenticate as its owner once they are re-pointed. Password recovery and OTP login look an +// account up by social id value alone (see requestPasswordReset, loginOtp). +const loginCapableSocialTypes = [SocialIdType.EMAIL, SocialIdType.HULY] + +/** + * Merging re-points the secondary person's social ids onto the primary person, so an unrestricted + * caller could both absorb the identifiers of a person they do not own and inject their own + * identifiers into somebody else's person. Restrict it to callers with authority over both persons. + */ +async function verifyMergePersonsAuthority ( + db: AccountDB, + { account, workspace, extra }: Token, + primaryPerson: PersonUuid, + secondaryPerson: PersonUuid, + shouldThrow = true +): Promise { + // Global admins and the tool/workspace services act on behalf of the whole installation, + // the same way the account level merge (mergeSpecifiedAccounts) allows them to. + // Note this must precede the workspace check below: such tokens carry no workspace. + if (extra?.admin === 'true' || verifyAllowedServices(['tool', 'workspace'], extra, false)) { + return true + } + + const forbidden = (): boolean => { + if (shouldThrow) { + throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {})) + } + + return false + } + + // Everybody else acts within a single workspace they maintain. + if (workspace == null) { + return forbidden() + } + + if (!verifyAllowedRole(await db.getWorkspaceRole(account, workspace), AccountRole.Maintainer, extra, false)) { + return forbidden() + } + + // The platform wide accounts are not anybody's to merge. + for (const person of [primaryPerson, secondaryPerson]) { + if (person === systemAccountUuid || person === readOnlyGuestAccountUuid) { + return forbidden() + } + + if ((await db.getWorkspaceRole(person as AccountUuid, workspace)) != null) { + // A member of the caller's workspace. + continue + } + + if ((await db.account.findOne({ uuid: person as AccountUuid })) != null) { + // An account outside of the caller's workspace: no workspace maintainer may take it over. + return forbidden() + } + } + + // Both persons are in reach of the caller by now, but the primary keeps receiving the secondary's + // social ids. When the primary is somebody else's account, a login capable social id would grant + // whoever controls it access to that account, so leave those merges to the verification flows. + // Note doMergePersons only refuses *verified* secondary social ids, which does not cover this. + if (primaryPerson !== account && (await db.account.findOne({ uuid: primaryPerson as AccountUuid })) != null) { + const secondarySocialIds = await db.socialId.find({ personUuid: secondaryPerson }) + + if (secondarySocialIds.some((si) => loginCapableSocialTypes.includes(si.type))) { + return forbidden() + } + } + + return true +} + export async function canMergeSpecifiedPersons ( ctx: MeasureContext, db: AccountDB, @@ -2887,7 +2966,7 @@ export async function canMergeSpecifiedPersons ( secondaryPerson: PersonUuid } ): Promise { - decodeTokenVerbose(ctx, token) + const decodedToken = decodeTokenVerbose(ctx, token) const { primaryPerson, secondaryPerson } = params if (primaryPerson == null || primaryPerson === '' || secondaryPerson == null || secondaryPerson === '') { @@ -2899,6 +2978,12 @@ export async function canMergeSpecifiedPersons ( return false } + // This is a predicate the merge dialog polls, so an unauthorized caller is answered + // rather than thrown at. mergeSpecifiedPersons below enforces the same rules. + if (!(await verifyMergePersonsAuthority(db, decodedToken, primaryPerson, secondaryPerson, false))) { + return false + } + const primaryPersonObj = await db.person.findOne({ uuid: primaryPerson }) if (primaryPersonObj == null) { throw new PlatformError(new Status(Severity.ERROR, platform.status.PersonNotFound, { person: primaryPerson })) @@ -2928,13 +3013,15 @@ export async function mergeSpecifiedPersons ( secondaryPerson: PersonUuid } ): Promise { - decodeTokenVerbose(ctx, token) + const decodedToken = decodeTokenVerbose(ctx, token) const { primaryPerson, secondaryPerson } = params if (primaryPerson == null || primaryPerson === '' || secondaryPerson == null || secondaryPerson === '') { throw new PlatformError(new Status(Severity.ERROR, platform.status.BadRequest, {})) } + await verifyMergePersonsAuthority(db, decodedToken, primaryPerson, secondaryPerson) + await doMergePersons(db, primaryPerson, secondaryPerson) } diff --git a/server/backup/src/__tests__/checkWorkspace.spec.ts b/server/backup/src/__tests__/checkWorkspace.spec.ts new file mode 100644 index 0000000000..be98044b19 --- /dev/null +++ b/server/backup/src/__tests__/checkWorkspace.spec.ts @@ -0,0 +1,83 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { compareDomainDigest, findMissingBlobs } from '../utils' + +describe('compareDomainDigest', () => { + it('reports nothing when workspace fully matches backup', () => { + const backup = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + const workspace = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] }) + }) + + it('reports documents present in backup but absent from workspace as missing', () => { + const backup = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + const workspace = new Map([['doc1', 'hash1']]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: ['doc2'], modified: [] }) + }) + + it('reports documents with a different hash as modified, not missing', () => { + const backup = new Map([['doc1', 'hash1']]) + const workspace = new Map([['doc1', 'hash1-changed']]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: ['doc1'] }) + }) + + it('ignores documents present in workspace but not in backup', () => { + const backup = new Map([['doc1', 'hash1']]) + const workspace = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] }) + }) + + it('treats quoted and unquoted equal hashes as the same (matches restore hash trimming)', () => { + const backup = new Map([['doc1', '"hash1"']]) + const workspace = new Map([['doc1', 'hash1']]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] }) + }) + + it('returns an empty result for an empty backup digest', () => { + const workspace = new Map([['doc1', 'hash1']]) + expect(compareDomainDigest(new Map(), workspace)).toEqual({ missing: [], modified: [] }) + }) +}) + +describe('findMissingBlobs', () => { + it('returns nothing when every backup blob exists in storage', () => { + expect(findMissingBlobs(['blob1', 'blob2'], new Set(['blob1', 'blob2', 'blob3']))).toEqual([]) + }) + + it('reports backup blobs absent from storage', () => { + expect(findMissingBlobs(['blob1', 'blob2'], new Set(['blob1']))).toEqual(['blob2']) + }) + + it('reports all backup blobs when storage is empty', () => { + expect(findMissingBlobs(['blob1', 'blob2'], new Set())).toEqual(['blob1', 'blob2']) + }) + + it('returns nothing for an empty list of backup blobs', () => { + expect(findMissingBlobs([], new Set(['blob1']))).toEqual([]) + }) +}) diff --git a/server/backup/src/check.ts b/server/backup/src/check.ts new file mode 100644 index 0000000000..9bb7edccb9 --- /dev/null +++ b/server/backup/src/check.ts @@ -0,0 +1,254 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + Doc, + Domain, + DOMAIN_BLOB, + MeasureContext, + Ref, + type Blob, + type LowLevelStorage, + type WorkspaceIds +} from '@hcengineering/core' +import { BackupClientOps, createDummyStorageAdapter, type Pipeline } from '@hcengineering/server-core' +import { gunzipSync } from 'zlib' +import { BackupStorage } from './storage' +import type { BackupDocId, BackupInfo, BackupSnapshot } from './types' +import { compareDomainDigest, findMissingBlobs, isAccountDomain, loadDigest } from './utils' +export * from './storage' + +/** + * @public + */ +export interface DomainCheckResult { + domain: Domain + backupCount: number + workspaceCount: number + missing: BackupDocId[] + modified: BackupDocId[] +} + +/** + * @public + */ +export interface BlobCheckResult { + total: number + missing: Ref[] + ok: boolean +} + +/** + * @public + */ +export interface WorkspaceCheckResult { + date: number + domains: DomainCheckResult[] + blobs: BlobCheckResult + ok: boolean +} + +async function resolveSnapshots ( + storage: BackupStorage, + date: number +): Promise<{ backupInfo: BackupInfo, snapshots: BackupSnapshot[], date: number }> { + const infoFile = 'backup.json.gz' + if (!(await storage.exists(infoFile))) { + throw new Error(`${infoFile} should present to check`) + } + const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + + let snapshots = backupInfo.snapshots + if (date !== -1) { + const bk = backupInfo.snapshots.findIndex((it) => it.date === date) + if (bk === -1) { + throw new Error(`${infoFile} has no snapshot at ${date}`) + } + snapshots = backupInfo.snapshots.slice(0, bk + 1) + } else { + date = snapshots[snapshots.length - 1]?.date ?? -1 + } + return { backupInfo, snapshots, date } +} + +/** + * Checks whether all documents recorded in a backup are present, and unchanged, in the given + * workspace's document domains, and whether every backed-up blob's content exists in blob + * storage (see {@link checkWorkspaceBlobs}). + * + * This is read-only: nothing is uploaded, removed, or otherwise modified in either the workspace + * or the backup. It is meant as a diagnostic to run before trusting a backup (or after a restore) + * — to find out if the workspace is missing data the backup has, without acting on it. + * + * Account domains (person/socialId) are skipped, since they live in the account database rather + * than in the workspace's own domains and can't be checked against `pipeline.context.lowLevelStorage`. + * + * @param date optional snapshot date to check against, defaults to the latest snapshot (-1). + * @public + */ +export async function checkWorkspaceBackup ( + ctx: MeasureContext, + pipeline: Pipeline, + wsIds: WorkspaceIds, + storage: BackupStorage, + date: number = -1 +): Promise { + const resolved = await resolveSnapshots(storage, date) + const snapshots = resolved.snapshots + date = resolved.date + + ctx.info('checking workspace against backup', { workspace: wsIds.uuid, date }) + + const domains = new Set() + for (const s of snapshots) { + Object.keys(s.domains).forEach((it) => domains.add(it as Domain)) + } + + const connection = pipeline.context.lowLevelStorage as LowLevelStorage + const ops = new BackupClientOps(connection) + + const results: DomainCheckResult[] = [] + + for (const domain of domains) { + if (isAccountDomain(domain)) { + continue + } + + ctx.info('checking domain', { domain }) + const backupDigest = (await loadDigest(ctx, storage, snapshots, domain, date)) as Map, string> + + const workspaceDigest = new Map, string>() + let idx: number | undefined + try { + while (true) { + const it = await ops.loadChunk(ctx, domain, idx) + idx = it.idx + for (const { id, hash } of it.docs) { + workspaceDigest.set(id as Ref, hash) + } + if (it.finished) { + break + } + } + } finally { + if (idx !== undefined) { + await ops.closeChunk(ctx, idx) + } + } + + const { missing, modified } = compareDomainDigest(backupDigest, workspaceDigest) + + const result: DomainCheckResult = { + domain, + backupCount: backupDigest.size, + workspaceCount: workspaceDigest.size, + missing, + modified + } + results.push(result) + + if (missing.length > 0 || modified.length > 0) { + ctx.warn('backup data not fully present in workspace', { + domain, + backupCount: result.backupCount, + workspaceCount: result.workspaceCount, + missing: missing.length, + modified: modified.length, + sampleMissing: missing.slice(0, 10), + sampleModified: modified.slice(0, 10) + }) + } else { + ctx.info('domain ok', { domain, count: result.backupCount }) + } + } + + const blobs = await checkWorkspaceBlobs(ctx, pipeline, wsIds, storage, date, snapshots) + + const ok = results.every((it) => it.missing.length === 0 && it.modified.length === 0) && blobs.ok + + ctx.info('check complete', { + workspace: wsIds.uuid, + ok, + domains: results.length, + missing: results.reduce((sum, it) => sum + it.missing.length, 0), + modified: results.reduce((sum, it) => sum + it.modified.length, 0), + missingBlobs: blobs.missing.length + }) + + return { date, domains: results, blobs, ok } +} + +/** + * Checks whether every blob recorded in a backup actually has its content present in the + * workspace's blob storage (S3/minio/datalake), as opposed to just a metadata record in + * `DOMAIN_BLOB`. + * + * Read-only: only lists and stats existing blobs, never uploads or removes anything. + * + * @param date optional snapshot date to check against, defaults to the latest snapshot (-1). + * @param snapshots pre-resolved snapshots, to avoid re-reading `backup.json.gz` when called from + * {@link checkWorkspaceBackup}. If omitted, it's resolved from `storage`/`date`. + * @public + */ +export async function checkWorkspaceBlobs ( + ctx: MeasureContext, + pipeline: Pipeline, + wsIds: WorkspaceIds, + storage: BackupStorage, + date: number = -1, + snapshots?: BackupSnapshot[] +): Promise { + if (snapshots === undefined) { + const resolved = await resolveSnapshots(storage, date) + snapshots = resolved.snapshots + date = resolved.date + } + + ctx.info('checking blobs against backup', { workspace: wsIds.uuid, date }) + + const backupDigest = await loadDigest(ctx, storage, snapshots, DOMAIN_BLOB, date) + + const storageAdapter = pipeline.context.storageAdapter ?? createDummyStorageAdapter() + const existingBlobIds = new Set() + const iterator = await storageAdapter.listStream(ctx, wsIds) + try { + while (true) { + const batch = await iterator.next() + if (batch.length === 0) { + break + } + for (const b of batch) { + existingBlobIds.add(b._id) + } + } + } finally { + await iterator.close() + } + + const missing = findMissingBlobs(backupDigest.keys(), existingBlobIds) as Ref[] + const ok = missing.length === 0 + + if (ok) { + ctx.info('blobs ok', { total: backupDigest.size }) + } else { + ctx.warn('backup blobs missing from storage', { + total: backupDigest.size, + missing: missing.length, + sampleMissing: missing.slice(0, 10) + }) + } + + return { total: backupDigest.size, missing, ok } +} diff --git a/server/backup/src/index.ts b/server/backup/src/index.ts index b2cd2bf630..43ea32b91a 100644 --- a/server/backup/src/index.ts +++ b/server/backup/src/index.ts @@ -13,6 +13,7 @@ // limitations under the License. // export * from './backup' +export * from './check' export * from './restore' export * from './service' export * from './types' diff --git a/server/backup/src/utils.ts b/server/backup/src/utils.ts index 5316d82521..149c5136cc 100644 --- a/server/backup/src/utils.ts +++ b/server/backup/src/utils.ts @@ -941,6 +941,51 @@ export function doTrimHash (s: string | undefined): string | undefined { return s } +/** + * Compares a per-domain digest reconstructed from a backup with a digest read from a live + * workspace and reports the difference from the backup's point of view. + * + * - `missing` — documents present in the backup but absent from the workspace. + * - `modified` — documents present in both, but with a different content hash (the workspace + * version diverged from the backed-up one). + * + * Documents present in the workspace but not in the backup are intentionally not reported here: + * this check only answers "is everything from the backup present in the workspace", not the + * reverse. + * @public + */ +export function compareDomainDigest ( + backupDigest: Map, + workspaceDigest: Map +): { missing: BackupDocId[], modified: BackupDocId[] } { + const missing: BackupDocId[] = [] + const modified: BackupDocId[] = [] + for (const [id, hash] of backupDigest) { + const workspaceHash = workspaceDigest.get(id) + if (workspaceHash === undefined) { + missing.push(id) + } else if (doTrimHash(workspaceHash) !== doTrimHash(hash)) { + modified.push(id) + } + } + return { missing, modified } +} + +/** + * Finds blob ids that are recorded in a backup but do not exist in the workspace's blob storage + * (e.g. S3/minio/datalake), as opposed to just the blob metadata record in a domain. + * @public + */ +export function findMissingBlobs (backupBlobIds: Iterable, existingBlobIds: Set): BackupDocId[] { + const missing: BackupDocId[] = [] + for (const id of backupBlobIds) { + if (!existingBlobIds.has(id as string)) { + missing.push(id) + } + } + return missing +} + export async function loadDigest ( ctx: MeasureContext, storage: BackupStorage, diff --git a/services/github/pod-github/src/platform.ts b/services/github/pod-github/src/platform.ts index edf1929209..97455c5eb0 100644 --- a/services/github/pod-github/src/platform.ts +++ b/services/github/pod-github/src/platform.ts @@ -738,7 +738,8 @@ export class PlatformWorker { } async checkRefreshToken (ctx: MeasureContext, auth: GithubUserRecord, force: boolean = false): Promise { - if (auth.refreshToken != null && auth.expiresIn != null && auth.expiresIn < Date.now() / 1000) { + const expired = auth.expiresIn != null && auth.expiresIn < Date.now() / 1000 + if (auth.refreshToken != null && (force || expired)) { const uri = 'https://github.com/login/oauth/access_token?' + makeQuery({ diff --git a/services/github/pod-github/src/users.ts b/services/github/pod-github/src/users.ts index 3771cd52be..ccb12372cd 100644 --- a/services/github/pod-github/src/users.ts +++ b/services/github/pod-github/src/users.ts @@ -36,11 +36,12 @@ export class UserManager { } private secretToUserRecord (secret: IntegrationSecret, login: string): GithubUserRecord | undefined { + const parsed = JSON.parse(secret.secret) ?? {} // TODO: Add security return { - ...(JSON.parse(secret.secret) ?? {}), // TODO: Add security + ...parsed, account: secret.socialId, _id: login, - accounts: {} + accounts: parsed.accounts ?? {} } } diff --git a/services/github/pod-github/src/worker.ts b/services/github/pod-github/src/worker.ts index 244de40828..530f0d35b0 100644 --- a/services/github/pod-github/src/worker.ts +++ b/services/github/pod-github/src/worker.ts @@ -643,6 +643,7 @@ export class GithubWorker implements IntegrationManager { ctx.info('get octokit', { account, recordId: record._id, workspace: this.workspace.uuid }) if (!(await this.platform.checkRefreshToken(ctx, record))) { record.octokit = undefined + return undefined } if (record.octokit !== undefined) { return record.octokit diff --git a/services/rekoni/src/__tests__/tempfile.test.ts b/services/rekoni/src/__tests__/tempfile.test.ts new file mode 100644 index 0000000000..158843d9ad --- /dev/null +++ b/services/rekoni/src/__tests__/tempfile.test.ts @@ -0,0 +1,53 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { stat } from 'fs/promises' +import { withTempFile } from '../tempfile' + +async function exists (path: string): Promise { + try { + await stat(path) + return true + } catch { + return false + } +} + +describe('withTempFile', () => { + it('removes the temp dir after a successful run', async () => { + let capturedDir = '' + const result = await withTempFile('content.txt', Buffer.from('hello'), async (filePath, tempDir) => { + capturedDir = tempDir + expect(await exists(filePath)).toBe(true) + return 'ok' + }) + + expect(result).toBe('ok') + expect(await exists(capturedDir)).toBe(false) + }) + + it('removes the temp dir even when the run callback throws', async () => { + let capturedDir = '' + + await expect( + withTempFile('content.txt', Buffer.from('hello'), async (_filePath, tempDir) => { + capturedDir = tempDir + throw new Error('boom') + }) + ).rejects.toThrow('boom') + + expect(capturedDir).not.toBe('') + expect(await exists(capturedDir)).toBe(false) + }) +}) diff --git a/services/rekoni/src/extractors/doc.ts b/services/rekoni/src/extractors/doc.ts index 1df888886d..510d08adb5 100644 --- a/services/rekoni/src/extractors/doc.ts +++ b/services/rekoni/src/extractors/doc.ts @@ -1,8 +1,6 @@ import { exec } from 'child_process' -import { mkdtemp, rm, rmdir, writeFile } from 'fs/promises' import { contentType } from 'mime-types' -import { tmpdir } from 'os' -import { join } from 'path' +import { withTempFile } from '../tempfile' import { rtfExtractor } from './rtf' import { DocumentExtractor } from './types' @@ -18,36 +16,32 @@ export const docExtractor: DocumentExtractor = { }, async extract (fileName: string, type: string, data): Promise { - const tempDir = await mkdtemp(join(tmpdir(), 'rekoni-')) - const distFileName = join(tempDir, 'content.doc') - await writeFile(distFileName, data) - const text = await new Promise((resolve, reject) => { - exec( - `antiword -i 1 -f -m UTF-8 "${distFileName}"`, - { encoding: 'utf-8', cwd: tempDir }, - (error, stdout, stderr) => { - if (error != null) { - if (stderr.includes('is not a Word Document. It is probably a Rich Text Format file')) { - rtfExtractor - .extract(fileName, type, data) - .then((value) => { - resolve(value) - }) - .catch((err) => { - reject(err) - }) - return + return await withTempFile('content.doc', data, async (distFileName, tempDir) => { + return await new Promise((resolve, reject) => { + exec( + `antiword -i 1 -f -m UTF-8 "${distFileName}"`, + { encoding: 'utf-8', cwd: tempDir }, + (error, stdout, stderr) => { + if (error != null) { + if (stderr.includes('is not a Word Document. It is probably a Rich Text Format file')) { + rtfExtractor + .extract(fileName, type, data) + .then((value) => { + resolve(value) + }) + .catch((err) => { + reject(err) + }) + return + } + reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) + } else { + resolve(stdout) } - reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) - } else { - resolve(stdout) } - } - ) + ) + }) }) - await rm(distFileName) - await rmdir(tempDir) - return text } } function isType (type: string): boolean { diff --git a/services/rekoni/src/extractors/pdf.ts b/services/rekoni/src/extractors/pdf.ts index ffd55c6736..cdff4b84b5 100644 --- a/services/rekoni/src/extractors/pdf.ts +++ b/services/rekoni/src/extractors/pdf.ts @@ -1,8 +1,6 @@ import { exec } from 'child_process' -import { mkdtemp, rm, rmdir, writeFile } from 'fs/promises' import { contentType } from 'mime-types' -import { tmpdir } from 'os' -import { join } from 'path' +import { withTempFile } from '../tempfile' import { DocumentExtractor } from './types' export const pdfExtractor: DocumentExtractor = { @@ -22,21 +20,16 @@ export const pdfExtractor: DocumentExtractor = { }, async extract (fileName: string, type: string, data): Promise { - const tempDir = await mkdtemp(join(tmpdir(), 'rekoni-')) - const distFileName = join(tempDir, 'content.pdf') - await writeFile(distFileName, data) - - const text = await new Promise((resolve, reject) => { - exec(`pdftotext -layout "${distFileName}" -`, { encoding: 'utf-8', cwd: tempDir }, (error, stdout, stderr) => { - if (error != null) { - reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) - } else { - resolve(stdout) - } + return await withTempFile('content.pdf', data, async (distFileName, tempDir) => { + return await new Promise((resolve, reject) => { + exec(`pdftotext -layout "${distFileName}" -`, { encoding: 'utf-8', cwd: tempDir }, (error, stdout, stderr) => { + if (error != null) { + reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) + } else { + resolve(stdout) + } + }) }) }) - await rm(distFileName) - await rmdir(tempDir) - return text } } diff --git a/services/rekoni/src/extractors/rtf.ts b/services/rekoni/src/extractors/rtf.ts index 896d52fb77..7825a4ca4a 100644 --- a/services/rekoni/src/extractors/rtf.ts +++ b/services/rekoni/src/extractors/rtf.ts @@ -1,8 +1,6 @@ import { exec } from 'child_process' -import { mkdtemp, rm, rmdir, writeFile } from 'fs/promises' import { contentType } from 'mime-types' -import { tmpdir } from 'os' -import { join } from 'path' +import { withTempFile } from '../tempfile' import { convertString } from './html' import { DocumentExtractor } from './types' @@ -18,23 +16,23 @@ export const rtfExtractor: DocumentExtractor = { }, async extract (fileName: string, type: string, data): Promise { - const tempDir = await mkdtemp(join(tmpdir(), 'rekoni-')) - const distFileName = join(tempDir, 'content.rtf') - await writeFile(distFileName, data) - const htmlText = await new Promise((resolve, reject) => { - exec(`unrtf --nopict --html "${distFileName}"`, { encoding: 'utf-8', cwd: tempDir }, (error, stdout, stderr) => { - if (error != null) { - reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) - } else { - resolve(stdout) - } + const htmlText = await withTempFile('content.rtf', data, async (distFileName, tempDir) => { + return await new Promise((resolve, reject) => { + exec( + `unrtf --nopict --html "${distFileName}"`, + { encoding: 'utf-8', cwd: tempDir }, + (error, stdout, stderr) => { + if (error != null) { + reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) + } else { + resolve(stdout) + } + } + ) }) }) - const text = convertString(htmlText) - await rm(distFileName) - await rmdir(tempDir) - return text + return convertString(htmlText) } } function isType (type: string): boolean { diff --git a/services/rekoni/src/tempfile.ts b/services/rekoni/src/tempfile.ts new file mode 100644 index 0000000000..c7ede4840b --- /dev/null +++ b/services/rekoni/src/tempfile.ts @@ -0,0 +1,42 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { mkdtemp, rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' + +/** + * Writes `data` into a fresh temp directory under a file named `fileName`, runs `run` against it, + * and guarantees the whole temp directory is removed afterwards — whether `run` succeeds or throws. + * + * Extractors shell out to external binaries (pdftotext, antiword, unrtf) that can fail on malformed + * or unsupported input. Without a `finally`-guaranteed cleanup, a failed extraction leaks its temp + * directory (including the uploaded file content) for the lifetime of the host. + * + * @public + */ +export async function withTempFile ( + fileName: string, + data: Buffer, + run: (filePath: string, tempDir: string) => Promise +): Promise { + const tempDir = await mkdtemp(join(tmpdir(), 'rekoni-')) + try { + const filePath = join(tempDir, fileName) + await writeFile(filePath, data) + return await run(filePath, tempDir) + } finally { + await rm(tempDir, { recursive: true, force: true }) + } +}