diff --git a/dev/client-resources/src/index.ts b/dev/client-resources/src/index.ts index 5c293decbd..7dc576b9c2 100644 --- a/dev/client-resources/src/index.ts +++ b/dev/client-resources/src/index.ts @@ -30,8 +30,11 @@ export default async () => { for (const op of migrateOperations) { console.log('Migrate', op[0]) await op[1].upgrade(client, { - log (...data) { - console.log(...data) + log (msg, data) { + console.log(msg, data) + }, + error (msg, data) { + console.error(msg, data) } }) } diff --git a/dev/tool/src/cleanOrphan.ts b/dev/tool/src/cleanOrphan.ts new file mode 100644 index 0000000000..f4e80889d9 --- /dev/null +++ b/dev/tool/src/cleanOrphan.ts @@ -0,0 +1,83 @@ +import { dropWorkspace, setWorkspaceDisabled, type Workspace } from '@hcengineering/account' +import core, { AccountRole, MeasureMetricsContext, SortingOrder } from '@hcengineering/core' +import contact from '@hcengineering/model-contact' +import { getWorkspaceDB } from '@hcengineering/mongo' +import { type StorageAdapter } from '@hcengineering/server-core' +import { connect } from '@hcengineering/server-tool' +import { type Db, type MongoClient } from 'mongodb' + +export async function checkOrphanWorkspaces ( + workspaces: Workspace[], + transactorUrl: string, + productId: string, + cmd: { remove: boolean, disable: boolean }, + db: Db, + client: MongoClient, + storageAdapter: StorageAdapter, + excludes: string[] +): Promise { + for (const ws of workspaces) { + if (excludes.includes(ws.workspace) || (ws.workspaceUrl != null && excludes.includes(ws.workspaceUrl))) { + continue + } + if ((ws.accounts ?? []).length === 0) { + // Potential orhpan workspace + // Let's connect and check activity. + const connection = await connect(transactorUrl, { name: ws.workspace, productId }, undefined, { admin: 'true' }) + + const accounts = await connection.findAll(contact.class.PersonAccount, {}) + const employees = await connection.findAll(contact.mixin.Employee, {}) + let activeOwners = 0 + for (const person of employees) { + const account = accounts.find((it) => it.person === person._id) + if (account !== undefined) { + if (account.role === AccountRole.Owner && person.active) { + activeOwners++ + } + // console.log('-----------', person.name, person.active, account.email, account.role) + } + } + + // Find last transaction index: + const wspace = { name: ws.workspace, productId } + const hasBucket = await storageAdapter.exists(wspace) + const [lastTx] = await connection.findAll( + core.class.Tx, + { + objectSpace: { $ne: core.space.Model }, + createdBy: { $nin: [core.account.System, core.account.ConfigUser] }, + modifiedBy: { $ne: core.account.System } + }, + { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } + ) + + await connection.close() + const lastTxHours = Math.floor((Date.now() - (lastTx?.modifiedOn ?? 0)) / 1000 / 60 / 60) + if (((activeOwners === 0 || lastTx == null) && lastTxHours > 1000) || !hasBucket) { + const createdOn = (ws.createdOn ?? 0) !== 0 ? new Date(ws.createdOn).toDateString() : '' + console.log( + 'Found orhpan workspace', + `'${ws.workspaceName}' id: '${ws.workspace}' url:${ws.workspaceUrl} by: ${ws.createdBy ?? ''} on: '${createdOn}'`, + lastTxHours + ' hours without modifications', + hasBucket + ) + if (cmd.disable) { + await setWorkspaceDisabled(db, ws._id, true) + } + if (cmd.remove) { + await dropWorkspace(new MeasureMetricsContext('tool', {}), db, productId, ws.workspace) + const workspaceDb = getWorkspaceDB(client, { name: ws.workspace, productId }) + await workspaceDb.dropDatabase() + if (storageAdapter !== undefined && hasBucket) { + const docs = await storageAdapter.list(wspace) + await storageAdapter.remove( + wspace, + docs.map((it) => it.name) + ) + await storageAdapter?.delete(wspace) + } + } + } + } + } +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 86c3b46bf3..83ba420f95 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -26,6 +26,7 @@ import { getWorkspaceById, listAccounts, listWorkspaces, + listWorkspacesPure, listWorkspacesRaw, replacePassword, setAccountAdmin, @@ -51,7 +52,15 @@ import { MongoClient, type Db } from 'mongodb' import { clearTelegramHistory } from './telegram' import { diffWorkspace, updateField } from './workspace' -import { RateLimiter, getWorkspaceId, type AccountRole, type Data, type Tx, type Version } from '@hcengineering/core' +import { + getWorkspaceId, + MeasureMetricsContext, + RateLimiter, + type AccountRole, + type Data, + type Tx, + type Version +} from '@hcengineering/core' import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model' import { openAIConfigDefaults } from '@hcengineering/openai' import { type StorageAdapter } from '@hcengineering/server-core' @@ -66,6 +75,7 @@ import { fixSkills, optimizeModel } from './clean' +import { checkOrphanWorkspaces } from './cleanOrphan' import { changeConfiguration } from './configuration' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { openAIConfig } from './openai' @@ -84,6 +94,7 @@ export function devTool ( productId: string, extendProgram?: (prog: Command) => void ): void { + const toolCtx = new MeasureMetricsContext('tool', {}) const serverSecret = process.env.SERVER_SECRET if (serverSecret === undefined) { console.error('please provide server secret') @@ -135,7 +146,7 @@ export function devTool ( const { mongodbUri } = prepareTools() await withDatabase(mongodbUri, async (db) => { console.log(`creating account ${cmd.first as string} ${cmd.last as string} (${email})...`) - await createAcc(db, productId, email, cmd.password, cmd.first, cmd.last, true) + await createAcc(toolCtx, db, productId, email, cmd.password, cmd.first, cmd.last, true) }) }) @@ -164,7 +175,7 @@ export function devTool ( } console.log('assigning to workspace', workspaceInfo) try { - await assignWorkspace(db, productId, email, workspaceInfo.workspace) + await assignWorkspace(toolCtx, db, productId, email, workspaceInfo.workspace) } catch (err: any) { console.error(err) } @@ -215,6 +226,7 @@ export function devTool ( const { mongodbUri, txes, version, migrateOperations } = prepareTools() await withDatabase(mongodbUri, async (db) => { const { client } = await createWorkspace( + toolCtx, version, txes, migrateOperations, @@ -292,6 +304,8 @@ export function devTool ( } const withError: string[] = [] + let toProcess = workspaces.length + const st = Date.now() async function _upgradeWorkspace (ws: WorkspaceInfo): Promise { if (ws.disabled === true) { @@ -301,7 +315,18 @@ export function devTool ( const logger = cmd.console ? consoleModelLogger : new FileModelLogger(path.join(cmd.logs, `${ws.workspace}.log`)) - console.log('---UPGRADING----', ws.workspace, !cmd.console ? (logger as FileModelLogger).file : '') + + const avgTime = (Date.now() - st) / (workspaces.length - toProcess + 1) + console.log( + '---UPGRADING----', + ws.workspace, + !cmd.console ? (logger as FileModelLogger).file : '', + 'pending: ', + toProcess, + 'ETA:', + avgTime * toProcess + ) + toProcess-- try { await upgradeWorkspace( version, @@ -347,6 +372,33 @@ export function devTool ( }) }) + program + .command('remove-unused-workspaces') + .description( + 'remove unused workspaces, please pass --remove to really delete them. Without it will only mark them disabled' + ) + .option('-r|--remove [remove]', 'Force remove', false) + .option('-d|--disable [disable]', 'Force disable', false) + .option('-e|--exclude [exclude]', 'A comma separated list of workspaces to exclude', '') + .action(async (cmd: { remove: boolean, disable: boolean, exclude: string }) => { + const { mongodbUri, storageAdapter } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + const workspaces = await listWorkspacesPure(db, productId) + + // We need to update workspaces with missing workspaceUrl + await checkOrphanWorkspaces( + workspaces, + transactorUrl, + productId, + cmd, + db, + client, + storageAdapter, + cmd.exclude.split(',') + ) + }) + }) + program .command('drop-workspace ') .description('drop workspace') @@ -358,7 +410,7 @@ export function devTool ( console.log('no workspace exists') return } - await dropWorkspace(db, productId, workspace) + await dropWorkspace(toolCtx, db, productId, workspace) }) }) @@ -368,7 +420,7 @@ export function devTool ( .action(async () => { const { mongodbUri, version } = prepareTools() await withDatabase(mongodbUri, async (db) => { - const workspacesJSON = JSON.stringify(await listWorkspaces(db, productId), null, 2) + const workspacesJSON = JSON.stringify(await listWorkspaces(toolCtx, db, productId), null, 2) console.info(workspacesJSON) console.log('latest model version:', JSON.stringify(version)) @@ -392,7 +444,7 @@ export function devTool ( .action(async (email: string, cmd) => { const { mongodbUri } = prepareTools() await withDatabase(mongodbUri, async (db) => { - await dropAccount(db, productId, email) + await dropAccount(toolCtx, db, productId, email) }) }) @@ -434,26 +486,24 @@ export function devTool ( .description('dump workspace transactions and minio resources') .action(async (bucketName: string, dirName: string, workspace: string, cmd) => { const { storageAdapter } = prepareTools() - const wsId = getWorkspaceId(workspace, productId) - const storage = await createStorageBackupStorage(storageAdapter, wsId, dirName) - await backup(transactorUrl, wsId, storage) + const storage = await createStorageBackupStorage(storageAdapter, getWorkspaceId(bucketName, productId), dirName) + await backup(transactorUrl, getWorkspaceId(workspace, productId), storage) }) program - .command('backup-s3-restore , [date]') + .command('backup-s3-restore [date]') .description('dump workspace transactions and minio resources') .action(async (bucketName: string, dirName: string, workspace: string, date, cmd) => { const { storageAdapter } = prepareTools() - const wsId = getWorkspaceId(bucketName, productId) - const storage = await createStorageBackupStorage(storageAdapter, wsId, dirName) - await restore(transactorUrl, wsId, storage, parseInt(date ?? '-1')) + const storage = await createStorageBackupStorage(storageAdapter, getWorkspaceId(bucketName), dirName) + await restore(transactorUrl, getWorkspaceId(workspace, productId), storage, parseInt(date ?? '-1')) }) program .command('backup-s3-list ') .description('list snaphost ids for backup') .action(async (bucketName: string, dirName: string, cmd) => { const { storageAdapter } = prepareTools() - const wsId = getWorkspaceId(bucketName, productId) - const storage = await createStorageBackupStorage(storageAdapter, wsId, dirName) + + const storage = await createStorageBackupStorage(storageAdapter, getWorkspaceId(bucketName, productId), dirName) await backupList(storage) }) @@ -510,7 +560,7 @@ export function devTool ( process.exit(1) } - const workspaces = await listWorkspaces(db, productId) + const workspaces = await listWorkspaces(toolCtx, db, productId) for (const w of workspaces) { console.log(`clearing ${w.workspace} history:`) diff --git a/packages/model/src/utils.ts b/packages/model/src/utils.ts index 9e0478fc88..3d80bff78a 100644 --- a/packages/model/src/utils.ts +++ b/packages/model/src/utils.ts @@ -56,14 +56,18 @@ export async function createOrUpdate ( * @public */ export interface ModelLogger { - log: (...data: any[]) => void + log: (msg: string, data: any) => void + error: (msg: string, err: any) => void } /** * @public */ export const consoleModelLogger: ModelLogger = { - log (...data: any[]): void { - console.log(...data) + log (msg: string, data: any): void { + console.log(msg, data) + }, + error (msg: string, data: any): void { + console.error(msg, data) } } diff --git a/packages/query/src/index.ts b/packages/query/src/index.ts index 66101a3b4d..53c330f555 100644 --- a/packages/query/src/index.ts +++ b/packages/query/src/index.ts @@ -206,7 +206,12 @@ export class LiveQuery implements WithTx, Client { } } } - if (options?.limit === 1 && options.total !== true) { + if ( + options?.limit === 1 && + options.total !== true && + options?.sort === undefined && + options?.projection === undefined + ) { const docs = this.documentRefs.get(classKey) if (docs !== undefined) { const _docs = Array.from(docs.values()).map((it) => it.doc) diff --git a/plugins/workbench-resources/src/components/Workbench.svelte b/plugins/workbench-resources/src/components/Workbench.svelte index 980f6db2de..791998babf 100644 --- a/plugins/workbench-resources/src/components/Workbench.svelte +++ b/plugins/workbench-resources/src/components/Workbench.svelte @@ -619,7 +619,7 @@ $: modern = currentApplication?.modern ?? false -{#if employee && !employee.active} +{#if employee && !employee.active && !isAdminUser()}