From f6b1b33794af9f1308136e64efaba8b2e2a5ddf6 Mon Sep 17 00:00:00 2001 From: Alexey Zinoviev Date: Fri, 8 Aug 2025 15:43:26 +0700 Subject: [PATCH] UBERF-12509: Trusted accounts migration tool (#9652) --- dev/tool/package.json | 3 +- dev/tool/src/db.ts | 342 +++++++++++++++++- dev/tool/src/index.ts | 28 ++ server/account-service/src/index.ts | 3 + server/account-service/src/migration/utils.ts | 2 + 5 files changed, 375 insertions(+), 3 deletions(-) diff --git a/dev/tool/package.json b/dev/tool/package.json index e245ba46ec..07f566c967 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -21,7 +21,7 @@ "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool", "run-local-mongo": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4700 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:19092' node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js", - "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 STORAGE_CONFIG='datalake|http://huly.local:4030' ACCOUNT_DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 REGION_INFO='cockroach|CockroachDB' MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:19092' node --expose-gc --max-old-space-size=18000 $TOOL_OPT ./bundle/bundle.js", + "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 STORAGE_CONFIG='datalake|http://huly.local:4030' ACCOUNT_DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 REGION_INFO='cockroach|CockroachDB' MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:19092' node --expose-gc --max-old-space-size=18000 $TOOL_OPT ./bundle/bundle.js", "run-local-brk": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js", "run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js", "upgrade-mongo": "rushx run-local-mongo upgrade-workspace -- $1", @@ -60,6 +60,7 @@ "dependencies": { "@elastic/elasticsearch": "^7.17.14", "@hcengineering/account": "^0.6.0", + "@hcengineering/account-service": "^0.6.0", "@hcengineering/workspace-service": "^0.6.0", "@hcengineering/attachment": "^0.6.14", "@hcengineering/calendar": "^0.6.24", diff --git a/dev/tool/src/db.ts b/dev/tool/src/db.ts index 66edc4c541..6f3abada1f 100644 --- a/dev/tool/src/db.ts +++ b/dev/tool/src/db.ts @@ -8,7 +8,8 @@ import { findFullSocialIdBySocialKey, findPersonBySocialKey, mergeSpecifiedPersons, - mergeSpecifiedAccounts + mergeSpecifiedAccounts, + createAccount } from '@hcengineering/account' import { getFirstName, getLastName } from '@hcengineering/contact' import { @@ -23,7 +24,11 @@ import { type AccountUuid, parseSocialIdString, DOMAIN_SPACE, - AccountRole + AccountRole, + generateId, + type WorkspaceDataId, + type WorkspaceUuid, + generateUuid } from '@hcengineering/core' import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' import { @@ -39,6 +44,11 @@ import { type DBDoc } from '@hcengineering/postgres/types/utils' import { getTransactorEndpoint } from '@hcengineering/server-client' import { generateToken } from '@hcengineering/server-token' import { connect } from '@hcengineering/server-tool' +import { + type MongoAccountDB as v6MongoAccountDB, + type Account as OldAccount, + type Workspace as OldWorkspace +} from '@hcengineering/account-service' import { type MongoClient } from 'mongodb' import type postgres from 'postgres' import { type Row } from 'postgres' @@ -1138,3 +1148,331 @@ export async function ensureGlobalPersonsForLocalAccounts ( pg.close() } } + +export async function migrateTrustedV6Accounts ( + ctx: MeasureMetricsContext, + accountDB: AccountDB, + mongoDb: v6MongoAccountDB, + dryRun: boolean, + skipWorkspaces: Set +): Promise { + // Mapping between + const accountsIdToUuid: Record = {} + // Mapping between + const accountsEmailToUuid: Record = {} + // Mapping between + const workspacesIdToUuid: Record = {} + + console.log('Migrating accounts database...') + let accountsProcessed = 0 + const accountsCursor = mongoDb.account.findCursor({}) + try { + while (await accountsCursor.hasNext()) { + const account = await accountsCursor.next() + if (account == null) { + break + } + + try { + const accountUuid = await migrateAccount(account, accountDB, dryRun) + if (accountUuid == null) { + console.log('Account not migrated', account) + continue + } + accountsIdToUuid[account._id.toString()] = accountUuid + accountsEmailToUuid[account.email] = accountUuid + + accountsProcessed++ + if (accountsProcessed % 100 === 0) { + console.log('Processed accounts:', accountsProcessed) + } + } catch (err: any) { + console.log('Failed to migrate account', account._id, account.email, err) + } + } + } catch (err: any) { + console.log('Failed to migrate accounts', err) + } finally { + await accountsCursor.close() + } + + console.log('Total accounts processed:', accountsProcessed) + + let processedWorkspaces = 0 + const workspacesCursor = mongoDb.workspace.findCursor({}) + try { + while (await workspacesCursor.hasNext()) { + const workspace = await workspacesCursor.next() + if (workspace == null) { + break + } + + if ( + skipWorkspaces.has(workspace.workspace) || + (workspace.workspaceUrl != null && skipWorkspaces.has(workspace.workspaceUrl)) + ) { + console.log('Skipping workspace', workspace.workspace, workspace.workspaceUrl) + continue + } + + try { + const workspaceUuid = await migrateWorkspace( + workspace, + accountDB, + accountsIdToUuid, + accountsEmailToUuid, + dryRun + ) + + if (workspaceUuid !== undefined) { + workspacesIdToUuid[workspace.workspace] = workspaceUuid + } + processedWorkspaces++ + if (processedWorkspaces % 100 === 0) { + console.log('Processed workspaces:', processedWorkspaces) + } + } catch (err: any) { + console.log('Failed to migrate workspace', workspace.workspaceUrl, workspace.workspace, err) + } + } + } catch (err: any) { + console.log('Failed to migrate workspaces', err) + } finally { + await workspacesCursor.close() + } + + console.log('Total workspaces processed:', processedWorkspaces) + console.log('Total workspaces created/ensured:', Object.values(workspacesIdToUuid).length) + + let invitesProcessed = 0 + const invitesCursor = mongoDb.invite.findCursor({}) + try { + while (await invitesCursor.hasNext()) { + const invite = await invitesCursor.next() + if (invite == null) { + break + } + + try { + const workspaceUuid = workspacesIdToUuid[invite.workspace.name] + if (workspaceUuid === undefined) { + console.log('No workspace with id', invite.workspace.name, 'found for invite', invite._id) + continue + } + + const existing = await accountDB.invite.findOne({ migratedFrom: invite._id.toString() }) + if (existing != null) { + continue + } + + const inviteRecord = { + migratedFrom: invite._id.toString(), + workspaceUuid, + expiresOn: invite.exp, + emailPattern: invite.emailMask, + remainingUses: invite.limit, + role: invite.role ?? AccountRole.User + } + + if (!dryRun) { + await accountDB.invite.insertOne(inviteRecord) + } else { + console.log('Creating invite record', inviteRecord) + } + + invitesProcessed++ + if (invitesProcessed % 100 === 0) { + console.log('Processed invites:', invitesProcessed) + } + } catch (err: any) { + console.log('Failed to migrate invite', invite._id, err) + } + } + } catch (err: any) { + console.log('Failed to migrate invites', err) + } finally { + await invitesCursor.close() + } + + console.log('Total invites processed:', invitesProcessed) +} + +async function migrateAccount ( + account: OldAccount, + accountDB: AccountDB, + dryRun = true +): Promise { + const primaryKey: SocialKey = { + type: SocialIdType.EMAIL, + value: account.email + } + + let personUuid: PersonUuid + const verified = account.confirmed === true ? { verifiedOn: Date.now() } : {} + + const existing = await accountDB.socialId.findOne(primaryKey) + if (existing == null) { + // Create new global person + const personRecord = { + firstName: account.first, + lastName: account.last + } + + if (!dryRun) { + personUuid = await accountDB.person.insertOne(personRecord) + } else { + console.log('Creating person record', personRecord) + personUuid = generateUuid() as PersonUuid + } + + const socialIdRecord = { + ...primaryKey, + personUuid, + ...verified + } + + if (!dryRun) { + await accountDB.socialId.insertOne(socialIdRecord) + } else { + console.log('Creating social id record', socialIdRecord) + } + + if (!dryRun) { + await createAccount(accountDB, personUuid, account.confirmed, false, account.createdOn) + } else { + console.log('Creating account record', { personUuid, confirmed: account.confirmed }) + } + + if (account.hash != null && account.salt != null) { + if (!dryRun) { + await accountDB.setPassword(personUuid as AccountUuid, account.hash, account.salt) + } else { + console.log('Updating account password', { personUuid }) + } + } + } else { + personUuid = existing.personUuid + + // if there's no existing account, create a new one + const existingAcc = await accountDB.account.findOne({ uuid: personUuid as AccountUuid }) + if (existingAcc == null) { + if (!dryRun) { + await createAccount(accountDB, personUuid, account.confirmed, false, account.createdOn) + } else { + console.log('Creating account record', { personUuid, confirmed: account.confirmed }) + } + + if (account.hash != null && account.salt != null) { + if (!dryRun) { + await accountDB.setPassword(personUuid as AccountUuid, account.hash, account.salt) + } else { + console.log('Updating account password', { personUuid }) + } + } + } + } + + return personUuid as AccountUuid +} + +async function migrateWorkspace ( + workspace: OldWorkspace, + accountDB: AccountDB, + accountsIdToUuid: Record, + accountsEmailToUuid: Record, + dryRun = true +): Promise { + if (workspace.workspaceUrl == null) { + console.log('No workspace url, skipping', workspace.workspace) + return + } + + const createdBy = workspace.createdBy !== undefined ? accountsEmailToUuid[workspace.createdBy] : undefined + if (createdBy === undefined) { + console.log('No account found for workspace', workspace.workspace, 'created by', workspace.createdBy) + } + + const existingByUrl = await accountDB.workspace.findOne({ url: workspace.workspaceUrl }) + const existingByUuid = await accountDB.workspace.findOne({ uuid: workspace.uuid }) + + let workspaceUuid: WorkspaceUuid + + if (existingByUuid == null) { + let url = workspace.workspaceUrl + if (existingByUrl != null) { + // generate new url + url = `${url}-${generateId('-')}` + console.log('Generating new url', url) + } + + const workspaceRecord = { + uuid: workspace.uuid, + name: workspace.workspaceName, + url, + dataId: workspace.workspace, + branding: workspace.branding, + region: workspace.region, + createdBy, + billingAccount: createdBy, + createdOn: workspace.createdOn ?? Date.now() + } + + if (!dryRun) { + workspaceUuid = await accountDB.workspace.insertOne(workspaceRecord) + } else { + console.log('Creating workspace record', workspaceRecord) + workspaceUuid = generateUuid() as WorkspaceUuid + } + } else { + workspaceUuid = existingByUuid.uuid + } + + const existingStatus = await accountDB.workspaceStatus.findOne({ workspaceUuid }) + + if (existingStatus == null) { + const statusRecord = { + workspaceUuid, + mode: workspace.mode, + processingProgress: workspace.progress !== undefined ? Math.floor(workspace.progress) : undefined, + versionMajor: workspace.version?.major, + versionMinor: workspace.version?.minor, + versionPatch: workspace.version?.patch, + lastProcessingTime: workspace.lastProcessingTime, + lastVisit: workspace.lastVisit, + isDisabled: workspace.disabled, + processingAttempts: workspace.attempts, + processingMessage: workspace.message, + backupInfo: workspace.backupInfo + } + + if (!dryRun) { + await accountDB.workspaceStatus.insertOne(statusRecord) + } else { + console.log('Creating workspace status record', statusRecord) + } + } + + const uniqueAccounts = Array.from(new Set((workspace.accounts ?? []).map((it) => it.toString()))) + const existingMembers = new Set((await accountDB.getWorkspaceMembers(workspaceUuid)).map((mi) => mi.person)) + for (const member of uniqueAccounts) { + const accountUuid = accountsIdToUuid[member] + + if (accountUuid === undefined) { + console.log('No account found for workspace', workspace.workspace, 'member', member) + continue + } + + if (existingMembers.has(accountUuid)) { + continue + } + + if (!dryRun) { + // Actual roles are being set in workspace migration + await accountDB.assignWorkspace(accountUuid, workspaceUuid, AccountRole.Guest) + } else { + console.log('Assigning account', member, accountUuid, 'to workspace', workspaceUuid) + } + } + + return workspaceUuid +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 1921c3ab0e..82c9792049 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -51,6 +51,7 @@ import { } from '@hcengineering/server-pipeline' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' +import { getMongoAccountDB } from '@hcengineering/account-service' import { faker } from '@faker-js/faker' import { getPlatformQueue } from '@hcengineering/kafka' @@ -110,6 +111,7 @@ import { filterMergedAccountsInMembers, migrateCreatedModifiedBy, migrateMergedAccounts, + migrateTrustedV6Accounts, moveAccountDbFromMongoToPG } from './db' import { performGithubAccountMigrations } from './github' @@ -2708,6 +2710,32 @@ export function devTool ( await performIntegrationMigrations(dbUrl, cmd.region ?? null, txes) }) + program + .command('migrate-trusted-v6-accounts') + .description('Migrate trusted v6 accounts') + .option('-s|--skip [skip]', 'A command separated list of workspaces to skip', '') + .option('-d|--dry [dry]', 'Dry run', false) + .action(async (cmd: { skip: string, dry: boolean }) => { + const { dbUrl } = prepareTools() + const mongodbUri = getMongoDBUrl() + + if (mongodbUri === dbUrl) { + throw new Error('MONGO_URL and DB_URL are the same') + } + + const mongoNs = process.env.OLD_ACCOUNTS_NS + const skipWorkspaces = new Set(cmd.skip.split(',').map((it) => it.trim())) + + await withAccountDatabase(async (pgDb) => { + const [v6MongoAccountDb, closeMongoAccountDb] = await getMongoAccountDB(mongodbUri, mongoNs) + try { + await migrateTrustedV6Accounts(toolCtx, pgDb, v6MongoAccountDb, cmd.dry, skipWorkspaces) + } finally { + closeMongoAccountDb() + } + }, dbUrl) + }) + extendProgram?.(program) process.on('unhandledRejection', (reason, promise) => { diff --git a/server/account-service/src/index.ts b/server/account-service/src/index.ts index e0d955a690..024c834c5f 100644 --- a/server/account-service/src/index.ts +++ b/server/account-service/src/index.ts @@ -29,6 +29,9 @@ import Router from 'koa-router' import os from 'os' import { migrateFromOldAccounts } from './migration/migration' +export * from './migration/utils' +export * from './migration/types' + const AUTH_TOKEN_COOKIE = 'account-metadata-Token' const KEEP_ALIVE_HEADERS = { diff --git a/server/account-service/src/migration/utils.ts b/server/account-service/src/migration/utils.ts index e073437ab8..f4e697b210 100644 --- a/server/account-service/src/migration/utils.ts +++ b/server/account-service/src/migration/utils.ts @@ -16,6 +16,8 @@ import { getMongoClient } from '@hcengineering/mongo' import { MongoAccountDB } from './collections/mongo' +export { MongoAccountDB } + export async function getMongoAccountDB (uri: string, dbNs?: string): Promise<[MongoAccountDB, () => void]> { const isMongo = uri.startsWith('mongodb://')