diff --git a/.vscode/launch.json b/.vscode/launch.json index 29718f7e2e..0bd68d2913 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -312,7 +312,7 @@ "MINIO_ACCESS_KEY": "minioadmin", "MINIO_SECRET_KEY": "minioadmin", "MINIO_ENDPOINT": "localhost", - "MODEL_VERSION": "0.7.75", + "MODEL_VERSION": "0.7.110", "WS_OPERATION": "all+backup", "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin", "BACKUP_BUCKET": "dev-backups", diff --git a/models/activity/src/migration.ts b/models/activity/src/migration.ts index bbe793783b..dc13f87603 100644 --- a/models/activity/src/migration.ts +++ b/models/activity/src/migration.ts @@ -26,7 +26,6 @@ import core, { type Doc, type Domain, groupByArray, - MeasureMetricsContext, type PersonId, type Ref, type Space @@ -203,12 +202,11 @@ async function migrateActivityMarkup (client: MigrationClient): Promise { } async function migrateAccountsToSocialIds (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('activity migrateAccountsToSocialIds', {}) const socialKeyByAccount = await getSocialKeyByOldAccount(client) const socialIdBySocialKey = new Map() const socialIdByOldAccount = new Map() - ctx.info('processing activity reactions ', {}) + client.logger.log('processing activity reactions ', {}) const iterator = await client.traverse(DOMAIN_REACTION, { _class: activity.class.Reaction }) try { @@ -247,12 +245,12 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('activity migrateAccountsToSocialIds', {}) const socialKeyByAccount = await getSocialKeyByOldAccount(client) const accountUuidBySocialKey = new Map() - ctx.info('processing activity doc updates ', {}) + client.logger.log('processing activity doc updates ', {}) function getUpdatedClass (attrKey: string): string { return ['members', 'owners', 'user'].includes(attrKey) ? core.class.TypeAccountUuid : core.class.TypePersonId @@ -354,13 +351,13 @@ async function migrateAccountsInDocUpdates (client: MigrationClient): Promise { } async function migrateAccountsToSocialIds (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('calendar migrateAccountsToSocialIds', {}) const hierarchy = client.hierarchy const socialKeyByAccount = await getSocialKeyByOldAccount(client) const eventClasses = hierarchy.getDescendants(calendar.class.Event) @@ -50,19 +42,19 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('calendar migrateSocialIdsToAccountUuids', {}) const hierarchy = client.hierarchy const accountUuidBySocialKey = new Map() @@ -134,19 +125,19 @@ async function migrateSocialIdsToAccountUuids (client: MigrationClient): Promise _class: calendar.class.Calendar }) - ctx.info('processing internal calendars') + client.logger.log('processing internal calendars', {}) for (const calendar of calendars) { const id = calendar._id if (!id.endsWith('_calendar')) { - ctx.warn('Wrong calendar id format', { calendar: calendar._id }) + client.logger.error('Wrong calendar id format', { calendar: calendar._id }) continue } const socialKey = id.substring(0, id.length - 9) const accountUuid = await getAccountUuidBySocialKey(client, socialKey, accountUuidBySocialKey) if (accountUuid == null) { - ctx.warn('no account uuid for social key', { socialKey }) + client.logger.error('no account uuid for social key', { socialKey }) continue } @@ -181,7 +172,7 @@ async function migrateSocialIdsToAccountUuids (client: MigrationClient): Promise const socialKey = id.substring(0, id.length - 9) const accountUuid = await getAccountUuidBySocialKey(client, socialKey, accountUuidBySocialKey) if (accountUuid == null) { - ctx.warn('no account uuid for social key', { socialKey }) + client.logger.error('no account uuid for social key', { socialKey }) continue } @@ -198,10 +189,10 @@ async function migrateSocialIdsToAccountUuids (client: MigrationClient): Promise } processedEvents += events.length - ctx.info('...processed events', { count: processedEvents }) + client.logger.log('...processed events', { count: processedEvents }) } - ctx.info('finished processing events') + client.logger.log('finished processing events', {}) } finally { await eventsIterator.close() } diff --git a/models/contact/src/migration.ts b/models/contact/src/migration.ts index 24078f5589..e0949b94ef 100644 --- a/models/contact/src/migration.ts +++ b/models/contact/src/migration.ts @@ -22,7 +22,6 @@ import { DOMAIN_TX, generateId, type MarkupBlobRef, - MeasureMetricsContext, type PersonId, type PersonUuid, type Ref, @@ -114,8 +113,7 @@ async function getOldPersonAccounts ( } async function fillAccountUuids (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('contact fillAccountUuids', {}) - ctx.info('filling account uuids...') + client.logger.log('filling account uuids...', {}) const iterator = await client.traverse(DOMAIN_CONTACT, { _class: contact.class.Person }) try { @@ -170,8 +168,7 @@ async function fillAccountUuids (client: MigrationClient): Promise { } async function assignWorkspaceRoles (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('contact assignWorkspaceRoles', {}) - ctx.info('assigning workspace roles...') + client.logger.log('assigning workspace roles...', {}) const oldPersonAccounts = await getOldPersonAccounts(client) for (const { person, email, role } of oldPersonAccounts) { // check it's an active employee @@ -187,16 +184,15 @@ async function assignWorkspaceRoles (client: MigrationClient): Promise { try { await client.accountClient.updateWorkspaceRoleBySocialKey(buildSocialIdString(socialKey), role) } catch (err: any) { - ctx.error('Failed to update workspace role', { email, ...socialKey, role, err }) + client.logger.error('Failed to update workspace role', { email, ...socialKey, role, err }) } } - ctx.info('finished assigning workspace roles', { users: oldPersonAccounts.length }) + client.logger.log('finished assigning workspace roles', { users: oldPersonAccounts.length }) } async function assignEmployeeRoles (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('contact assignEmployeeRoles', {}) - ctx.info('assigning roles to employees...') + client.logger.log('assigning roles to employees...', {}) const wsMembers = await client.accountClient.getWorkspaceMembers() const persons = await client.traverse(DOMAIN_CONTACT, { @@ -241,13 +237,12 @@ async function assignEmployeeRoles (client: MigrationClient): Promise { } } finally { await persons.close() - ctx.info('finished assigning roles to employees...') + client.logger.log('finished assigning roles to employees...', {}) } } async function createSocialIdentities (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('createSocialIdentities', {}) - ctx.info('processing person accounts ', {}) + client.logger.log('processing person accounts ', {}) const socialIdBySocialKey = new Map() const personAccountsTxes: any[] = await client.find>(DOMAIN_MODEL_TX, { @@ -287,8 +282,7 @@ async function createSocialIdentities (client: MigrationClient): Promise { } async function migrateMergedAccounts (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('migrateMergedAccounts', {}) - ctx.info('migrating merged person accounts ', {}) + client.logger.log('migrating merged person accounts ', {}) const accountsByPerson = new Map() const personAccountsTxes: any[] = await client.find>(DOMAIN_MODEL_TX, { objectClass: 'contact:class:PersonAccount' as Ref> @@ -363,14 +357,13 @@ async function migrateMergedAccounts (client: MigrationClient): Promise { await client.accountClient.addSocialIdToPerson(primaryAccount, addTarget.type, addTarget.value, false) } } catch (err: any) { - ctx.error('Failed to merge accounts for person', { person, oldAccounts, err }) + client.logger.error('Failed to merge accounts for person', { person, oldAccounts, err }) } } } async function ensureGlobalPersonsForLocalAccounts (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('contact ensureGlobalPersonsForLocalAccounts', {}) - ctx.info('ensuring global persons for local accounts ', {}) + client.logger.log('ensuring global persons for local accounts ', {}) const personAccountsTxes: any[] = await client.find>(DOMAIN_MODEL_TX, { objectClass: 'contact:class:PersonAccount' as Ref> @@ -393,16 +386,21 @@ async function ensureGlobalPersonsForLocalAccounts (client: MigrationClient): Pr await client.accountClient.ensurePerson(socialIdKey.type, socialIdKey.value, effectiveFirstName, lastName) count++ } catch (err: any) { - ctx.error('Failed to ensure person', { socialIdKey, email: pAcc.email, firstName, lastName, effectiveFirstName }) + client.logger.error('Failed to ensure person', { + socialIdKey, + email: pAcc.email, + firstName, + lastName, + effectiveFirstName + }) console.error(err) } } - ctx.info('finished ensuring global persons for local accounts. Total persons ensured: ', { count }) + client.logger.log('finished ensuring global persons for local accounts. Total persons ensured: ', { count }) } async function createUserProfiles (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('contact createUserProfiles', {}) - ctx.info('creating user profiles for persons...') + client.logger.log('creating user profiles for persons...', {}) const persons = await client.traverse(DOMAIN_CONTACT, { _class: contact.class.Person, @@ -455,13 +453,12 @@ async function createUserProfiles (client: MigrationClient): Promise { } } finally { await persons.close() - ctx.info('finished creating user profiles for persons...') + client.logger.log('finished creating user profiles for persons...', {}) } } async function fixSocialIdCase (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('contact fixSocialIdCase', {}) - ctx.info('Fixing social id case...') + client.logger.log('Fixing social id case...', {}) const socialIds = await client.traverse(DOMAIN_CHANNEL, { _class: contact.class.SocialIdentity @@ -483,7 +480,7 @@ async function fixSocialIdCase (client: MigrationClient): Promise { } } } - ctx.info('Finished fixing social id case. Total updated:', { updated }) + client.logger.log('Finished fixing social id case. Total updated:', { updated }) } export const contactOperation: MigrateOperation = { diff --git a/models/controlled-documents/src/migration.ts b/models/controlled-documents/src/migration.ts index 1071b96e6b..871cbe5d9f 100644 --- a/models/controlled-documents/src/migration.ts +++ b/models/controlled-documents/src/migration.ts @@ -32,7 +32,6 @@ import { DOMAIN_TX, generateId, makeDocCollabId, - MeasureMetricsContext, type Ref, SortingOrder, toIdMap, @@ -280,7 +279,6 @@ async function migrateSpaceTypes (client: MigrationClient): Promise { } async function migrateDocSections (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('migrate_doc_sections', {}) const storage = client.storageAdapter const targetDocuments = await client.find(DOMAIN_DOCUMENTS, { @@ -303,7 +301,7 @@ async function migrateDocSections (client: MigrationClient): Promise { // Migrate sections headers + content try { const collabId = makeDocCollabId(document, 'content') - const ydoc = await loadCollabYdoc(ctx, storage, client.wsIds, collabId) + const ydoc = await loadCollabYdoc(client.ctx, storage, client.wsIds, collabId) if (ydoc === undefined) { // no content, ignore continue @@ -349,9 +347,9 @@ async function migrateDocSections (client: MigrationClient): Promise { } }) - await saveCollabYdoc(ctx, storage, client.wsIds, collabId, ydoc) + await saveCollabYdoc(client.ctx, storage, client.wsIds, collabId, ydoc) } catch (err) { - ctx.error('error collaborative document content migration', { error: err, document: document.title }) + client.logger.error('error collaborative document content migration', { error: err, document: document.title }) } attachmentsOps.push({ diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 0629c25f50..bb09acdf73 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -27,7 +27,6 @@ import core, { makeCollabJsonId, makeCollabYdocId, makeDocCollabId, - MeasureMetricsContext, RateLimiter, SocialIdType, systemAccountUuid, @@ -40,7 +39,6 @@ import core, { type Class, type Doc, type Domain, - type MeasureContext, type PersonId, type Ref, type Role, @@ -169,7 +167,6 @@ async function migrateStatusTransactions (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('migrate_content', {}) const storageAdapter = client.storageAdapter const hierarchy = client.hierarchy @@ -190,8 +187,8 @@ async function migrateCollaborativeContentToStorage (client: MigrationClient): P const iterator = await client.traverse(domain, query) try { - ctx.info('processing', { _class }) - await processMigrateContentFor(ctx, domain, attributes, client, storageAdapter, iterator) + client.logger.log('processing', { _class }) + await processMigrateContentFor(domain, attributes, client, storageAdapter, iterator) } finally { await iterator.close() } @@ -199,7 +196,6 @@ async function migrateCollaborativeContentToStorage (client: MigrationClient): P } async function processMigrateContentFor ( - ctx: MeasureContext, domain: Domain, attributes: AnyAttribute[], client: MigrationClient, @@ -239,9 +235,9 @@ async function processMigrateContentFor ( if (value != null && value.startsWith('{')) { try { const buffer = Buffer.from(value) - await storageAdapter.put(ctx, client.wsIds, blobId, buffer, 'application/json', buffer.length) + await storageAdapter.put(client.ctx, client.wsIds, blobId, buffer, 'application/json', buffer.length) } catch (err) { - ctx.error('failed to process document', { _class: doc._class, _id: doc._id, err }) + client.logger.error('failed to process document', { _class: doc._class, _id: doc._id, err }) } update[attributeName] = blobId @@ -263,7 +259,7 @@ async function processMigrateContentFor ( } processed += docs.length - ctx.info('...processed', { count: processed }) + client.logger.log('...processed', { count: processed }) } } @@ -303,7 +299,6 @@ export async function migrateBackupMixins (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('migrateCollaborativeDocsToJson', {}) const storageAdapter = client.storageAdapter const hierarchy = client.hierarchy @@ -324,8 +319,8 @@ async function migrateCollaborativeDocsToJson (client: MigrationClient): Promise const iterator = await client.traverse(domain, query) try { - ctx.info('processing', { _class }) - await processMigrateJsonForDomain(ctx, domain, attributes, client, storageAdapter, iterator) + client.logger.log('processing', { _class }) + await processMigrateJsonForDomain(domain, attributes, client, storageAdapter, iterator) } finally { await iterator.close() } @@ -398,13 +393,12 @@ export function getSocialKeyByOldEmail (rawEmail: string): SocialKey { * @returns */ async function migrateAccounts (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('core migrateAccounts', {}) const hierarchy = client.hierarchy const socialKeyByAccount = await getSocialKeyByOldAccount(client) const socialIdBySocialKey = new Map() const socialIdByOldAccount = new Map() - ctx.info('migrating createdBy and modifiedBy') + client.logger.log('migrating createdBy and modifiedBy', {}) function chunkArray (array: T[], chunkSize: number): T[][] { const chunks: T[][] = [] for (let i = 0; i < array.length; i += chunkSize) { @@ -414,7 +408,7 @@ async function migrateAccounts (client: MigrationClient): Promise { } for (const domain of client.hierarchy.domains()) { - ctx.info('processing domain ', { domain }) + client.logger.log('processing domain ', { domain }) const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] const groupByCreated = await client.groupBy(domain, 'createdBy', {}) const groupByModified = await client.groupBy(domain, 'modifiedBy', {}) @@ -459,7 +453,7 @@ async function migrateAccounts (client: MigrationClient): Promise { if (operations.length > 0) { const operationsChunks = chunkArray(operations, 40) - ctx.info('chunks to process ', { total: operationsChunks.length }) + client.logger.log('chunks to process ', { total: operationsChunks.length }) let processed = 0 for (const operationsChunk of operationsChunks) { if (operationsChunk.length === 0) continue @@ -467,15 +461,15 @@ async function migrateAccounts (client: MigrationClient): Promise { await client.bulk(domain, operationsChunk) processed++ if (operationsChunks.length > 1) { - ctx.info('processed chunk', { processed, of: operationsChunks.length }) + client.logger.log('processed chunk', { processed, of: operationsChunks.length }) } } } else { - ctx.info('no user accounts to migrate') + client.logger.log('no user accounts to migrate', {}) } } - ctx.info('finished migrating createdBy and modifiedBy') + client.logger.log('finished migrating createdBy and modifiedBy', {}) const spaceTypes = client.model.findAllSync(core.class.SpaceType, {}) const spaceTypesById = toIdMap(spaceTypes) @@ -493,7 +487,7 @@ async function migrateAccounts (client: MigrationClient): Promise { const accountUuidBySocialKey = new Map() - ctx.info('processing spaces members, owners and roles assignment', {}) + client.logger.log('processing spaces members, owners and roles assignment', {}) let processedSpaces = 0 const spacesIterator = await client.traverse(DOMAIN_SPACE, {}) @@ -563,15 +557,15 @@ async function migrateAccounts (client: MigrationClient): Promise { } processedSpaces += spaces.length - ctx.info('...spaces processed', { count: processedSpaces }) + client.logger.log('...spaces processed', { count: processedSpaces }) } - ctx.info('finished processing spaces members, owners and roles assignment', { processedSpaces }) + client.logger.log('finished processing spaces members, owners and roles assignment', { processedSpaces }) } finally { await spacesIterator.close() } - ctx.info('processing space types members', {}) + client.logger.log('processing space types members', {}) let updatedSpaceTypes = 0 for (const spaceType of spaceTypes) { if (spaceType.members === undefined || spaceType.members.length === 0) continue @@ -601,7 +595,10 @@ async function migrateAccounts (client: MigrationClient): Promise { await client.create(DOMAIN_MODEL_TX, tx) updatedSpaceTypes++ } - ctx.info('finished processing space types members', { totalSpaceTypes: spaceTypes.length, updatedSpaceTypes }) + client.logger.log('finished processing space types members', { + totalSpaceTypes: spaceTypes.length, + updatedSpaceTypes + }) } export async function getAccountUuidBySocialKey ( @@ -746,7 +743,6 @@ export async function getUniqueAccountsFromOldAccounts ( } async function processMigrateJsonForDomain ( - ctx: MeasureContext, domain: Domain, attributes: AnyAttribute[], client: MigrationClient, @@ -767,7 +763,7 @@ async function processMigrateJsonForDomain ( for (const doc of docs) { await rateLimiter.add(async () => { - const update = await processMigrateJsonForDoc(ctx, doc, attributes, client, storageAdapter) + const update = await processMigrateJsonForDoc(doc, attributes, client, storageAdapter) if (Object.keys(update).length > 0) { operations.push({ filter: { _id: doc._id }, update }) } @@ -781,12 +777,11 @@ async function processMigrateJsonForDomain ( } processed += docs.length - ctx.info('...processed', { count: processed }) + client.logger.log('...processed', { count: processed }) } } async function processMigrateJsonForDoc ( - ctx: MeasureContext, doc: Doc, attributes: AnyAttribute[], client: MigrationClient, @@ -813,7 +808,7 @@ async function processMigrateJsonForDoc ( if (value.startsWith('{')) { // For some reason we have documents that are already markups const jsonId = await retry(5, async () => { - return await saveCollabJson(ctx, storageAdapter, wsIds, collabId, value) + return await saveCollabJson(client.ctx, storageAdapter, wsIds, collabId, value) }) update[attributeName] = jsonId @@ -835,17 +830,22 @@ async function processMigrateJsonForDoc ( const ydocId = makeCollabYdocId(collabId) if (ydocId !== currentYdocId) { await retry(5, async () => { - const stat = await storageAdapter.stat(ctx, wsIds, currentYdocId) + const stat = await storageAdapter.stat(client.ctx, wsIds, currentYdocId) if (stat !== undefined) { - const data = await storageAdapter.read(ctx, wsIds, currentYdocId) + const data = await storageAdapter.read(client.ctx, wsIds, currentYdocId) const buffer = Buffer.concat(data as any) - await storageAdapter.put(ctx, wsIds, ydocId, buffer, 'application/ydoc', buffer.length) + await storageAdapter.put(client.ctx, wsIds, ydocId, buffer, 'application/ydoc', buffer.length) } }) } } catch (err) { const error = err instanceof Error ? err.message : String(err) - ctx.warn('failed to process collaborative doc', { workspace: wsIds.uuid, collabId, currentYdocId, error }) + client.logger.error('failed to process collaborative doc', { + workspace: wsIds.uuid, + collabId, + currentYdocId, + error + }) } const unset = update.$unset ?? {} diff --git a/models/document/src/migration.ts b/models/document/src/migration.ts index 091b18ae1d..caa20ec578 100644 --- a/models/document/src/migration.ts +++ b/models/document/src/migration.ts @@ -17,7 +17,6 @@ import { DOMAIN_MODEL_TX, DOMAIN_TX, makeDocCollabId, - MeasureMetricsContext, type Ref, SortingOrder, type Class, @@ -184,7 +183,6 @@ async function renameFields (client: MigrationClient): Promise { } async function renameFieldsRevert (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('renameFieldsRevert', {}) const storage = client.storageAdapter type ExDocument = Document & { @@ -209,7 +207,7 @@ async function renameFieldsRevert (client: MigrationClient): Promise { try { const collabId = makeDocCollabId(document, 'content') - const ydoc = await loadCollabYdoc(ctx, storage, client.wsIds, collabId) + const ydoc = await loadCollabYdoc(client.ctx, storage, client.wsIds, collabId) if (ydoc === undefined) { continue } @@ -220,9 +218,9 @@ async function renameFieldsRevert (client: MigrationClient): Promise { yDocCopyXmlField(ydoc, 'description', 'content') - await saveCollabYdoc(ctx, storage, client.wsIds, collabId, ydoc) + await saveCollabYdoc(client.ctx, storage, client.wsIds, collabId, ydoc) } catch (err) { - ctx.error('error document content migration', { error: err, document: document.title }) + client.logger.error('error document content migration', { error: err, document: document.title }) } } @@ -245,7 +243,6 @@ async function renameFieldsRevert (client: MigrationClient): Promise { } async function restoreContentField (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('restoreContentField', {}) const storage = client.storageAdapter const documents = await client.find(DOMAIN_DOCUMENT, { @@ -256,9 +253,9 @@ async function restoreContentField (client: MigrationClient): Promise { for (const document of documents) { try { const collabId = makeDocCollabId(document, 'content') - const ydoc = await loadCollabYdoc(ctx, storage, client.wsIds, collabId) + const ydoc = await loadCollabYdoc(client.ctx, storage, client.wsIds, collabId) if (ydoc === undefined) { - ctx.error('document content not found', { document: document.title }) + client.logger.error('document content not found', { document: document.title }) continue } @@ -270,13 +267,13 @@ async function restoreContentField (client: MigrationClient): Promise { if (ydoc.share.has('')) { yDocCopyXmlField(ydoc, '', 'content') if (ydoc.share.has('content')) { - await saveCollabYdoc(ctx, storage, client.wsIds, collabId, ydoc) + await saveCollabYdoc(client.ctx, storage, client.wsIds, collabId, ydoc) } else { - ctx.error('document content still not found', { document: document.title }) + client.logger.error('document content still not found', { document: document.title }) } } } catch (err) { - ctx.error('error document content migration', { error: err, document: document.title }) + client.logger.error('error document content migration', { error: err, document: document.title }) } } } @@ -292,10 +289,9 @@ async function migrateRanks (client: MigrationClient): Promise { } async function migrateAccountsToSocialIds (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('document migrateAccountsToSocialIds', {}) const socialKeyByAccount = await getSocialKeyByOldAccount(client) - ctx.info('processing document lockedBy ', {}) + client.logger.log('processing document lockedBy ', {}) const iterator = await client.traverse(DOMAIN_DOCUMENT, { _class: document.class.Document }) try { @@ -328,19 +324,18 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('document migrateSocialIdsToGlobalAccounts', {}) const accountUuidBySocialKey = new Map() - ctx.info('processing document lockedBy ', {}) + client.logger.log('processing document lockedBy ', {}) const iterator = await client.traverse(DOMAIN_DOCUMENT, { _class: document.class.Document }) try { @@ -375,12 +370,12 @@ async function migrateSocialIdsToGlobalAccounts (client: MigrationClient): Promi } processed += docs.length - ctx.info('...processed', { count: processed }) + client.logger.log('...processed', { count: processed }) } } finally { await iterator.close() } - ctx.info('finished processing document lockedBy ', {}) + client.logger.log('finished processing document lockedBy ', {}) } async function removeOldClasses (client: MigrationClient): Promise { diff --git a/models/notification/src/migration.ts b/models/notification/src/migration.ts index 85afd06867..3c7bc95fd3 100644 --- a/models/notification/src/migration.ts +++ b/models/notification/src/migration.ts @@ -17,7 +17,6 @@ import chunter from '@hcengineering/chunter' import contact, { type PersonSpace } from '@hcengineering/contact' import core, { DOMAIN_TX, - MeasureMetricsContext, type PersonId, type Class, type Doc, @@ -247,17 +246,16 @@ export async function migrateDuplicateContexts (client: MigrationClient): Promis * @returns */ async function migrateAccounts (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('notification migrateAccounts', {}) const hierarchy = client.hierarchy const socialKeyByAccount = await getSocialKeyByOldAccount(client) const socialIdBySocialKey = new Map() const socialIdByOldAccount = new Map() const accountUuidByOldAccount = new Map() - ctx.info('processing collaborators ', {}) + client.logger.log('processing collaborators ', {}) for (const domain of client.hierarchy.domains()) { if (['tx'].includes(domain)) continue - ctx.info('processing domain ', { domain }) + client.logger.log('processing domain ', { domain }) let processed = 0 const iterator = await client.traverse(domain, {}) @@ -298,17 +296,17 @@ async function migrateAccounts (client: MigrationClient): Promise { } processed += docs.length - ctx.info('...processed', { count: processed }) + client.logger.log('...processed', { count: processed }) } - ctx.info('finished processing domain ', { domain, processed }) + client.logger.log('finished processing domain ', { domain, processed }) } finally { await iterator.close() } } - ctx.info('finished processing collaborators ', {}) + client.logger.log('finished processing collaborators ', {}) - ctx.info('processing notifications fields ', {}) + client.logger.log('processing notifications fields ', {}) function chunkArray (array: T[], chunkSize: number): T[][] { const chunks: T[][] = [] for (let i = 0; i < array.length; i += chunkSize) { @@ -387,16 +385,16 @@ async function migrateAccounts (client: MigrationClient): Promise { await client.bulk(DOMAIN_NOTIFICATION, operationsChunk) processed++ if (operationsChunks.length > 1) { - ctx.info('processed chunk', { processed, of: operationsChunks.length }) + client.logger.log('processed chunk', { processed, of: operationsChunks.length }) } } } else { - ctx.info('no user accounts to migrate') + client.logger.log('no user accounts to migrate', {}) } - ctx.info('finished processing notifications fields ', {}) + client.logger.log('finished processing notifications fields ', {}) - ctx.info('processing doc notify contexts ', {}) + client.logger.log('processing doc notify contexts ', {}) const dncIterator = await client.traverse(DOMAIN_DOC_NOTIFY, { _class: notification.class.DocNotifyContext }) @@ -432,14 +430,14 @@ async function migrateAccounts (client: MigrationClient): Promise { } processed += docs.length - ctx.info('...processed', { count: processed }) + client.logger.log('...processed', { count: processed }) } } finally { await dncIterator.close() } - ctx.info('finished processing doc notify contexts ', {}) + client.logger.log('finished processing doc notify contexts ', {}) - ctx.info('processing push subscriptions ', {}) + client.logger.log('processing push subscriptions ', {}) const psIterator = await client.traverse(DOMAIN_USER_NOTIFY, { _class: notification.class.PushSubscription }) @@ -475,12 +473,12 @@ async function migrateAccounts (client: MigrationClient): Promise { } processed += docs.length - ctx.info('...processed', { count: processed }) + client.logger.log('...processed', { count: processed }) } } finally { await psIterator.close() } - ctx.info('finished processing push subscriptions ', {}) + client.logger.log('finished processing push subscriptions ', {}) } export async function migrateSettings (client: MigrationClient): Promise { diff --git a/models/setting/src/migration.ts b/models/setting/src/migration.ts index 6b6ff9bfe3..3e23216f39 100644 --- a/models/setting/src/migration.ts +++ b/models/setting/src/migration.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import core, { type AccountUuid, MeasureMetricsContext, type Ref, type Space } from '@hcengineering/core' +import core, { type AccountUuid, type Ref, type Space } from '@hcengineering/core' import { migrateSpace, type MigrateUpdate, @@ -35,11 +35,10 @@ import { DOMAIN_SETTING } from '.' * @returns */ async function migrateAccounts (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('setting migrateAccounts', {}) const socialKeyByAccount = await getSocialKeyByOldAccount(client) const accountUuidByOldAccount = new Map() - ctx.info('processing setting integration shared ', {}) + client.logger.log('processing setting integration shared ', {}) const iterator = await client.traverse(DOMAIN_SETTING, { _class: setting.class.Integration }) try { @@ -77,12 +76,12 @@ async function migrateAccounts (client: MigrationClient): Promise { } processed += docs.length - ctx.info('...processed', { count: processed }) + client.logger.log('...processed', { count: processed }) } } finally { await iterator.close() } - ctx.info('finished processing setting integration shared ', {}) + client.logger.log('finished processing setting integration shared ', {}) } export const settingOperation: MigrateOperation = { diff --git a/models/view/src/migration.ts b/models/view/src/migration.ts index fc6046fb90..49fafa760f 100644 --- a/models/view/src/migration.ts +++ b/models/view/src/migration.ts @@ -24,7 +24,7 @@ import { import { DOMAIN_PREFERENCE } from '@hcengineering/preference' import view, { type Filter, type FilteredView, type ViewletPreference, viewId } from '@hcengineering/view' import { getSocialIdFromOldAccount, getSocialKeyByOldAccount, getUniqueAccounts } from '@hcengineering/model-core' -import core, { type AccountUuid, MeasureMetricsContext, type PersonId } from '@hcengineering/core' +import core, { type AccountUuid, type PersonId } from '@hcengineering/core' import { DOMAIN_VIEW } from '.' @@ -82,10 +82,9 @@ async function removeDoneStateFilter (client: MigrationClient): Promise { } async function migrateAccountsToSocialIds (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('view migrateAccountsToSocialIds', {}) const socialKeyByAccount = await getSocialKeyByOldAccount(client) - ctx.info('processing view filtered view users ', {}) + client.logger.log('processing view filtered view users ', {}) const iterator = await client.traverse(DOMAIN_VIEW, { _class: view.class.FilteredView }) try { @@ -118,19 +117,18 @@ async function migrateAccountsToSocialIds (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('view migrateSocialIdsToGlobalAccounts', {}) const accountUuidBySocialKey = new Map() - ctx.info('processing view filtered view users ', {}) + client.logger.log('processing view filtered view users ', {}) const iterator = await client.traverse(DOMAIN_VIEW, { _class: view.class.FilteredView }) try { @@ -163,22 +161,21 @@ async function migrateSocialIdsToGlobalAccounts (client: MigrationClient): Promi } processed += docs.length - ctx.info('...processed', { count: processed }) + client.logger.log('...processed', { count: processed }) } } finally { await iterator.close() } - ctx.info('finished processing view filtered view users ', {}) + client.logger.log('finished processing view filtered view users ', {}) } async function migrateAccsInSavedFilters (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('view migrateAccsInSavedFilters', {}) const hierarchy = client.hierarchy const socialKeyByAccount = await getSocialKeyByOldAccount(client) const socialIdBySocialKey = new Map() const socialIdByOldAccount = new Map() - ctx.info('processing view filtered view accounts in filters ', {}) + client.logger.log('processing view filtered view accounts in filters ', {}) const affectedViews = await client.find(DOMAIN_VIEW, { _class: view.class.FilteredView, filters: { $regex: '%core:class:Account%' } @@ -243,7 +240,7 @@ async function migrateAccsInSavedFilters (client: MigrationClient): Promise { } async function migrateTabsToSocialIds (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('workbench migrateTabsToSocialIds', {}) - ctx.info('migrating workbench tabs to social ids...') + client.logger.log('migrating workbench tabs to social ids...', {}) const socialKeyByAccount = await getSocialKeyByOldAccount(client) const tabs = await client.find(DOMAIN_PREFERENCE, { _class: workbench.class.WorkbenchTab }) for (const tab of tabs) { @@ -40,12 +39,11 @@ async function migrateTabsToSocialIds (client: MigrationClient): Promise { await client.update(DOMAIN_PREFERENCE, { _id: tab._id }, { attachedTo: newAttachedTo }) } } - ctx.info('migrating workbench tabs to social ids completed...') + client.logger.log('migrating workbench tabs to social ids completed...', {}) } async function migrateSocialIdsToGlobalAccounts (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('workbench migrateSocialIdsToGlobalAccounts', {}) - ctx.info('migrating workbench tabs to global accounts...') + client.logger.log('migrating workbench tabs to global accounts...', {}) const accountUuidBySocialKey = new Map() const tabs = await client.find(DOMAIN_PREFERENCE, { _class: workbench.class.WorkbenchTab }) @@ -55,7 +53,7 @@ async function migrateSocialIdsToGlobalAccounts (client: MigrationClient): Promi await client.update(DOMAIN_PREFERENCE, { _id: tab._id }, { attachedTo: newAttachedTo }) } } - ctx.info('migrating workbench tabs to global accounts completed...') + client.logger.log('migrating workbench tabs to global accounts completed...', {}) } export const workbenchOperation: MigrateOperation = { diff --git a/packages/core/src/measurements/context.ts b/packages/core/src/measurements/context.ts index 8d5e7e4d53..7827365af1 100644 --- a/packages/core/src/measurements/context.ts +++ b/packages/core/src/measurements/context.ts @@ -191,6 +191,10 @@ export class MeasureMetricsContext implements MeasureContext { end (): void { this.done() } + + getParams (): ParamsType { + return this.params + } } export class NoMetricsContext implements MeasureContext { @@ -260,6 +264,10 @@ export class NoMetricsContext implements MeasureContext { } end (): void {} + + getParams (): ParamsType { + return {} + } } /** diff --git a/packages/core/src/measurements/types.ts b/packages/core/src/measurements/types.ts index 85c7cceed3..52925ce27a 100644 --- a/packages/core/src/measurements/types.ts +++ b/packages/core/src/measurements/types.ts @@ -100,6 +100,7 @@ export interface MeasureContext { logger: MeasureLogger parent?: MeasureContext + getParams: () => ParamsType measure: (name: string, value: number, override?: boolean) => void diff --git a/packages/model/src/migration.ts b/packages/model/src/migration.ts index 490358e3c2..ae786ed733 100644 --- a/packages/model/src/migration.ts +++ b/packages/model/src/migration.ts @@ -11,6 +11,7 @@ import core, { Domain, FindOptions, Hierarchy, + MeasureContext, MigrationState, ModelDb, ObjQueryType, @@ -118,6 +119,8 @@ export interface MigrationClient { wsIds: WorkspaceIds reindex: (domain: Domain, classes: Ref>[]) => Promise + readonly logger: ModelLogger + readonly ctx: MeasureContext } /** @@ -174,10 +177,10 @@ export async function tryMigrate ( if (states.has(migration.state)) continue if (migration.mode == null || migration.mode === mode) { try { - console.log('running migration', plugin, migration.state) + client.logger.log('running migration', { plugin, state: migration.state }) await migration.func(client, mode) } catch (err: any) { - console.error(err) + client.logger.error('Failed to run migration', { plugin, state: migration.state, err }) Analytics.handleError(err) continue } diff --git a/pods/server/src/server.ts b/pods/server/src/server.ts index 42b802f8cc..3fb68c94ff 100644 --- a/pods/server/src/server.ts +++ b/pods/server/src/server.ts @@ -140,7 +140,7 @@ export function start ( ctx.newChild('💬 communication api', {}), workspace.uuid, dbUrl, - broadcastSessions + broadcastSessions as any // FIXME when communication will be inside the repo ) } diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index f29903b57b..3a9e00fea2 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -268,6 +268,7 @@ export async function upgradeModel ( const { hierarchy, modelDb, model } = await buildModel(ctx, newModel) const { migrateClient: preMigrateClient } = await prepareMigrationClient( + ctx, pipeline, hierarchy, modelDb, @@ -302,6 +303,7 @@ export async function upgradeModel ( }) const { migrateClient, migrateState } = await prepareMigrationClient( + ctx, pipeline, hierarchy, modelDb, @@ -390,6 +392,7 @@ export async function upgradeModel ( } async function prepareMigrationClient ( + ctx: MeasureContext, pipeline: Pipeline, hierarchy: Hierarchy, model: ModelDb, @@ -410,7 +413,8 @@ async function prepareMigrationClient ( storageAdapter, accountClient, wsIds, - queue + queue, + ctx ) const states = await migrateClient.find(DOMAIN_MIGRATION, { _class: core.class.MigrationState }) const sts = Array.from(groupByArray(states, (it) => it.plugin).entries()) diff --git a/server/tool/src/upgrade.ts b/server/tool/src/upgrade.ts index 66fbbfa506..f059f5b402 100644 --- a/server/tool/src/upgrade.ts +++ b/server/tool/src/upgrade.ts @@ -36,13 +36,14 @@ export class MigrateClientImpl implements MigrationClient { readonly storageAdapter: StorageAdapter, readonly accountClient: AccountClient, readonly wsIds: WorkspaceIds, - readonly queue: PlatformQueueProducer + readonly queue: PlatformQueueProducer, + ctx?: MeasureContext ) { if (this.pipeline.context.lowLevelStorage === undefined) { throw new Error('lowLevelStorage is not defined') } this.lowLevel = this.pipeline.context.lowLevelStorage - this.ctx = new MeasureMetricsContext('migrateClient', {}) + this.ctx = ctx ?? new MeasureMetricsContext('migrateClient', {}) } migrateState = new Map>() diff --git a/server/workspace-service/src/service.ts b/server/workspace-service/src/service.ts index c28ca01dc9..9dff8b7def 100644 --- a/server/workspace-service/src/service.ts +++ b/server/workspace-service/src/service.ts @@ -19,6 +19,7 @@ import { isArchivingMode, isMigrationMode, isRestoringMode, + MeasureMetricsContext, systemAccountUuid, type BrandingMap, type Data, @@ -38,6 +39,7 @@ import { import { generateToken } from '@hcengineering/server-token' import { FileModelLogger, prepareTools } from '@hcengineering/server-tool' import path from 'path' +import { randomUUID } from 'crypto' import { Analytics } from '@hcengineering/analytics' import { @@ -188,14 +190,14 @@ export class WorkspaceWorker { await this.doSleep(ctx, opt) } else { void this.exec(async () => { - await ctx - .with('workspaceOperation', { mode: workspace.mode }, (ctx) => - this.doWorkspaceOperation(ctx, workspace, opt) - ) - .catch((err) => { - Analytics.handleError(err) - ctx.error('error', { err }) - }) + const job = randomUUID().slice(-8) + const opContext = new MeasureMetricsContext(`ws op with job ${job}`, { job }) + try { + await this.doWorkspaceOperation(opContext, workspace, opt) + } catch (err: any) { + Analytics.handleError(err) + ctx.error('error', { err }) + } }) } } @@ -234,6 +236,7 @@ export class WorkspaceWorker { ctx.info('---CREATING----', { workspace: ws.uuid, + mode: ws.mode, version: this.version, region: this.region }) @@ -339,6 +342,7 @@ export class WorkspaceWorker { ctx.info('---UPGRADING----', { workspace: ws.uuid, + mode: ws.mode, workspaceVersion, requestedVersion: this.version, region: this.region diff --git a/server/workspace-service/src/ws-operations.ts b/server/workspace-service/src/ws-operations.ts index 7b13b87c86..e560bee2a0 100644 --- a/server/workspace-service/src/ws-operations.ts +++ b/server/workspace-service/src/ws-operations.ts @@ -50,7 +50,7 @@ export async function createWorkspace ( ) => Promise, external: boolean = false ): Promise { - const childLogger = ctx.newChild('createWorkspace', {}, {}) + const childLogger = ctx.newChild('createWorkspace', ctx.getParams(), {}) const ctxModellogger: ModelLogger = { log: (msg, data) => { childLogger.info(msg, data) diff --git a/services/gmail/pod-gmail/src/__tests__/attachments.test.ts b/services/gmail/pod-gmail/src/__tests__/attachments.test.ts index ce158d2620..34983df25d 100644 --- a/services/gmail/pod-gmail/src/__tests__/attachments.test.ts +++ b/services/gmail/pod-gmail/src/__tests__/attachments.test.ts @@ -40,7 +40,8 @@ describe('AttachmentHandler', () => { error: jest.fn(), info: jest.fn(), warn: jest.fn(), - end: jest.fn() + end: jest.fn(), + getParams: jest.fn() } const mockWorkspaceLoginInfo: WorkspaceLoginInfo = { endpoint: 'wss://test-endpoint.com',