From 3353c531bd9982833427b3ec3e5247479e8add32 Mon Sep 17 00:00:00 2001 From: Artyom Savchenko Date: Sun, 5 Jul 2026 08:01:30 +0700 Subject: [PATCH] Fix accounts restore error (#10955) Signed-off-by: Artyom Savchenko --- .../src/__tests__/restoreSocialIds.spec.ts | 100 +++++++++++++++++ server/backup/src/restore.ts | 102 +++++++++++++----- 2 files changed, 174 insertions(+), 28 deletions(-) create mode 100644 server/backup/src/__tests__/restoreSocialIds.spec.ts diff --git a/server/backup/src/__tests__/restoreSocialIds.spec.ts b/server/backup/src/__tests__/restoreSocialIds.spec.ts new file mode 100644 index 0000000000..fb1c37b30d --- /dev/null +++ b/server/backup/src/__tests__/restoreSocialIds.spec.ts @@ -0,0 +1,100 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import type { AccountDB, SocialId } from '@hcengineering/account' +import { MeasureMetricsContext } from '@hcengineering/core' + +import { restoreSocialIds } from '../restore' + +interface MockDb { + accountDb: AccountDB + inserted: any[] +} + +function createMockAccountDb (existingPersonUuids: string[], existingSocialIdKeys: string[] = []): MockDb { + const inserted: any[] = [] + const accountDb = { + person: { + find: jest.fn(async (query: any) => + (query.uuid.$in as string[]).filter((uuid) => existingPersonUuids.includes(uuid)).map((uuid) => ({ uuid })) + ) + }, + socialId: { + find: jest.fn(async (query: any) => + (query.key.$in as string[]).filter((key) => existingSocialIdKeys.includes(key)).map((key) => ({ key })) + ), + insertMany: jest.fn(async (docs: any[]) => { + inserted.push(...docs) + return docs.map((it) => it._id) + }) + } + } as unknown as AccountDB + return { accountDb, inserted } +} + +function socialId (id: string, key: string, personUuid: string): SocialId { + return { + _id: id, + key, + personUuid, + type: 'email', + value: key + } as unknown as SocialId +} + +const ctx = new MeasureMetricsContext('test', {}) + +describe('restoreSocialIds', () => { + it('skips social ids referencing missing persons and inserts the rest', async () => { + const { accountDb, inserted } = createMockAccountDb(['person-1']) + + await restoreSocialIds(ctx, accountDb, [ + socialId('sid-1', 'email:a@b.c', 'person-1'), + socialId('sid-2', 'email:orphan@b.c', 'person-missing') + ]) + + expect(inserted).toHaveLength(1) + expect(inserted[0]._id).toBe('sid-1') + }) + + it('does not insert social ids which already exist', async () => { + const { accountDb, inserted } = createMockAccountDb(['person-1'], ['email:a@b.c']) + + await restoreSocialIds(ctx, accountDb, [socialId('sid-1', 'email:a@b.c', 'person-1')]) + + expect(inserted).toHaveLength(0) + expect((accountDb.socialId.insertMany as jest.Mock).mock.calls).toHaveLength(0) + }) + + it('does not call insertMany when all social ids are orphaned', async () => { + const { accountDb, inserted } = createMockAccountDb([]) + + await restoreSocialIds(ctx, accountDb, [socialId('sid-1', 'email:a@b.c', 'person-missing')]) + + expect(inserted).toHaveLength(0) + expect((accountDb.socialId.insertMany as jest.Mock).mock.calls).toHaveLength(0) + }) + + it('strips key and %hash% from inserted records', async () => { + const { accountDb, inserted } = createMockAccountDb(['person-1']) + const sid = { ...socialId('sid-1', 'email:a@b.c', 'person-1'), '%hash%': 'abc' } + + await restoreSocialIds(ctx, accountDb, [sid]) + + expect(inserted).toHaveLength(1) + expect(inserted[0].key).toBeUndefined() + expect(inserted[0]['%hash%']).toBeUndefined() + }) +}) diff --git a/server/backup/src/restore.ts b/server/backup/src/restore.ts index 8454182c3e..7a8d96f67d 100644 --- a/server/backup/src/restore.ts +++ b/server/backup/src/restore.ts @@ -578,40 +578,60 @@ export async function restore ( const limiter = new RateLimiter(opt.parallel ?? 1) + const isSkipped = (c: Domain): boolean => + (opt.include !== undefined && !opt.include.has(c)) || opt.skip?.has(c) === true + + async function processDomainWithRetry (c: Domain): Promise { + ctx.info('processing domain', { domain: c, workspaceId }) + let retry = 5 + let delay = 1 + while (retry > 0) { + retry-- + try { + const doProcessDomain = isAccountDomain(c) ? processAccountDomain : processDomain + await doProcessDomain(c) + if (delay > 1) { + ctx.warn('retry-success', { retry, delay, workspaceId }) + } + break + } catch (err: any) { + ctx.error('failed to process domain', { err, domain: c, workspaceId }) + if (retry !== 0) { + ctx.warn('cool-down to retry', { delay, domain: c, workspaceId }) + await new Promise((resolve) => setTimeout(resolve, delay * 1000)) + delay++ + } + } + } + } + try { let i = 0 - for (const c of domains) { + // Account domains are restored sequentially and in a fixed order: + // persons first, then social ids, since social_id.person_uuid references person. + const orderedAccountDomains = [toAccountDomain('person'), toAccountDomain('socialId')] + const accountDomainsToProcess = [ + ...orderedAccountDomains.filter((c) => domains.has(c)), + ...Array.from(domains).filter((c) => isAccountDomain(c) && !orderedAccountDomains.includes(c)) + ].filter((c) => !isSkipped(c)) + for (const c of accountDomainsToProcess) { if (opt.progress !== undefined) { await opt.progress?.(domainProgress) } - if (opt.include !== undefined && !opt.include.has(c)) { + await processDomainWithRetry(c) + domainProgress = Math.round(i / domains.size) * 100 + i++ + } + + for (const c of domains) { + if (isAccountDomain(c) || isSkipped(c)) { continue } - if (opt.skip?.has(c) === true) { - continue + if (opt.progress !== undefined) { + await opt.progress?.(domainProgress) } await limiter.add(async () => { - ctx.info('processing domain', { domain: c, workspaceId }) - let retry = 5 - let delay = 1 - while (retry > 0) { - retry-- - try { - const doProcessDomain = isAccountDomain(c) ? processAccountDomain : processDomain - await doProcessDomain(c) - if (delay > 1) { - ctx.warn('retry-success', { retry, delay, workspaceId }) - } - break - } catch (err: any) { - ctx.error('failed to process domain', { err, domain: c, workspaceId }) - if (retry !== 0) { - ctx.warn('cool-down to retry', { delay, domain: c, workspaceId }) - await new Promise((resolve) => setTimeout(resolve, delay * 1000)) - delay++ - } - } - } + await processDomainWithRetry(c) domainProgress = Math.round(i / domains.size) * 100 i++ }) @@ -686,7 +706,11 @@ async function restorePersons ( } } -async function restoreSocialIds (ctx: MeasureContext, accountDb: AccountDB, socialIds: SocialId[]): Promise { +export async function restoreSocialIds ( + ctx: MeasureContext, + accountDb: AccountDB, + socialIds: SocialId[] +): Promise { const chunks = chunkArray(socialIds, accountBatchSize) for (const chunk of chunks) { const ids = chunk.map((s) => s.key) @@ -695,9 +719,31 @@ async function restoreSocialIds (ctx: MeasureContext, accountDb: AccountDB, soci const existingSocialIds = await accountDb.socialId.find({ key: { $in: ids } }) const existingIds = new Set(existingSocialIds.map((s) => s.key)) + const missing = chunk.filter((s) => !existingIds.has(s.key)) + if (missing.length === 0) { + continue + } + + // social_id.person_uuid references person (social_id_person_fk). + // A backup may contain social ids whose person is not present in it + // (e.g. the workspace had a SocialIdentity without a matching contact + // Person with personUuid). Inserting such records would fail the whole + // batch, so filter them out and restore the rest. + const personUuids = Array.from(new Set(missing.map((s) => s.personUuid))) + const existingPersons = await accountDb.person.find({ uuid: { $in: personUuids } }) + const existingPersonUuids = new Set(existingPersons.map((p) => p.uuid)) + + const orphaned = missing.filter((s) => !existingPersonUuids.has(s.personUuid)) + if (orphaned.length > 0) { + ctx.warn('skipping social ids with missing persons', { + count: orphaned.length, + socialIds: orphaned.map((s) => ({ _id: s._id, key: s.key, personUuid: s.personUuid })) + }) + } + // Insert missing socialIds - const socialIdsToInsert: SocialId[] = chunk - .filter((s) => !existingIds.has(s.key)) + const socialIdsToInsert: SocialId[] = missing + .filter((s) => existingPersonUuids.has(s.personUuid)) .map((it) => { const { '%hash%': _1, key: _2, ...data } = it as any return data