diff --git a/plugins/export-resources/src/export.ts b/plugins/export-resources/src/export.ts index ef1ad58ff7..21f902c95f 100644 --- a/plugins/export-resources/src/export.ts +++ b/plugins/export-resources/src/export.ts @@ -41,7 +41,9 @@ export async function exportToWorkspace ( major: 1, minor: 0, reviewers: [], - controlledState: '' + controlledState: '', + seqNumber: '$ensureUnique', + code: '$ensureUnique' }, 'documents:class:DocumentMeta': { author: '$currentUser', diff --git a/services/export/pod-export/README.md b/services/export/pod-export/README.md index 4118ae0e4d..6162c16c87 100644 --- a/services/export/pod-export/README.md +++ b/services/export/pod-export/README.md @@ -111,7 +111,9 @@ Exports documents from the current workspace to another workspace. This endpoint "documents:class:ControlledDocument": { "author": "$currentUser", "owner": "$currentUser", - "state": "draft" + "state": "draft", + "code": "$ensureUnique", + "seqNumber": "$ensureUnique" } }, "objectId": "doc-id-1", @@ -131,9 +133,13 @@ Exports documents from the current workspace to another workspace. This endpoint - `field`: Field name containing the relation - `class`: Class of related documents - `direction`: `"forward"` (dependencies) or `"inverse"` (references) -- `fieldMappers` (optional): Field value overrides per document class - - Special value `"$currentUser"` is replaced with the current account's employee ID - - Example: `{ "author": "$currentUser" }` sets author to current user +- `fieldMappers` (optional): Field value overrides per document class. Supports special values: + - `$currentUser`: Replaced with current account's employee ID + - Example: `{ "author": "$currentUser" }` sets author to current user + - `$ensureUnique`: Ensures the field value is unique by checking the database and modifying if needed + - For strings: appends a suffix like "-1", "-2", etc. + - For numbers: increments until unique + - Example: `{ "code": "$ensureUnique", "seqNumber": "$ensureUnique" }` ensures unique code and seqNumber - `objectId` (optional): ID of the primary document for notification context - `objectSpace` (optional): Space of the primary document for notification context diff --git a/services/export/pod-export/src/__tests__/data-mapper.test.ts b/services/export/pod-export/src/__tests__/data-mapper.test.ts new file mode 100644 index 0000000000..27308baabe --- /dev/null +++ b/services/export/pod-export/src/__tests__/data-mapper.test.ts @@ -0,0 +1,700 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type Class, + type Doc, + type Ref, + type Space, + type TxOperations, + type MeasureContext, + type Hierarchy, + generateId, + platformNow +} from '@hcengineering/core' +import { DataMapper } from '../workspace/data-mapper' +import { type ExportState } from '../workspace/types' + +// Mock document classes +const mockDocClass = 'test:class:ControlledDocument' as Ref> +const mockSpaceId = 'test:space:1' as Ref + +function createMockHierarchy (): Hierarchy { + return { + findDomain: jest.fn(() => 'test_domain'), + isDerived: jest.fn(() => false), + getAllAttributes: jest.fn(() => new Map()), + isMixin: jest.fn(() => false), + hasMixin: jest.fn(() => undefined), + getClass: jest.fn((classRef: Ref>) => ({ + _id: classRef, + label: 'Test Class', + extends: undefined + })), + findAttribute: jest.fn(() => ({ + _id: 'test:attr:code' as any, + label: 'Code', + type: { _class: 'core:class:TypeString' as Ref> } + })) + } as unknown as Hierarchy +} + +function createMockTxOperations (existingDocs: Array> = []): TxOperations { + const docsMap = new Map() + for (const doc of existingDocs) { + docsMap.set(doc._id, doc as Doc) + } + + return { + findAll: jest.fn(async (classRef: Ref>, query: any, options?: any): Promise => { + const results: T[] = [] + for (const doc of docsMap.values()) { + if (doc._class !== classRef) continue + + // Match query + let matches = true + for (const [key, value] of Object.entries(query)) { + if (key === '_class') continue + + const docValue = (doc as any)[key] + + const queryValue = value as any + if (queryValue?.$like !== undefined) { + // Simple LIKE pattern matching (prefix-%) + const pattern = queryValue.$like as string + if (typeof docValue === 'string') { + if (pattern.endsWith('%')) { + const prefix = pattern.slice(0, -1) + // Unescape prefix (remove backslashes used for escaping) + const unescapedPrefix = prefix.replace(/\\(.)/g, '$1') + if (!docValue.startsWith(unescapedPrefix)) { + matches = false + break + } + } else { + matches = false + break + } + } else { + matches = false + break + } + } else if (queryValue?.$gte !== undefined) { + if (typeof docValue === 'number' && docValue < queryValue.$gte) { + matches = false + break + } + } else if (docValue !== value) { + matches = false + break + } + } + + if (matches) { + // Apply projection if specified + if (options?.projection !== undefined) { + const projected: any = { _id: doc._id, _class: doc._class } + for (const key in options.projection) { + if (options.projection[key] === 1) { + projected[key] = (doc as any)[key] + } + } + results.push(projected as T) + } else { + results.push(doc as T) + } + } + } + return results + }), + findOne: jest.fn(async function ( + this: any, + classRef: Ref>, + query: any, + options?: any + ): Promise { + const results = await this.findAll(classRef, query, options) + return results.length > 0 ? results[0] : undefined + }), + getHierarchy: jest.fn(() => createMockHierarchy()), + createDoc: jest.fn(), + addCollection: jest.fn(), + updateDoc: jest.fn() + } as unknown as TxOperations +} + +function createMockMeasureContext (): MeasureContext { + return { + info: jest.fn(), + error: jest.fn(), + warn: jest.fn() + } as unknown as MeasureContext +} + +describe('DataMapper - ensureFieldUnique', () => { + let mockContext: MeasureContext + let mockClient: TxOperations + let state: ExportState + + beforeEach(() => { + mockContext = createMockMeasureContext() + state = { + idMapping: new Map(), + spaceMapping: new Map(), + processingDocs: new Set(), + uniqueFieldValues: new Map() + } + jest.clearAllMocks() + }) + + describe('String values with prefix pattern', () => { + it('should find unique value by querying all prefix values at once', async () => { + // Setup: existing docs with codes DOC-1, DOC-2, DOC-5, all with same prefix + const testPrefix = 'DOC' + const existingDocs = [ + { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-1', + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + }, + { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-2', + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + }, + { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-5', + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + } + ] + mockClient = createMockTxOperations(existingDocs) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + code: '$ensureUnique' + } + }, + undefined + ) + + const doc = { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-1', + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + + const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false) + + // DOC-1 conflicts with existing, should use max(1,2,5) + 1 = 6 + expect(result.code).toBe('DOC-6') + // Should query by prefix field (global, no space filter) + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(mockClient.findAll).toHaveBeenCalledWith( + mockDocClass, + { prefix: testPrefix }, + { projection: { code: 1, prefix: 1 } } + ) + }) + + it('should handle prefix with special characters', async () => { + const testPrefix = 'DOC-TEST' + const existingDocs = [ + { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-TEST-1', + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + }, + { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-TEST-2', + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + } + ] + mockClient = createMockTxOperations(existingDocs) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + code: '$ensureUnique' + } + }, + undefined + ) + + const doc = { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-TEST-1', + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + + const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false) + + // Should use max(1,2) + 1 = 3 + expect(result.code).toBe('DOC-TEST-3') + }) + + it('should handle values already used in export batch', async () => { + const testPrefix = 'DOC' + const existingDocs: any[] = [] + mockClient = createMockTxOperations(existingDocs) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + code: '$ensureUnique' + } + }, + undefined + ) + + // First document + const doc1 = { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-1', + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + const result1 = await dataMapper.prepareDocumentData(doc1, mockSpaceId, false) + expect(result1.code).toBe('DOC-1') + + // Second document with same code + const doc2 = { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-1', + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + const result2 = await dataMapper.prepareDocumentData(doc2, mockSpaceId, false) + + // Should increment to DOC-2 since DOC-1 is already used in batch + expect(result2.code).toBe('DOC-2') + }) + + it('should handle string without prefix pattern', async () => { + const existingDocs = [ + { + _id: generateId(), + _class: mockDocClass, + code: 'SIMPLE', + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + } + ] + mockClient = createMockTxOperations(existingDocs) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + code: '$ensureUnique' + } + }, + undefined + ) + + const doc = { + _id: generateId(), + _class: mockDocClass, + code: 'SIMPLE', + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + + const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false) + + // Should append suffix + expect(result.code).toBe('SIMPLE-1') + // Should use findOne for exact match (global, no space filter) + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(mockClient.findOne).toHaveBeenCalledWith(mockDocClass, { code: 'SIMPLE' }, { projection: { code: 1 } }) + }) + }) + + describe('Numeric values', () => { + it('should find unique value by querying all values >= current', async () => { + // Setup: existing docs with seqNumber 10, 11, 15 + const existingDocs = [ + { + _id: generateId(), + _class: mockDocClass, + seqNumber: 10, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + }, + { + _id: generateId(), + _class: mockDocClass, + seqNumber: 11, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + }, + { + _id: generateId(), + _class: mockDocClass, + seqNumber: 15, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + } + ] + mockClient = createMockTxOperations(existingDocs) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + seqNumber: '$ensureUnique' + } + }, + undefined + ) + + const doc = { + _id: generateId(), + _class: mockDocClass, + seqNumber: 10, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + + const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false) + + // Should use max(10,11,15) + 1 = 16 + expect(result.seqNumber).toBe(16) + // Should query with $gte (global, no space filter) + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(mockClient.findAll).toHaveBeenCalledWith( + mockDocClass, + { seqNumber: { $gte: 10 } }, + { projection: { seqNumber: 1 } } + ) + }) + + it('should handle numeric values already used in export batch', async () => { + const existingDocs: any[] = [] + mockClient = createMockTxOperations(existingDocs) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + seqNumber: '$ensureUnique' + } + }, + undefined + ) + + // First document + const doc1 = { + _id: generateId(), + _class: mockDocClass, + seqNumber: 5, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + const result1 = await dataMapper.prepareDocumentData(doc1, mockSpaceId, false) + expect(result1.seqNumber).toBe(5) + + // Second document with same seqNumber + const doc2 = { + _id: generateId(), + _class: mockDocClass, + seqNumber: 5, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + const result2 = await dataMapper.prepareDocumentData(doc2, mockSpaceId, false) + + // Should increment to 6 since 5 is already used in batch + expect(result2.seqNumber).toBe(6) + }) + + it('should handle numeric value when no conflicts exist', async () => { + const existingDocs: any[] = [] + mockClient = createMockTxOperations(existingDocs) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + seqNumber: '$ensureUnique' + } + }, + undefined + ) + + const doc = { + _id: generateId(), + _class: mockDocClass, + seqNumber: 1, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + + const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false) + + // Should keep original value if unique + expect(result.seqNumber).toBe(1) + }) + }) + + describe('Projection usage', () => { + it('should use projection to only load the field being checked', async () => { + const existingDocs = [ + { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-1', + title: 'Title 1', + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + } + ] + mockClient = createMockTxOperations(existingDocs) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + code: '$ensureUnique' + } + }, + undefined + ) + + const doc = { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-2', + title: 'Title 2', + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + + await dataMapper.prepareDocumentData(doc, mockSpaceId, false) + + // Verify projection was used + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(mockClient.findAll).toHaveBeenCalledWith(mockDocClass, expect.anything(), { projection: { code: 1 } }) + }) + }) + + describe('Edge cases', () => { + it('should handle null/undefined values gracefully', async () => { + mockClient = createMockTxOperations([]) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + code: '$ensureUnique' + } + }, + undefined + ) + + const doc = { + _id: generateId(), + _class: mockDocClass, + code: null, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + + const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false) + + // Should not modify null values + expect(result.code).toBeNull() + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(mockClient.findAll).not.toHaveBeenCalled() + }) + + it('should handle unsupported types', async () => { + mockClient = createMockTxOperations([]) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + code: '$ensureUnique' + } + }, + undefined + ) + + const doc = { + _id: generateId(), + _class: mockDocClass, + code: { complex: 'object' }, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + + const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false) + + // Should not modify unsupported types + expect(result.code).toEqual({ complex: 'object' }) + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(mockContext.warn).toHaveBeenCalledWith(expect.stringContaining('Cannot ensure uniqueness')) + }) + + it('should handle empty database with prefix pattern', async () => { + mockClient = createMockTxOperations([]) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + code: '$ensureUnique' + } + }, + undefined + ) + + const doc = { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-5', + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + + const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false) + + expect(result.code).toBe('DOC-5') + }) + + it('should handle multiple fields with $ensureUnique', async () => { + const testPrefix = 'DOC' + const existingDocs = [ + { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-1', + seqNumber: 10, + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() + } + ] + mockClient = createMockTxOperations(existingDocs) + + const dataMapper = new DataMapper( + mockContext, + mockClient, + state, + { + [mockDocClass]: { + code: '$ensureUnique', + seqNumber: '$ensureUnique' + } + }, + undefined + ) + + const doc = { + _id: generateId(), + _class: mockDocClass, + code: 'DOC-1', + seqNumber: 10, + prefix: testPrefix, + space: mockSpaceId, + modifiedOn: platformNow(), + modifiedBy: generateId() as any + } + + const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false) + + // Both fields should be made unique + expect(result.code).toBe('DOC-2') + expect(result.seqNumber).toBe(11) + }) + }) +}) diff --git a/services/export/pod-export/src/workspace/data-mapper.ts b/services/export/pod-export/src/workspace/data-mapper.ts index d641c60c8d..4259b39283 100644 --- a/services/export/pod-export/src/workspace/data-mapper.ts +++ b/services/export/pod-export/src/workspace/data-mapper.ts @@ -100,7 +100,9 @@ export class DataMapper { /** * Apply field mappers for specific document classes. * Field mappers format: { className: { fieldName: value, ... } } - * Special value '$currentUser' is replaced with current account's employee ID + * Special values: + * - '$currentUser' is replaced with current account's employee ID + * - '$ensureUnique' ensures the field value is unique by checking database and modifying if needed */ private async applyFieldMappers (docClass: Ref>, data: Record): Promise { const hierarchy = this.targetClient.getHierarchy() @@ -149,6 +151,9 @@ export class DataMapper { } else { this.context.warn(`Cannot map ${fieldName}: $currentUser but current account employee not found`) } + } else if (fieldValue === '$ensureUnique') { + // Ensure field value is unique (globally, not per space) + await this.ensureFieldUnique(docClass, fieldName, data) } else if (fieldValue === '') { // Empty string means clear the field data[fieldName] = undefined @@ -159,6 +164,232 @@ export class DataMapper { } } + /** + * Ensure a field value is unique by checking the database and modifying if needed. + * Uses the document's `prefix` field to query documents with the same template/prefix, + * ensuring uniqueness within that group. Falls back to global uniqueness if no prefix is available. + * - For documents with prefix: queries by prefix field and ensures uniqueness within that group + * - For documents without prefix: uses fallback queries (exact match for strings, >= for numbers) + */ + private async ensureFieldUnique ( + docClass: Ref>, + fieldName: string, + data: Record + ): Promise { + const currentValue = data[fieldName] + if (currentValue === undefined || currentValue === null) { + return + } + + // Initialize unique values tracking if not exists + if (this.state.uniqueFieldValues === undefined) { + this.state.uniqueFieldValues = new Map() + } + + const classKey = docClass + if (!this.state.uniqueFieldValues.has(classKey)) { + this.state.uniqueFieldValues.set(classKey, new Map()) + } + + let fieldMap = this.state.uniqueFieldValues.get(classKey) + if (fieldMap === undefined) { + fieldMap = new Map() + this.state.uniqueFieldValues.set(classKey, fieldMap) + } + + let usedValues = fieldMap.get(fieldName) + if (usedValues === undefined) { + usedValues = new Set() + fieldMap.set(fieldName, usedValues) + } + + const projection = { [fieldName]: 1 } as any + + let uniqueValue: string | number = currentValue + + if (typeof currentValue === 'string') { + const documentPrefix = data.prefix + if (documentPrefix !== undefined && typeof documentPrefix === 'string' && documentPrefix !== '') { + const codeMatch = currentValue.match(/-(\d+)$/) + const baseNum = codeMatch !== null ? parseInt(codeMatch[1], 10) : parseInt(currentValue, 10) + + const query: any = { prefix: documentPrefix } + + const prefixProjection = { [fieldName]: 1, prefix: 1 } as any + + const existingDocs = await this.targetClient.findAll(docClass, query, { projection: prefixProjection }) + + const existingValues = new Set() + for (const doc of existingDocs) { + const value = (doc as any)[fieldName] + if (value !== undefined && value !== null) { + existingValues.add(String(value)) + } + } + + for (const usedValue of usedValues) { + if (typeof usedValue === 'string') { + existingValues.add(usedValue) + } + } + + const isCurrentValueUnique = !existingValues.has(currentValue) + + if (isCurrentValueUnique) { + uniqueValue = currentValue + } else { + const existingNumbers = new Set() + for (const val of existingValues) { + const match = val.match(/-(\d+)$/) + if (match !== null) { + existingNumbers.add(parseInt(match[1], 10)) + } else { + const num = parseInt(val, 10) + if (!isNaN(num)) { + existingNumbers.add(num) + } + } + } + + const maxNum = existingNumbers.size > 0 ? Math.max(...Array.from(existingNumbers), baseNum - 1) : baseNum - 1 + // Generate new value: if original had pattern "PREFIX-N", use same pattern, otherwise just use number + if (codeMatch !== null) { + const originalPrefix = currentValue.substring(0, currentValue.lastIndexOf('-')) + uniqueValue = `${originalPrefix}-${maxNum + 1}` + } else { + uniqueValue = String(maxNum + 1) + } + this.context.info( + `ensureFieldUnique: ${fieldName} value ${currentValue} conflicts within prefix "${documentPrefix}", generating new value: ${uniqueValue}` + ) + } + } else { + // No prefix pattern, check if exact value exists - global uniqueness + const query: any = { [fieldName]: currentValue } + + const existing = await this.targetClient.findOne(docClass, query, { projection }) + const isUsedInBatch = usedValues.has(currentValue) + + if (existing === undefined && !isUsedInBatch) { + // Value is unique, use it as-is + uniqueValue = currentValue + } else { + // Value exists, append suffix + let attempt = 1 + while (attempt < 10) { + uniqueValue = `${currentValue}-${attempt}` + const checkQuery: any = { [fieldName]: uniqueValue } + const checkExisting = await this.targetClient.findOne(docClass, checkQuery, { projection }) + if (checkExisting === undefined && !usedValues.has(uniqueValue)) { + break + } + attempt++ + } + if (attempt >= 10) { + this.context.error( + `ensureFieldUnique: Failed to find unique value for field ${fieldName} after 10 attempts` + ) + return + } + } + } + } else if (typeof currentValue === 'number') { + // For numeric fields like seqNumber, check if document has a prefix field + // If so, ensure uniqueness within the same prefix (template) group + const documentPrefix = data.prefix + if (documentPrefix !== undefined && typeof documentPrefix === 'string' && documentPrefix !== '') { + // Query all documents with the same prefix (same template) - global uniqueness + const query: any = { prefix: documentPrefix } + + // Project both the field we're checking and prefix to verify + const prefixProjection = { [fieldName]: 1, prefix: 1 } as any + + const existingDocs = await this.targetClient.findAll(docClass, query, { projection: prefixProjection }) + + // Extract all numbers from existing values + const existingNumbers = new Set() + for (const doc of existingDocs) { + const value = (doc as any)[fieldName] + if (typeof value === 'number') { + existingNumbers.add(value) + } + } + + // Also check values used in this export batch + for (const usedValue of usedValues) { + if (typeof usedValue === 'number') { + existingNumbers.add(usedValue) + } + } + + // Check if current value is already unique + const isCurrentValueUnique = !existingNumbers.has(currentValue) && !usedValues.has(currentValue) + + if (isCurrentValueUnique) { + // Current value is unique, keep it + uniqueValue = currentValue + } else { + // Find max number and use max + 1 + const maxNum = existingNumbers.size > 0 ? Math.max(...Array.from(existingNumbers)) : currentValue - 1 + uniqueValue = maxNum + 1 + this.context.info( + `ensureFieldUnique: ${fieldName} value ${currentValue} conflicts within prefix "${documentPrefix}", generating new value: ${uniqueValue}` + ) + } + } else { + // No prefix field, fall back to querying all values >= current - global uniqueness + const query: any = { [fieldName]: { $gte: currentValue } } + + const existingDocs = await this.targetClient.findAll(docClass, query, { projection }) + + // Extract all numbers from existing values + const existingNumbers = new Set() + for (const doc of existingDocs) { + const value = (doc as any)[fieldName] + if (typeof value === 'number') { + existingNumbers.add(value) + } + } + + // Also check values used in this export batch + for (const usedValue of usedValues) { + if (typeof usedValue === 'number' && usedValue >= currentValue) { + existingNumbers.add(usedValue) + } + } + + // Check if current value is already unique + const isCurrentValueUnique = !existingNumbers.has(currentValue) && !usedValues.has(currentValue) + + if (isCurrentValueUnique) { + // Current value is unique, keep it + uniqueValue = currentValue + } else { + // Find max number and use max + 1 + const maxNum = existingNumbers.size > 0 ? Math.max(...Array.from(existingNumbers)) : currentValue - 1 + uniqueValue = maxNum + 1 + this.context.info( + `ensureFieldUnique: ${fieldName} value ${currentValue} conflicts, generating new value: ${uniqueValue}` + ) + } + } + } else { + // Unsupported type, skip uniqueness check + this.context.warn(`Cannot ensure uniqueness for field ${fieldName} with type ${typeof currentValue}`) + return + } + + // Update data with unique value + data[fieldName] = uniqueValue + usedValues.add(uniqueValue) + + if (uniqueValue !== currentValue) { + this.context.info( + `ensureFieldUnique: Updated ${fieldName} from ${currentValue} to ${uniqueValue} (class: ${docClass})` + ) + } + } + /** * Recursively remap a value, handling nested objects and arrays. */ diff --git a/services/export/pod-export/src/workspace/types.ts b/services/export/pod-export/src/workspace/types.ts index af403a5c77..e51e28b61b 100644 --- a/services/export/pod-export/src/workspace/types.ts +++ b/services/export/pod-export/src/workspace/types.ts @@ -63,4 +63,6 @@ export interface ExportState { idMapping: Map, Ref> spaceMapping: Map, Ref> processingDocs: Set> + // Track unique field values per class: { className: { fieldName: Set } } + uniqueFieldValues: Map>> } diff --git a/services/export/pod-export/src/workspace/workspace-exporter.ts b/services/export/pod-export/src/workspace/workspace-exporter.ts index 4b9ca04266..4774439b69 100644 --- a/services/export/pod-export/src/workspace/workspace-exporter.ts +++ b/services/export/pod-export/src/workspace/workspace-exporter.ts @@ -60,7 +60,8 @@ export class CrossWorkspaceExporter { this.state = { idMapping: new Map, Ref>(), spaceMapping: new Map, Ref>(), - processingDocs: new Set>() + processingDocs: new Set>(), + uniqueFieldValues: new Map() } // Initialize component exporters