Ensure controlled document codes and sequences are unique (#10324)

Signed-off-by: Artem Savchenko <armisav@gmail.com>
This commit is contained in:
Artyom Savchenko
2025-12-23 22:12:39 +05:00
committed by GitHub
parent 29c96ff9fb
commit f6f01ed3d3
6 changed files with 949 additions and 7 deletions
+3 -1
View File
@@ -41,7 +41,9 @@ export async function exportToWorkspace (
major: 1,
minor: 0,
reviewers: [],
controlledState: ''
controlledState: '',
seqNumber: '$ensureUnique',
code: '$ensureUnique'
},
'documents:class:DocumentMeta': {
author: '$currentUser',
+10 -4
View File
@@ -111,7 +111,9 @@ Exports documents from the current workspace to another workspace. This endpoint
"documents:class:ControlledDocument": {
"author": "$currentUser",
"owner": "$currentUser",
"state": "draft"
"state": "draft",
"code": "$ensureUnique",
"seqNumber": "$ensureUnique"
}
},
"objectId": "doc-id-1",
@@ -131,9 +133,13 @@ Exports documents from the current workspace to another workspace. This endpoint
- `field`: Field name containing the relation
- `class`: Class of related documents
- `direction`: `"forward"` (dependencies) or `"inverse"` (references)
- `fieldMappers` (optional): Field value overrides per document class
- Special value `"$currentUser"` is replaced with the current account's employee ID
- Example: `{ "author": "$currentUser" }` sets author to current user
- `fieldMappers` (optional): Field value overrides per document class. Supports special values:
- `$currentUser`: Replaced with current account's employee ID
- Example: `{ "author": "$currentUser" }` sets author to current user
- `$ensureUnique`: Ensures the field value is unique by checking the database and modifying if needed
- For strings: appends a suffix like "-1", "-2", etc.
- For numbers: increments until unique
- Example: `{ "code": "$ensureUnique", "seqNumber": "$ensureUnique" }` ensures unique code and seqNumber
- `objectId` (optional): ID of the primary document for notification context
- `objectSpace` (optional): Space of the primary document for notification context
@@ -0,0 +1,700 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import {
type Class,
type Doc,
type Ref,
type Space,
type TxOperations,
type MeasureContext,
type Hierarchy,
generateId,
platformNow
} from '@hcengineering/core'
import { DataMapper } from '../workspace/data-mapper'
import { type ExportState } from '../workspace/types'
// Mock document classes
const mockDocClass = 'test:class:ControlledDocument' as Ref<Class<Doc>>
const mockSpaceId = 'test:space:1' as Ref<Space>
function createMockHierarchy (): Hierarchy {
return {
findDomain: jest.fn(() => 'test_domain'),
isDerived: jest.fn(() => false),
getAllAttributes: jest.fn(() => new Map()),
isMixin: jest.fn(() => false),
hasMixin: jest.fn(() => undefined),
getClass: jest.fn((classRef: Ref<Class<Doc>>) => ({
_id: classRef,
label: 'Test Class',
extends: undefined
})),
findAttribute: jest.fn(() => ({
_id: 'test:attr:code' as any,
label: 'Code',
type: { _class: 'core:class:TypeString' as Ref<Class<Doc>> }
}))
} as unknown as Hierarchy
}
function createMockTxOperations (existingDocs: Array<Record<string, any>> = []): TxOperations {
const docsMap = new Map<string, Doc>()
for (const doc of existingDocs) {
docsMap.set(doc._id, doc as Doc)
}
return {
findAll: jest.fn(async <T extends Doc>(classRef: Ref<Class<T>>, query: any, options?: any): Promise<T[]> => {
const results: T[] = []
for (const doc of docsMap.values()) {
if (doc._class !== classRef) continue
// Match query
let matches = true
for (const [key, value] of Object.entries(query)) {
if (key === '_class') continue
const docValue = (doc as any)[key]
const queryValue = value as any
if (queryValue?.$like !== undefined) {
// Simple LIKE pattern matching (prefix-%)
const pattern = queryValue.$like as string
if (typeof docValue === 'string') {
if (pattern.endsWith('%')) {
const prefix = pattern.slice(0, -1)
// Unescape prefix (remove backslashes used for escaping)
const unescapedPrefix = prefix.replace(/\\(.)/g, '$1')
if (!docValue.startsWith(unescapedPrefix)) {
matches = false
break
}
} else {
matches = false
break
}
} else {
matches = false
break
}
} else if (queryValue?.$gte !== undefined) {
if (typeof docValue === 'number' && docValue < queryValue.$gte) {
matches = false
break
}
} else if (docValue !== value) {
matches = false
break
}
}
if (matches) {
// Apply projection if specified
if (options?.projection !== undefined) {
const projected: any = { _id: doc._id, _class: doc._class }
for (const key in options.projection) {
if (options.projection[key] === 1) {
projected[key] = (doc as any)[key]
}
}
results.push(projected as T)
} else {
results.push(doc as T)
}
}
}
return results
}),
findOne: jest.fn(async function <T extends Doc>(
this: any,
classRef: Ref<Class<T>>,
query: any,
options?: any
): Promise<T | undefined> {
const results = await this.findAll(classRef, query, options)
return results.length > 0 ? results[0] : undefined
}),
getHierarchy: jest.fn(() => createMockHierarchy()),
createDoc: jest.fn(),
addCollection: jest.fn(),
updateDoc: jest.fn()
} as unknown as TxOperations
}
function createMockMeasureContext (): MeasureContext {
return {
info: jest.fn(),
error: jest.fn(),
warn: jest.fn()
} as unknown as MeasureContext
}
describe('DataMapper - ensureFieldUnique', () => {
let mockContext: MeasureContext
let mockClient: TxOperations
let state: ExportState
beforeEach(() => {
mockContext = createMockMeasureContext()
state = {
idMapping: new Map(),
spaceMapping: new Map(),
processingDocs: new Set(),
uniqueFieldValues: new Map()
}
jest.clearAllMocks()
})
describe('String values with prefix pattern', () => {
it('should find unique value by querying all prefix values at once', async () => {
// Setup: existing docs with codes DOC-1, DOC-2, DOC-5, all with same prefix
const testPrefix = 'DOC'
const existingDocs = [
{
_id: generateId(),
_class: mockDocClass,
code: 'DOC-1',
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
},
{
_id: generateId(),
_class: mockDocClass,
code: 'DOC-2',
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
},
{
_id: generateId(),
_class: mockDocClass,
code: 'DOC-5',
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
}
]
mockClient = createMockTxOperations(existingDocs)
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
code: '$ensureUnique'
}
},
undefined
)
const doc = {
_id: generateId(),
_class: mockDocClass,
code: 'DOC-1',
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false)
// DOC-1 conflicts with existing, should use max(1,2,5) + 1 = 6
expect(result.code).toBe('DOC-6')
// Should query by prefix field (global, no space filter)
// eslint-disable-next-line @typescript-eslint/unbound-method
expect(mockClient.findAll).toHaveBeenCalledWith(
mockDocClass,
{ prefix: testPrefix },
{ projection: { code: 1, prefix: 1 } }
)
})
it('should handle prefix with special characters', async () => {
const testPrefix = 'DOC-TEST'
const existingDocs = [
{
_id: generateId(),
_class: mockDocClass,
code: 'DOC-TEST-1',
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
},
{
_id: generateId(),
_class: mockDocClass,
code: 'DOC-TEST-2',
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
}
]
mockClient = createMockTxOperations(existingDocs)
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
code: '$ensureUnique'
}
},
undefined
)
const doc = {
_id: generateId(),
_class: mockDocClass,
code: 'DOC-TEST-1',
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false)
// Should use max(1,2) + 1 = 3
expect(result.code).toBe('DOC-TEST-3')
})
it('should handle values already used in export batch', async () => {
const testPrefix = 'DOC'
const existingDocs: any[] = []
mockClient = createMockTxOperations(existingDocs)
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
code: '$ensureUnique'
}
},
undefined
)
// First document
const doc1 = {
_id: generateId(),
_class: mockDocClass,
code: 'DOC-1',
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result1 = await dataMapper.prepareDocumentData(doc1, mockSpaceId, false)
expect(result1.code).toBe('DOC-1')
// Second document with same code
const doc2 = {
_id: generateId(),
_class: mockDocClass,
code: 'DOC-1',
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result2 = await dataMapper.prepareDocumentData(doc2, mockSpaceId, false)
// Should increment to DOC-2 since DOC-1 is already used in batch
expect(result2.code).toBe('DOC-2')
})
it('should handle string without prefix pattern', async () => {
const existingDocs = [
{
_id: generateId(),
_class: mockDocClass,
code: 'SIMPLE',
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
}
]
mockClient = createMockTxOperations(existingDocs)
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
code: '$ensureUnique'
}
},
undefined
)
const doc = {
_id: generateId(),
_class: mockDocClass,
code: 'SIMPLE',
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false)
// Should append suffix
expect(result.code).toBe('SIMPLE-1')
// Should use findOne for exact match (global, no space filter)
// eslint-disable-next-line @typescript-eslint/unbound-method
expect(mockClient.findOne).toHaveBeenCalledWith(mockDocClass, { code: 'SIMPLE' }, { projection: { code: 1 } })
})
})
describe('Numeric values', () => {
it('should find unique value by querying all values >= current', async () => {
// Setup: existing docs with seqNumber 10, 11, 15
const existingDocs = [
{
_id: generateId(),
_class: mockDocClass,
seqNumber: 10,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
},
{
_id: generateId(),
_class: mockDocClass,
seqNumber: 11,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
},
{
_id: generateId(),
_class: mockDocClass,
seqNumber: 15,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
}
]
mockClient = createMockTxOperations(existingDocs)
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
seqNumber: '$ensureUnique'
}
},
undefined
)
const doc = {
_id: generateId(),
_class: mockDocClass,
seqNumber: 10,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false)
// Should use max(10,11,15) + 1 = 16
expect(result.seqNumber).toBe(16)
// Should query with $gte (global, no space filter)
// eslint-disable-next-line @typescript-eslint/unbound-method
expect(mockClient.findAll).toHaveBeenCalledWith(
mockDocClass,
{ seqNumber: { $gte: 10 } },
{ projection: { seqNumber: 1 } }
)
})
it('should handle numeric values already used in export batch', async () => {
const existingDocs: any[] = []
mockClient = createMockTxOperations(existingDocs)
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
seqNumber: '$ensureUnique'
}
},
undefined
)
// First document
const doc1 = {
_id: generateId(),
_class: mockDocClass,
seqNumber: 5,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result1 = await dataMapper.prepareDocumentData(doc1, mockSpaceId, false)
expect(result1.seqNumber).toBe(5)
// Second document with same seqNumber
const doc2 = {
_id: generateId(),
_class: mockDocClass,
seqNumber: 5,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result2 = await dataMapper.prepareDocumentData(doc2, mockSpaceId, false)
// Should increment to 6 since 5 is already used in batch
expect(result2.seqNumber).toBe(6)
})
it('should handle numeric value when no conflicts exist', async () => {
const existingDocs: any[] = []
mockClient = createMockTxOperations(existingDocs)
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
seqNumber: '$ensureUnique'
}
},
undefined
)
const doc = {
_id: generateId(),
_class: mockDocClass,
seqNumber: 1,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false)
// Should keep original value if unique
expect(result.seqNumber).toBe(1)
})
})
describe('Projection usage', () => {
it('should use projection to only load the field being checked', async () => {
const existingDocs = [
{
_id: generateId(),
_class: mockDocClass,
code: 'DOC-1',
title: 'Title 1',
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
}
]
mockClient = createMockTxOperations(existingDocs)
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
code: '$ensureUnique'
}
},
undefined
)
const doc = {
_id: generateId(),
_class: mockDocClass,
code: 'DOC-2',
title: 'Title 2',
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
await dataMapper.prepareDocumentData(doc, mockSpaceId, false)
// Verify projection was used
// eslint-disable-next-line @typescript-eslint/unbound-method
expect(mockClient.findAll).toHaveBeenCalledWith(mockDocClass, expect.anything(), { projection: { code: 1 } })
})
})
describe('Edge cases', () => {
it('should handle null/undefined values gracefully', async () => {
mockClient = createMockTxOperations([])
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
code: '$ensureUnique'
}
},
undefined
)
const doc = {
_id: generateId(),
_class: mockDocClass,
code: null,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false)
// Should not modify null values
expect(result.code).toBeNull()
// eslint-disable-next-line @typescript-eslint/unbound-method
expect(mockClient.findAll).not.toHaveBeenCalled()
})
it('should handle unsupported types', async () => {
mockClient = createMockTxOperations([])
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
code: '$ensureUnique'
}
},
undefined
)
const doc = {
_id: generateId(),
_class: mockDocClass,
code: { complex: 'object' },
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false)
// Should not modify unsupported types
expect(result.code).toEqual({ complex: 'object' })
// eslint-disable-next-line @typescript-eslint/unbound-method
expect(mockContext.warn).toHaveBeenCalledWith(expect.stringContaining('Cannot ensure uniqueness'))
})
it('should handle empty database with prefix pattern', async () => {
mockClient = createMockTxOperations([])
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
code: '$ensureUnique'
}
},
undefined
)
const doc = {
_id: generateId(),
_class: mockDocClass,
code: 'DOC-5',
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false)
expect(result.code).toBe('DOC-5')
})
it('should handle multiple fields with $ensureUnique', async () => {
const testPrefix = 'DOC'
const existingDocs = [
{
_id: generateId(),
_class: mockDocClass,
code: 'DOC-1',
seqNumber: 10,
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId()
}
]
mockClient = createMockTxOperations(existingDocs)
const dataMapper = new DataMapper(
mockContext,
mockClient,
state,
{
[mockDocClass]: {
code: '$ensureUnique',
seqNumber: '$ensureUnique'
}
},
undefined
)
const doc = {
_id: generateId(),
_class: mockDocClass,
code: 'DOC-1',
seqNumber: 10,
prefix: testPrefix,
space: mockSpaceId,
modifiedOn: platformNow(),
modifiedBy: generateId() as any
}
const result = await dataMapper.prepareDocumentData(doc, mockSpaceId, false)
// Both fields should be made unique
expect(result.code).toBe('DOC-2')
expect(result.seqNumber).toBe(11)
})
})
})
@@ -100,7 +100,9 @@ export class DataMapper {
/**
* Apply field mappers for specific document classes.
* Field mappers format: { className: { fieldName: value, ... } }
* Special value '$currentUser' is replaced with current account's employee ID
* Special values:
* - '$currentUser' is replaced with current account's employee ID
* - '$ensureUnique' ensures the field value is unique by checking database and modifying if needed
*/
private async applyFieldMappers (docClass: Ref<Class<Doc>>, data: Record<string, any>): Promise<void> {
const hierarchy = this.targetClient.getHierarchy()
@@ -149,6 +151,9 @@ export class DataMapper {
} else {
this.context.warn(`Cannot map ${fieldName}: $currentUser but current account employee not found`)
}
} else if (fieldValue === '$ensureUnique') {
// Ensure field value is unique (globally, not per space)
await this.ensureFieldUnique(docClass, fieldName, data)
} else if (fieldValue === '') {
// Empty string means clear the field
data[fieldName] = undefined
@@ -159,6 +164,232 @@ export class DataMapper {
}
}
/**
* Ensure a field value is unique by checking the database and modifying if needed.
* Uses the document's `prefix` field to query documents with the same template/prefix,
* ensuring uniqueness within that group. Falls back to global uniqueness if no prefix is available.
* - For documents with prefix: queries by prefix field and ensures uniqueness within that group
* - For documents without prefix: uses fallback queries (exact match for strings, >= for numbers)
*/
private async ensureFieldUnique (
docClass: Ref<Class<Doc>>,
fieldName: string,
data: Record<string, any>
): Promise<void> {
const currentValue = data[fieldName]
if (currentValue === undefined || currentValue === null) {
return
}
// Initialize unique values tracking if not exists
if (this.state.uniqueFieldValues === undefined) {
this.state.uniqueFieldValues = new Map()
}
const classKey = docClass
if (!this.state.uniqueFieldValues.has(classKey)) {
this.state.uniqueFieldValues.set(classKey, new Map())
}
let fieldMap = this.state.uniqueFieldValues.get(classKey)
if (fieldMap === undefined) {
fieldMap = new Map()
this.state.uniqueFieldValues.set(classKey, fieldMap)
}
let usedValues = fieldMap.get(fieldName)
if (usedValues === undefined) {
usedValues = new Set()
fieldMap.set(fieldName, usedValues)
}
const projection = { [fieldName]: 1 } as any
let uniqueValue: string | number = currentValue
if (typeof currentValue === 'string') {
const documentPrefix = data.prefix
if (documentPrefix !== undefined && typeof documentPrefix === 'string' && documentPrefix !== '') {
const codeMatch = currentValue.match(/-(\d+)$/)
const baseNum = codeMatch !== null ? parseInt(codeMatch[1], 10) : parseInt(currentValue, 10)
const query: any = { prefix: documentPrefix }
const prefixProjection = { [fieldName]: 1, prefix: 1 } as any
const existingDocs = await this.targetClient.findAll(docClass, query, { projection: prefixProjection })
const existingValues = new Set<string>()
for (const doc of existingDocs) {
const value = (doc as any)[fieldName]
if (value !== undefined && value !== null) {
existingValues.add(String(value))
}
}
for (const usedValue of usedValues) {
if (typeof usedValue === 'string') {
existingValues.add(usedValue)
}
}
const isCurrentValueUnique = !existingValues.has(currentValue)
if (isCurrentValueUnique) {
uniqueValue = currentValue
} else {
const existingNumbers = new Set<number>()
for (const val of existingValues) {
const match = val.match(/-(\d+)$/)
if (match !== null) {
existingNumbers.add(parseInt(match[1], 10))
} else {
const num = parseInt(val, 10)
if (!isNaN(num)) {
existingNumbers.add(num)
}
}
}
const maxNum = existingNumbers.size > 0 ? Math.max(...Array.from(existingNumbers), baseNum - 1) : baseNum - 1
// Generate new value: if original had pattern "PREFIX-N", use same pattern, otherwise just use number
if (codeMatch !== null) {
const originalPrefix = currentValue.substring(0, currentValue.lastIndexOf('-'))
uniqueValue = `${originalPrefix}-${maxNum + 1}`
} else {
uniqueValue = String(maxNum + 1)
}
this.context.info(
`ensureFieldUnique: ${fieldName} value ${currentValue} conflicts within prefix "${documentPrefix}", generating new value: ${uniqueValue}`
)
}
} else {
// No prefix pattern, check if exact value exists - global uniqueness
const query: any = { [fieldName]: currentValue }
const existing = await this.targetClient.findOne(docClass, query, { projection })
const isUsedInBatch = usedValues.has(currentValue)
if (existing === undefined && !isUsedInBatch) {
// Value is unique, use it as-is
uniqueValue = currentValue
} else {
// Value exists, append suffix
let attempt = 1
while (attempt < 10) {
uniqueValue = `${currentValue}-${attempt}`
const checkQuery: any = { [fieldName]: uniqueValue }
const checkExisting = await this.targetClient.findOne(docClass, checkQuery, { projection })
if (checkExisting === undefined && !usedValues.has(uniqueValue)) {
break
}
attempt++
}
if (attempt >= 10) {
this.context.error(
`ensureFieldUnique: Failed to find unique value for field ${fieldName} after 10 attempts`
)
return
}
}
}
} else if (typeof currentValue === 'number') {
// For numeric fields like seqNumber, check if document has a prefix field
// If so, ensure uniqueness within the same prefix (template) group
const documentPrefix = data.prefix
if (documentPrefix !== undefined && typeof documentPrefix === 'string' && documentPrefix !== '') {
// Query all documents with the same prefix (same template) - global uniqueness
const query: any = { prefix: documentPrefix }
// Project both the field we're checking and prefix to verify
const prefixProjection = { [fieldName]: 1, prefix: 1 } as any
const existingDocs = await this.targetClient.findAll(docClass, query, { projection: prefixProjection })
// Extract all numbers from existing values
const existingNumbers = new Set<number>()
for (const doc of existingDocs) {
const value = (doc as any)[fieldName]
if (typeof value === 'number') {
existingNumbers.add(value)
}
}
// Also check values used in this export batch
for (const usedValue of usedValues) {
if (typeof usedValue === 'number') {
existingNumbers.add(usedValue)
}
}
// Check if current value is already unique
const isCurrentValueUnique = !existingNumbers.has(currentValue) && !usedValues.has(currentValue)
if (isCurrentValueUnique) {
// Current value is unique, keep it
uniqueValue = currentValue
} else {
// Find max number and use max + 1
const maxNum = existingNumbers.size > 0 ? Math.max(...Array.from(existingNumbers)) : currentValue - 1
uniqueValue = maxNum + 1
this.context.info(
`ensureFieldUnique: ${fieldName} value ${currentValue} conflicts within prefix "${documentPrefix}", generating new value: ${uniqueValue}`
)
}
} else {
// No prefix field, fall back to querying all values >= current - global uniqueness
const query: any = { [fieldName]: { $gte: currentValue } }
const existingDocs = await this.targetClient.findAll(docClass, query, { projection })
// Extract all numbers from existing values
const existingNumbers = new Set<number>()
for (const doc of existingDocs) {
const value = (doc as any)[fieldName]
if (typeof value === 'number') {
existingNumbers.add(value)
}
}
// Also check values used in this export batch
for (const usedValue of usedValues) {
if (typeof usedValue === 'number' && usedValue >= currentValue) {
existingNumbers.add(usedValue)
}
}
// Check if current value is already unique
const isCurrentValueUnique = !existingNumbers.has(currentValue) && !usedValues.has(currentValue)
if (isCurrentValueUnique) {
// Current value is unique, keep it
uniqueValue = currentValue
} else {
// Find max number and use max + 1
const maxNum = existingNumbers.size > 0 ? Math.max(...Array.from(existingNumbers)) : currentValue - 1
uniqueValue = maxNum + 1
this.context.info(
`ensureFieldUnique: ${fieldName} value ${currentValue} conflicts, generating new value: ${uniqueValue}`
)
}
}
} else {
// Unsupported type, skip uniqueness check
this.context.warn(`Cannot ensure uniqueness for field ${fieldName} with type ${typeof currentValue}`)
return
}
// Update data with unique value
data[fieldName] = uniqueValue
usedValues.add(uniqueValue)
if (uniqueValue !== currentValue) {
this.context.info(
`ensureFieldUnique: Updated ${fieldName} from ${currentValue} to ${uniqueValue} (class: ${docClass})`
)
}
}
/**
* Recursively remap a value, handling nested objects and arrays.
*/
@@ -63,4 +63,6 @@ export interface ExportState {
idMapping: Map<Ref<Doc>, Ref<Doc>>
spaceMapping: Map<Ref<Space>, Ref<Space>>
processingDocs: Set<Ref<Doc>>
// Track unique field values per class: { className: { fieldName: Set<values> } }
uniqueFieldValues: Map<string, Map<string, Set<string | number>>>
}
@@ -60,7 +60,8 @@ export class CrossWorkspaceExporter {
this.state = {
idMapping: new Map<Ref<Doc>, Ref<Doc>>(),
spaceMapping: new Map<Ref<Space>, Ref<Space>>(),
processingDocs: new Set<Ref<Doc>>()
processingDocs: new Set<Ref<Doc>>(),
uniqueFieldValues: new Map()
}
// Initialize component exporters