diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts index 1003a81635..14a8d357b4 100644 --- a/dev/tool/src/clean.ts +++ b/dev/tool/src/clean.ts @@ -1270,7 +1270,7 @@ async function updateYDoc ( doc: RelatedDocument ): Promise { try { - const ydoc = await loadCollaborativeDoc(storage, workspaceId, _id, ctx) + const ydoc = await loadCollaborativeDoc(ctx, storage, workspaceId, _id) if (ydoc === undefined) { ctx.error('document content not found', { document: contentDoc._id }) return @@ -1284,7 +1284,7 @@ async function updateYDoc ( }) if (updatedYDoc !== undefined) { - await saveCollaborativeDoc(storage, workspaceId, _id, updatedYDoc, ctx) + await saveCollaborativeDoc(ctx, storage, workspaceId, _id, updatedYDoc) } } catch { // do nothing, the collaborative doc does not sem to exist yet diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index 4783bd6d20..c898757a54 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -199,7 +199,7 @@ async function processMigrateMarkupFor ( if (blob === undefined) { try { const ydoc = markupToYDoc(value, attribute.name) - await saveCollaborativeDoc(storageAdapter, workspaceId, collaborativeDoc, ydoc, ctx) + await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, collaborativeDoc, ydoc) } catch (err) { console.error('failed to process document', doc._class, doc._id, err) } @@ -298,7 +298,7 @@ export async function restoreLostMarkup ( console.log(doc._class, doc._id, attr.name, markup) if (command === 'restore') { const ydoc = markupToYDoc(markup, attr.name) - await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx) + await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, value, ydoc) } restored = true break @@ -329,7 +329,7 @@ export async function restoreLostMarkup ( console.log(doc._class, doc._id, attr.name, markup) if (command === 'restore') { const ydoc = markupToYDoc(markup, attr.name) - await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx) + await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, value, ydoc) } } } diff --git a/models/controlled-documents/src/migration.ts b/models/controlled-documents/src/migration.ts index 0672f170cb..0135c160ed 100644 --- a/models/controlled-documents/src/migration.ts +++ b/models/controlled-documents/src/migration.ts @@ -292,7 +292,7 @@ async function migrateDocSections (client: MigrationClient): Promise { // Migrate sections headers + content try { - const ydoc = await loadCollaborativeDoc(storage, client.workspaceId, document.content, ctx) + const ydoc = await loadCollaborativeDoc(ctx, storage, client.workspaceId, document.content) if (ydoc === undefined) { ctx.error('collaborative document content not found', { document: document.title }) continue @@ -334,7 +334,7 @@ async function migrateDocSections (client: MigrationClient): Promise { } }) - await saveCollaborativeDoc(storage, client.workspaceId, document.content, ydoc, ctx) + await saveCollaborativeDoc(ctx, storage, client.workspaceId, document.content, ydoc) } catch (err) { ctx.error('error collaborative document content migration', { error: err, document: document.title }) } diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 5645eec853..29e7311bf0 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -229,7 +229,7 @@ async function processMigrateContentFor ( if (blob === undefined) { try { const ydoc = markupToYDoc(value, attribute.name) - await saveCollaborativeDoc(storageAdapter, client.workspaceId, collaborativeDoc, ydoc, ctx) + await saveCollaborativeDoc(ctx, storageAdapter, client.workspaceId, collaborativeDoc, ydoc) } catch (err) { console.error('failed to process document', doc._class, doc._id, err) } diff --git a/models/document/src/migration.ts b/models/document/src/migration.ts index 39792b4b56..4abfd77e50 100644 --- a/models/document/src/migration.ts +++ b/models/document/src/migration.ts @@ -100,36 +100,6 @@ async function migrateTeamspacesMixins (client: MigrationClient): Promise ) } -async function migrateContentField (client: MigrationClient): Promise { - const ctx = new MeasureMetricsContext('migrate_content_field', {}) - const storage = client.storageAdapter - - const documents = await client.find(DOMAIN_DOCUMENT, { - _class: document.class.Document, - content: { $exists: true } - }) - - for (const document of documents) { - try { - const ydoc = await loadCollaborativeDoc(storage, client.workspaceId, document.content, ctx) - if (ydoc === undefined) { - ctx.error('document content not found', { document: document.title }) - continue - } - - if (!ydoc.share.has('') || ydoc.share.has('content')) { - continue - } - - yDocCopyXmlField(ydoc, '', 'content') - - await saveCollaborativeDoc(storage, client.workspaceId, document.content, ydoc, ctx) - } catch (err) { - ctx.error('error document content migration', { error: err, document: document.title }) - } - } -} - async function migrateRank (client: MigrationClient): Promise { const documents = await client.find( DOMAIN_DOCUMENT, @@ -228,7 +198,7 @@ async function renameFieldsRevert (client: MigrationClient): Promise { if (document.description.includes('%description:')) { try { - const ydoc = await loadCollaborativeDoc(storage, client.workspaceId, document.description, ctx) + const ydoc = await loadCollaborativeDoc(ctx, storage, client.workspaceId, document.description) if (ydoc === undefined) { continue } @@ -239,7 +209,7 @@ async function renameFieldsRevert (client: MigrationClient): Promise { yDocCopyXmlField(ydoc, 'description', 'content') - await saveCollaborativeDoc(storage, client.workspaceId, document.description, ydoc, ctx) + await saveCollaborativeDoc(ctx, storage, client.workspaceId, document.description, ydoc) } catch (err) { ctx.error('error document content migration', { error: err, document: document.title }) } @@ -264,6 +234,42 @@ async function renameFieldsRevert (client: MigrationClient): Promise { } } +async function restoreContentField (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('restoreContentField', {}) + const storage = client.storageAdapter + + const documents = await client.find(DOMAIN_DOCUMENT, { + _class: document.class.Document, + content: { $exists: true } + }) + + for (const document of documents) { + try { + const ydoc = await loadCollaborativeDoc(ctx, storage, client.workspaceId, document.content) + if (ydoc === undefined) { + ctx.error('document content not found', { document: document.title }) + continue + } + + // ignore if content is already present + if (ydoc.share.has('content') || ydoc.share.has('description')) { + continue + } + + if (ydoc.share.has('')) { + yDocCopyXmlField(ydoc, '', 'content') + if (ydoc.share.has('content')) { + await saveCollaborativeDoc(ctx, storage, client.workspaceId, document.content, ydoc) + } else { + ctx.error('document content still not found', { document: document.title }) + } + } + } catch (err) { + ctx.error('error document content migration', { error: err, document: document.title }) + } + } +} + export const documentOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, documentId, [ @@ -279,10 +285,6 @@ export const documentOperation: MigrateOperation = { state: 'migrate-teamspaces-mixins', func: migrateTeamspacesMixins }, - { - state: 'migrateContentField', - func: migrateContentField - }, { state: 'migrateRank', func: migrateRank @@ -300,6 +302,10 @@ export const documentOperation: MigrateOperation = { { state: 'renameFieldsRevert', func: renameFieldsRevert + }, + { + state: 'restoreContentField', + func: restoreContentField } ]) }, diff --git a/server-plugins/activity-resources/src/references.ts b/server-plugins/activity-resources/src/references.ts index f260299f70..34323e191c 100644 --- a/server-plugins/activity-resources/src/references.ts +++ b/server-plugins/activity-resources/src/references.ts @@ -415,7 +415,7 @@ async function getCreateReferencesTxes ( } else if (attr.type._class === core.class.TypeCollaborativeDoc) { const collaborativeDoc = (createdDoc as any)[attr.name] as CollaborativeDoc try { - const ydoc = await loadCollaborativeDoc(storage, control.workspace, collaborativeDoc, control.ctx) + const ydoc = await loadCollaborativeDoc(ctx, storage, control.workspace, collaborativeDoc) if (ydoc !== undefined) { const attrReferences = getReferencesData( srcDocId, @@ -467,7 +467,7 @@ async function getUpdateReferencesTxes ( hasReferenceAttrs = true try { const collaborativeDoc = (updatedDoc as any)[attr.name] as CollaborativeDoc - const ydoc = await loadCollaborativeDoc(storage, control.workspace, collaborativeDoc, control.ctx) + const ydoc = await loadCollaborativeDoc(ctx, storage, control.workspace, collaborativeDoc) if (ydoc !== undefined) { const attrReferences = getReferencesData( srcDocId, diff --git a/server/collaboration/src/utils/__tests__/ydoc.test.ts b/server/collaboration/src/utils/__tests__/ydoc.test.ts index 6e599642b9..6dc0acaeb4 100644 --- a/server/collaboration/src/utils/__tests__/ydoc.test.ts +++ b/server/collaboration/src/utils/__tests__/ydoc.test.ts @@ -43,10 +43,12 @@ describe('ydoc', () => { const source = ydoc.getXmlFragment('source') source.insertAfter(null, [new YXmlElement('p'), new YXmlText('foo'), new YXmlElement('p')]) + expect(ydoc.share.has('target')).toBeFalsy() yDocCopyXmlField(ydoc, 'source', 'target') const target = ydoc.getXmlFragment('target') + expect(ydoc.share.has('target')).toBeTruthy() expect(target.toJSON()).toEqual(source.toJSON()) }) @@ -61,6 +63,7 @@ describe('ydoc', () => { expect(target.toJSON()).not.toEqual(source.toJSON()) yDocCopyXmlField(ydoc, 'source', 'target') + expect(ydoc.share.has('target')).toBeTruthy() expect(target.toJSON()).toEqual(source.toJSON()) }) }) diff --git a/server/collaboration/src/utils/collaborative-doc.ts b/server/collaboration/src/utils/collaborative-doc.ts index 8ec043493d..037279491a 100644 --- a/server/collaboration/src/utils/collaborative-doc.ts +++ b/server/collaboration/src/utils/collaborative-doc.ts @@ -78,10 +78,10 @@ async function loadCollaborativeDocVersion ( /** @public */ export async function loadCollaborativeDoc ( + ctx: MeasureContext, storageAdapter: StorageAdapter, workspace: WorkspaceId, - collaborativeDoc: CollaborativeDoc, - ctx: MeasureContext + collaborativeDoc: CollaborativeDoc ): Promise { const sources = collaborativeDocUnchain(collaborativeDoc) @@ -101,24 +101,24 @@ export async function loadCollaborativeDoc ( /** @public */ export async function saveCollaborativeDoc ( + ctx: MeasureContext, storageAdapter: StorageAdapter, workspace: WorkspaceId, collaborativeDoc: CollaborativeDoc, - ydoc: YDoc, - ctx: MeasureContext + ydoc: YDoc ): Promise { const { documentId, versionId } = collaborativeDocParse(collaborativeDoc) - await saveCollaborativeDocVersion(storageAdapter, workspace, documentId, versionId, ydoc, ctx) + await saveCollaborativeDocVersion(ctx, storageAdapter, workspace, documentId, versionId, ydoc) } /** @public */ export async function saveCollaborativeDocVersion ( + ctx: MeasureContext, storageAdapter: StorageAdapter, workspace: WorkspaceId, documentId: string, versionId: CollaborativeDocVersion, - ydoc: YDoc, - ctx: MeasureContext + ydoc: YDoc ): Promise { await ctx.with('saveCollaborativeDoc', {}, async (ctx) => { if (versionId === 'HEAD') { diff --git a/server/collaborator/src/storage/platform.ts b/server/collaborator/src/storage/platform.ts index 7a69e94758..49ddea1973 100644 --- a/server/collaborator/src/storage/platform.ts +++ b/server/collaborator/src/storage/platform.ts @@ -124,7 +124,7 @@ export class PlatformStorageAdapter implements CollabStorageAdapter { return await ctx.with('load-document', {}, async (ctx) => { return await withRetry(ctx, 5, async () => { - return await loadCollaborativeDoc(this.storage, context.workspaceId, collaborativeDoc, ctx) + return await loadCollaborativeDoc(ctx, this.storage, context.workspaceId, collaborativeDoc) }) }) } @@ -139,7 +139,7 @@ export class PlatformStorageAdapter implements CollabStorageAdapter { await ctx.with('save-document', {}, async (ctx) => { await withRetry(ctx, 5, async () => { - await saveCollaborativeDoc(this.storage, context.workspaceId, collaborativeDoc, document, ctx) + await saveCollaborativeDoc(ctx, this.storage, context.workspaceId, collaborativeDoc, document) }) }) } diff --git a/server/tool/src/initializer.ts b/server/tool/src/initializer.ts index 3b2aa0e263..e2782444aa 100644 --- a/server/tool/src/initializer.ts +++ b/server/tool/src/initializer.ts @@ -272,7 +272,7 @@ export class WorkspaceInitializer { const json = parseMessageMarkdown(data ?? '', this.imageUrl) const yDoc = jsonToYDocNoSchema(json, field) - await saveCollaborativeDoc(this.storageAdapter, this.wsUrl, collabId, yDoc, this.ctx) + await saveCollaborativeDoc(this.ctx, this.storageAdapter, this.wsUrl, collabId, yDoc) return collabId }