diff --git a/.vscode/launch.json b/.vscode/launch.json index 47bd54f286..cb08a80418 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -286,12 +286,14 @@ "MINIO_ENDPOINT": "localhost:9000", "TRANSACTOR_URL": "ws://localhost:3333", "MONGO_URL": "mongodb://localhost:27017", + "DB_URL": "mongodb://localhost:27017", "ACCOUNTS_URL": "http://localhost:3000", "TELEGRAM_DATABASE": "telegram-service", "ELASTIC_URL": "http://localhost:9200", "REKONI_URL": "http://localhost:4004", "MODEL_VERSION": "0.6.287" }, + "runtimeVersion": "20", "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "sourceMaps": true, "outputCapture": "std", diff --git a/dev/doc-import-tool/src/import.ts b/dev/doc-import-tool/src/import.ts index f8c414d409..c76a8d0b9c 100644 --- a/dev/doc-import-tool/src/import.ts +++ b/dev/doc-import-tool/src/import.ts @@ -15,14 +15,13 @@ import documents, { import core, { AttachedData, BackupClient, - CollaborativeDoc, Client as CoreClient, Data, MeasureContext, Ref, TxOperations, generateId, - makeCollaborativeDoc, + makeDocCollabId, systemAccountEmail, type Blob } from '@hcengineering/core' @@ -101,7 +100,7 @@ async function createDocument ( abstract: '', effectiveDate: 0, reviewInterval: DEFAULT_PERIODIC_REVIEW_INTERVAL, - content: makeCollaborativeDoc(generateId()), + content: null, snapshots: 0, plannedEffectiveDate: 0 } @@ -115,11 +114,6 @@ async function createDocument ( console.log('Creating controlled doc from template') - const copyContent = async (source: CollaborativeDoc, target: CollaborativeDoc): Promise => { - // intentionally left empty - // even though the template has some content, it won't be used - } - const { success } = await createControlledDocFromTemplate( txops, templateId, @@ -128,8 +122,7 @@ async function createDocument ( space, undefined, undefined, - documents.class.ControlledDocument, - copyContent + documents.class.ControlledDocument ) if (!success) { throw new Error('Failed to create controlled document from template') @@ -184,7 +177,7 @@ async function createTemplateIfNotExist ( approvers: [], coAuthors: [], changeControl: ccRecordId, - content: makeCollaborativeDoc(generateId()), + content: null, snapshots: 0, plannedEffectiveDate: 0 } @@ -229,9 +222,7 @@ async function createSections ( console.log('Creating document content') - const collabId = doc.content - - console.log(`Collab doc ID: ${collabId}`) + const collabId = makeDocCollabId(doc, 'content') try { let content: string = '' @@ -245,7 +236,7 @@ async function createSections ( content += `

${section.title}

${section.content}` } - await collaborator.updateContent(collabId, { content }) + await collaborator.updateMarkup(collabId, content) } finally { // do nothing } diff --git a/dev/import-tool/src/importer/importer.ts b/dev/import-tool/src/importer/importer.ts index 13d2363f08..93f285ee70 100644 --- a/dev/import-tool/src/importer/importer.ts +++ b/dev/import-tool/src/importer/importer.ts @@ -14,18 +14,18 @@ // import attachment, { type Attachment } from '@hcengineering/attachment' import chunter, { type ChatMessage } from '@hcengineering/chunter' -import { yDocToBuffer } from '@hcengineering/collaboration' import { type Person } from '@hcengineering/contact' import core, { type Account, type AttachedData, type Class, + type Blob as PlatformBlob, type CollaborativeDoc, type Data, type Doc, type DocumentQuery, generateId, - makeCollaborativeDoc, + makeCollabId, type Mixin, type Ref, SortingOrder, @@ -42,7 +42,7 @@ import task, { type TaskType, type TaskTypeWithFactory } from '@hcengineering/task' -import { jsonToMarkup, jsonToYDocNoSchema, parseMessageMarkdown } from '@hcengineering/text' +import { jsonToMarkup, parseMessageMarkdown } from '@hcengineering/text' import tracker, { type Issue, type IssueParentInfo, @@ -52,7 +52,7 @@ import tracker, { TimeReportDayType } from '@hcengineering/tracker' import { type MarkdownPreprocessor, NoopMarkdownPreprocessor } from './preprocessor' -import { type FileUploader, type UploadResult } from './uploader' +import { type FileUploader } from './uploader' export interface ImportWorkspace { projectTypes?: ImportProjectType[] @@ -281,14 +281,15 @@ export class WorkspaceImporter { ): Promise> { const id = doc.id ?? generateId() const content = await doc.descrProvider() - const collabId = await this.createCollaborativeContent(id, 'content', content, teamspaceId) + const collabId = makeCollabId(document.class.Document, id, 'content') + const contentId = await this.createCollaborativeContent(id, collabId, content, teamspaceId) const lastRank = await getFirstRank(this.client, teamspaceId, parentId) const rank = makeRank(lastRank, undefined) const attachedData: Data = { title: doc.title, - content: collabId, + content: contentId, parent: parentId, attachments: 0, embeddings: 0, @@ -391,7 +392,8 @@ export class WorkspaceImporter { ): Promise<{ id: Ref, identifier: string }> { const issueId = issue.id ?? generateId() const content = await issue.descrProvider() - const collabId = await this.createCollaborativeContent(issueId, 'description', content, project._id) + const collabId = makeCollabId(tracker.class.Issue, issueId, 'description') + const contentId = await this.createCollaborativeContent(issueId, collabId, content, project._id) const { number, identifier } = issue.number !== undefined @@ -412,7 +414,7 @@ export class WorkspaceImporter { const issueData: AttachedData = { title: issue.title, - description: collabId, + description: contentId, assignee: issue.assignee ?? null, component: null, number, @@ -532,14 +534,10 @@ export class WorkspaceImporter { } const file = new File([blob], attachment.title) - const attachmentId = await this.createAttachment( - attachment.id ?? generateId(), - file, - spaceId, - parentId, - parentClass - ) - if (attachmentId === null) { + + try { + await this.createAttachment(attachment.id ?? generateId(), file, spaceId, parentId, parentClass) + } catch { console.warn('Failed to upload attachment file: ', attachment.title) } } @@ -550,22 +548,9 @@ export class WorkspaceImporter { spaceId: Ref, parentId: Ref, parentClass: Ref>> - ): Promise | null> { - const response = await this.fileUploader.uploadFile(id, id, file) - if (response.status !== 200) { - return null - } - - const responseText = await response.text() - if (responseText === undefined) { - return null - } - - const uploadResult = JSON.parse(responseText) as UploadResult[] - if (!Array.isArray(uploadResult) || uploadResult.length === 0) { - return null - } - + ): Promise> { + const attachmentId = generateId() + const blobId = await this.fileUploader.uploadFile(id, file) await this.client.addCollection( attachment.class.Attachment, spaceId, @@ -573,7 +558,7 @@ export class WorkspaceImporter { parentClass, 'attachments', { - file: uploadResult[0].id, + file: blobId, lastModified: Date.now(), name: file.name, size: file.size, @@ -581,25 +566,23 @@ export class WorkspaceImporter { }, id ) - return id + return attachmentId } // Collaborative content handling private async createCollaborativeContent ( id: Ref, - field: string, + collabId: CollaborativeDoc, content: string, spaceId: Ref - ): Promise { + ): Promise> { const json = parseMessageMarkdown(content ?? '', 'image://') const processedJson = this.preprocessor.process(json, id, spaceId) - const collabId = makeCollaborativeDoc(id, 'description') - const yDoc = jsonToYDocNoSchema(processedJson, field) - const buffer = yDocToBuffer(yDoc) + const markup = jsonToMarkup(processedJson) + const buffer = Buffer.from(markup) - await this.fileUploader.uploadCollaborativeDoc(id, collabId, buffer) - return collabId + return await this.fileUploader.uploadCollaborativeDoc(collabId, buffer) } async findIssueStatusByName (name: string): Promise> { diff --git a/dev/import-tool/src/importer/uploader.ts b/dev/import-tool/src/importer/uploader.ts index 64269f1176..d2ae4fb7d9 100644 --- a/dev/import-tool/src/importer/uploader.ts +++ b/dev/import-tool/src/importer/uploader.ts @@ -13,25 +13,19 @@ // limitations under the License. // import { - concatLink, type Ref, type Blob as PlatformBlob, - type Doc, type CollaborativeDoc, - collaborativeDocParse + concatLink, + makeCollabJsonId } from '@hcengineering/core' export interface FileUploader { - uploadFile: (id: Ref, name: string, file: File, contentType?: string) => Promise - uploadCollaborativeDoc: (id: Ref, collabId: CollaborativeDoc, data: Buffer) => Promise + uploadFile: (name: string, file: Blob) => Promise> + uploadCollaborativeDoc: (collabId: CollaborativeDoc, data: Buffer) => Promise> getFileUrl: (id: string) => string } -export interface UploadResult { - key: 'file' - id: Ref -} - export class FrontFileUploader implements FileUploader { constructor ( private readonly frontUrl: string, @@ -41,31 +35,32 @@ export class FrontFileUploader implements FileUploader { this.getFileUrl = this.getFileUrl.bind(this) } - public async uploadFile (id: Ref, name: string, file: File, contentType?: string): Promise { + public async uploadFile (name: string, file: Blob): Promise> { const form = new FormData() form.append('file', file, name) - form.append('type', contentType ?? file.type) - form.append('size', file.size.toString()) - form.append('name', file.name) - form.append('id', id) - form.append('data', new Blob([file])) - return await fetch(concatLink(this.frontUrl, '/files'), { + const res = await fetch(concatLink(this.frontUrl, '/files'), { method: 'POST', headers: { Authorization: 'Bearer ' + this.token }, body: form }) + + if (res.ok && res.status === 200) { + return name as Ref + } + + throw new Error('Failed to upload file') } public getFileUrl (id: string): string { return concatLink(this.frontUrl, `/files/${this.workspaceId}/${id}?file=${id}&workspace=${this.workspaceId}`) } - public async uploadCollaborativeDoc (id: Ref, collabId: CollaborativeDoc, data: Buffer): Promise { - const file = new File([data], collabId) - const { documentId } = collaborativeDocParse(collabId) - return await this.uploadFile(id, documentId, file, 'application/ydoc') + public async uploadCollaborativeDoc (collabId: CollaborativeDoc, data: Buffer): Promise> { + const blobId = makeCollabJsonId(collabId) + const blob = new Blob([data], { type: 'application/json' }) + return await this.uploadFile(blobId, blob) } } diff --git a/dev/import-tool/src/notion/notion.ts b/dev/import-tool/src/notion/notion.ts index 7af6e37082..aff260f759 100644 --- a/dev/import-tool/src/notion/notion.ts +++ b/dev/import-tool/src/notion/notion.ts @@ -12,21 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. // -import { yDocToBuffer } from '@hcengineering/collaboration' import { type AttachedData, type Blob, type Data, - type Doc, - generateId, - makeCollaborativeDoc, type Ref, - type TxOperations + type TxOperations, + generateId, + makeCollabId } from '@hcengineering/core' -import document, { type Document, getFirstRank, type Teamspace } from '@hcengineering/document' +import document, { type Document, type Teamspace, getFirstRank } from '@hcengineering/document' import { makeRank } from '@hcengineering/rank' import { - jsonToYDocNoSchema, + jsonToMarkup, MarkupMarkType, type MarkupNode, MarkupNodeType, @@ -325,8 +323,6 @@ async function createDBPageWithAttachments ( documentMetaMap?: Map ): Promise { const pageId = docMeta.id as Ref - const collabId = makeCollaborativeDoc(pageId, 'content') - const parentId = parentMeta !== undefined ? (parentMeta.id as Ref) : document.ids.NoParent const lastRank = await getFirstRank(client, space, parentId) @@ -334,7 +330,7 @@ async function createDBPageWithAttachments ( const object: Data = { title: docMeta.name, - content: collabId, + content: null, parent: parentId, attachments: 0, embeddings: 0, @@ -409,7 +405,7 @@ async function importAttachment ( } const file = new File([data], docMeta.name) - await fileUploader.uploadFile(docMeta.id as Ref, docMeta.id, file) + await fileUploader.uploadFile(docMeta.id, file) const attachedData: AttachedData = { file: docMeta.id as Ref, @@ -444,13 +440,12 @@ async function importPageDocument ( if (documentMetaMap !== undefined) { preProcessMarkdown(json, documentMetaMap, fileUploader) } - const yDoc = jsonToYDocNoSchema(json, 'content') - const buffer = yDocToBuffer(yDoc) + const markup = jsonToMarkup(json) + const buffer = Buffer.from(markup) const id = docMeta.id as Ref - const collabId = makeCollaborativeDoc(id, 'description') - - await fileUploader.uploadCollaborativeDoc(id, collabId, buffer) + const collabId = makeCollabId(document.class.Document, id, 'content') + const blobId = await fileUploader.uploadCollaborativeDoc(collabId, buffer) const parent = (parentMeta?.id as Ref) ?? document.ids.NoParent @@ -459,7 +454,7 @@ async function importPageDocument ( const attachedData: Data = { title: docMeta.name, - content: collabId, + content: blobId, parent, attachments: 0, embeddings: 0, diff --git a/dev/tool/src/benchmark.ts b/dev/tool/src/benchmark.ts index fcae5fd39f..7143746c0e 100644 --- a/dev/tool/src/benchmark.ts +++ b/dev/tool/src/benchmark.ts @@ -21,7 +21,6 @@ import core, { concatLink, generateId, getWorkspaceId, - makeCollaborativeDoc, metricsToString, newMetrics, systemAccountEmail, @@ -594,7 +593,7 @@ async function generateVacancy (client: TxOperations, members: Ref { try { - const ydoc = await loadCollaborativeDoc(ctx, storage, workspaceId, _id) + const ydoc = await loadCollabYdoc(ctx, storage, workspaceId, _id) if (ydoc === undefined) { ctx.error('document content not found', { document: contentDoc._id }) return @@ -1264,7 +1265,7 @@ async function updateYDoc ( }) if (updatedYDoc !== undefined) { - await saveCollaborativeDoc(ctx, storage, workspaceId, _id, updatedYDoc) + await saveCollabYdoc(ctx, storage, workspaceId, _id, updatedYDoc) } } catch { // do nothing, the collaborative doc does not sem to exist yet diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index d49d0380ad..829c9eca48 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -58,13 +58,7 @@ import serverClientPlugin, { listAccountWorkspaces, updateBackupInfo } from '@hcengineering/server-client' -import { - createBackupPipeline, - getConfig, - getServerPipeline, - registerServerPlugins, - registerStringLoaders -} from '@hcengineering/server-pipeline' +import { createBackupPipeline, getConfig } from '@hcengineering/server-pipeline' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool' import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' @@ -89,8 +83,7 @@ import core, { type Ref, type Tx, type Version, - type WorkspaceId, - type WorkspaceIdWithUrl + type WorkspaceId } from '@hcengineering/core' import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model' import contact from '@hcengineering/model-contact' @@ -123,7 +116,6 @@ import { } from './clean' import { changeConfiguration } from './configuration' import { moveAccountDbFromMongoToPG, moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db' -import { fixJsonMarkup, migrateMarkup } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { fixAccountEmails, renameAccount } from './renameAccount' import { moveFiles, showLostFiles } from './storage' @@ -1723,63 +1715,6 @@ export function devTool ( }) }) - program - .command('fix-json-markup-mongo ') - .description('fixes double converted json markup') - .action(async (workspace: string) => { - const mongodbUri = getMongoDBUrl() - await withStorage(async (adapter) => { - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await fixJsonMarkup(toolCtx, mongodbUri, adapter, wsid, endpoint) - }) - }) - - program - .command('migrate-markup-mongo') - .description('migrates collaborative markup to storage') - .option('-w, --workspace ', 'Selected workspace only', '') - .option('-c, --concurrency ', 'Number of documents being processed concurrently', '10') - .action(async (cmd: { workspace: string, concurrency: string }) => { - const { dbUrl, txes } = prepareTools() - const mongodbUri = getMongoDBUrl() - await withDatabase(dbUrl, async (db) => { - await withStorage(async (adapter) => { - const workspaces = await listWorkspacesPure(db) - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() - let index = 0 - try { - for (const workspace of workspaces) { - if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - continue - } - - const wsId = getWorkspaceId(workspace.workspace) - console.log('processing workspace', workspace.workspace, index, workspaces.length) - const wsUrl: WorkspaceIdWithUrl = { - name: workspace.workspace, - workspaceName: workspace.workspaceName ?? '', - workspaceUrl: workspace.workspaceUrl ?? '' - } - - registerServerPlugins() - registerStringLoaders() - - const { pipeline } = await getServerPipeline(toolCtx, txes, dbUrl, wsUrl) - - await migrateMarkup(toolCtx, adapter, wsId, _client, pipeline, parseInt(cmd.concurrency)) - - console.log('...done', workspace.workspace) - index++ - } - } finally { - client.close() - } - }) - }) - }) - program .command('remove-duplicates-ids-mongo ') .description('remove duplicates ids for futue migration') diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts deleted file mode 100644 index 649d78e959..0000000000 --- a/dev/tool/src/markup.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { saveCollaborativeDoc } from '@hcengineering/collaboration' -import core, { - type AnyAttribute, - type Class, - type Client as CoreClient, - type Doc, - type Domain, - type Hierarchy, - type MeasureContext, - type Ref, - type WorkspaceId, - collaborativeDocParse, - makeCollaborativeDoc, - RateLimiter -} from '@hcengineering/core' -import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' -import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core' -import { connect } from '@hcengineering/server-tool' -import { jsonToText, markupToYDoc } from '@hcengineering/text' -import { type Db, type FindCursor, type MongoClient } from 'mongodb' - -export async function fixJsonMarkup ( - ctx: MeasureContext, - mongoUrl: string, - storageAdapter: StorageAdapter, - workspaceId: WorkspaceId, - transactorUrl: string -): Promise { - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup' - })) as unknown as CoreClient - const hierarchy = connection.getHierarchy() - - const client = getMongoClient(mongoUrl) - const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, workspaceId) - - try { - const classes = hierarchy.getDescendants(core.class.Doc) - for (const _class of classes) { - const domain = hierarchy.findDomain(_class) - if (domain === undefined) continue - - const attributes = hierarchy.getAllAttributes(_class) - const filtered = Array.from(attributes.values()).filter((attribute) => { - return hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) - }) - if (filtered.length === 0) continue - - await processFixJsonMarkupFor(ctx, domain, _class, filtered, workspaceId, db, storageAdapter) - } - } finally { - client.close() - await connection.close() - } -} - -async function processFixJsonMarkupFor ( - ctx: MeasureContext, - domain: Domain, - _class: Ref>, - attributes: AnyAttribute[], - workspaceId: WorkspaceId, - db: Db, - storageAdapter: StorageAdapter -): Promise { - const collection = db.collection(domain) - const docs = await collection.find({ _class }).toArray() - for (const doc of docs) { - const update: Record = {} - const remove = [] - - for (const attribute of attributes) { - try { - const value = (doc as any)[attribute.name] - if (value != null) { - let res = value - while (true) { - try { - const json = JSON.parse(res) - const text = jsonToText(json) - JSON.parse(text) - res = text - } catch { - break - } - } - if (res !== value) { - update[attribute.name] = res - remove.push(makeCollaborativeDoc(doc._id, attribute.name)) - } - } - } catch {} - } - - if (Object.keys(update).length > 0) { - try { - await collection.updateOne({ _id: doc._id }, { $set: update }) - } catch (err) { - console.error('failed to update document', doc._class, doc._id, err) - } - } - - if (remove.length > 0) { - try { - await storageAdapter.remove(ctx, workspaceId, remove) - } catch (err) { - console.error('failed to remove objects from storage', doc._class, doc._id, remove, err) - } - } - } -} - -export async function migrateMarkup ( - ctx: MeasureContext, - storageAdapter: StorageAdapter, - workspaceId: WorkspaceId, - client: MongoClient, - pipeline: Pipeline, - concurrency: number -): Promise { - const hierarchy = pipeline.context.hierarchy - - const workspaceDb = client.db(workspaceId.name) - - const classes = hierarchy.getDescendants(core.class.Doc) - for (const _class of classes) { - const domain = hierarchy.findDomain(_class) - if (domain === undefined) continue - - const allAttributes = hierarchy.getAllAttributes(_class) - const attributes = Array.from(allAttributes.values()).filter((attribute) => { - return hierarchy.isDerived(attribute.type._class, 'core:class:TypeCollaborativeMarkup' as Ref>) - }) - - if (attributes.length === 0) continue - if (hierarchy.isMixin(_class) && attributes.every((p) => p.attributeOf !== _class)) continue - - const collection = workspaceDb.collection(domain) - - const filter = hierarchy.isMixin(_class) ? { [_class]: { $exists: true } } : { _class } - const iterator = collection.find(filter) - - try { - await processMigrateMarkupFor(ctx, hierarchy, storageAdapter, workspaceId, attributes, iterator, concurrency) - } finally { - await iterator.close() - } - } -} - -async function processMigrateMarkupFor ( - ctx: MeasureContext, - hierarchy: Hierarchy, - storageAdapter: StorageAdapter, - workspaceId: WorkspaceId, - attributes: AnyAttribute[], - iterator: FindCursor, - concurrency: number -): Promise { - const rateLimiter = new RateLimiter(concurrency) - - let processed = 0 - - while (true) { - const doc = await iterator.next() - if (doc === null) break - - const timestamp = Date.now() - const revisionId = `${timestamp}` - - await rateLimiter.exec(async () => { - for (const attribute of attributes) { - const collaborativeDoc = makeCollaborativeDoc(doc._id, attribute.name, revisionId) - const { documentId } = collaborativeDocParse(collaborativeDoc) - - const value = hierarchy.isMixin(attribute.attributeOf) - ? ((doc as any)[attribute.attributeOf]?.[attribute.name] as string) - : ((doc as any)[attribute.name] as string) - - if (value != null && value.startsWith('{')) { - const blob = await storageAdapter.stat(ctx, workspaceId, documentId) - // only for documents not in storage - if (blob === undefined) { - try { - const ydoc = markupToYDoc(value, attribute.name) - await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, collaborativeDoc, ydoc) - } catch (err) { - console.error('failed to process document', doc._class, doc._id, err) - } - } - } - } - }) - - processed += 1 - - if (processed % 100 === 0) { - await rateLimiter.waitProcessing() - console.log('...processing', processed) - } - } - - await rateLimiter.waitProcessing() - - console.log('processed', processed) -} diff --git a/models/contact/src/index.ts b/models/contact/src/index.ts index e7a597a31a..6bcaa00a87 100644 --- a/models/contact/src/index.ts +++ b/models/contact/src/index.ts @@ -39,7 +39,7 @@ import { IndexKind, type Blob, type Class, - type CollaborativeDoc, + type MarkupBlobRef, type Domain, type Ref, type Timestamp @@ -175,7 +175,7 @@ export class TMember extends TAttachedDoc implements Member { export class TOrganization extends TContact implements Organization { @Prop(TypeCollaborativeDoc(), core.string.Description) @Index(IndexKind.FullText) - description!: CollaborativeDoc + description!: MarkupBlobRef | null @Prop(Collection(contact.class.Member), contact.string.Members) members!: number diff --git a/models/controlled-documents/src/migration.ts b/models/controlled-documents/src/migration.ts index a609ba304f..c199e79fb0 100644 --- a/models/controlled-documents/src/migration.ts +++ b/models/controlled-documents/src/migration.ts @@ -4,12 +4,12 @@ import attachment, { type Attachment } from '@hcengineering/attachment' import { - clone, - loadCollaborativeDoc, - saveCollaborativeDoc, - YAbstractType, YXmlElement, - YXmlText + YXmlText, + YAbstractType, + yXmlElementClone, + loadCollabYdoc, + saveCollabYdoc } from '@hcengineering/collaboration' import { type ChangeControl, @@ -26,7 +26,7 @@ import { type Doc, DOMAIN_TX, generateId, - makeCollaborativeDoc, + makeDocCollabId, MeasureMetricsContext, type Ref, SortingOrder, @@ -144,7 +144,7 @@ async function createProductChangeControlTemplate (tx: TxOperations): Promise { // Migrate sections headers + content try { - const ydoc = await loadCollaborativeDoc(ctx, storage, client.workspaceId, document.content) + const collabId = makeDocCollabId(document, 'content') + const ydoc = await loadCollabYdoc(ctx, storage, client.workspaceId, collabId) if (ydoc === undefined) { // no content, ignore continue @@ -331,13 +332,17 @@ async function migrateDocSections (client: MigrationClient): Promise { ...(sectionContent .toArray() .map((item) => - item instanceof YAbstractType ? (item instanceof YXmlElement ? clone(item) : item.clone()) : item + item instanceof YAbstractType + ? item instanceof YXmlElement + ? yXmlElementClone(item) + : item.clone() + : item ) as any) ]) } }) - await saveCollaborativeDoc(ctx, storage, client.workspaceId, document.content, ydoc) + await saveCollabYdoc(ctx, storage, client.workspaceId, collabId, ydoc) } catch (err) { ctx.error('error collaborative document content migration', { error: err, document: document.title }) } diff --git a/models/controlled-documents/src/types.ts b/models/controlled-documents/src/types.ts index 6fbe902438..4f8dc62038 100644 --- a/models/controlled-documents/src/types.ts +++ b/models/controlled-documents/src/types.ts @@ -48,17 +48,17 @@ import { DateRangeMode, IndexKind, type Class, + type MarkupBlobRef, type Doc, + type Domain, type Ref, type Timestamp, type Type, type CollectionSize, - type CollaborativeDoc, type Role, type TypedSpace, type Account, - type RolesAssignment, - type Domain + type RolesAssignment } from '@hcengineering/core' import { ArrOf, @@ -259,7 +259,7 @@ export class TDocument extends TDoc implements Document { state!: DocumentState @Prop(TypeCollaborativeDoc(), documents.string.CollaborativeDocument) - content!: CollaborativeDoc + content!: MarkupBlobRef | null @Prop(Collection(tags.class.TagReference), documents.string.Labels) labels?: CollectionSize @@ -425,7 +425,7 @@ export class TDocumentSnapshot extends TAttachedDoc implements DocumentSnapshot @Prop(TypeCollaborativeDoc(), documents.string.CollaborativeDocument) @Hidden() - content!: CollaborativeDoc + content!: MarkupBlobRef | null @Prop(TypeDocumentState(), documents.string.Status) state?: DocumentState diff --git a/models/core/src/core.ts b/models/core/src/core.ts index 60512e0cf2..2907480e95 100644 --- a/models/core/src/core.ts +++ b/models/core/src/core.ts @@ -28,7 +28,7 @@ import { type Card, type Class, type ClassifierKind, - type CollaborativeDoc, + type MarkupBlobRef, type Collection, type Configuration, type ConfigurationElement, @@ -121,7 +121,7 @@ export class TCard extends TDoc implements Card { title!: string @Prop(TypeCollaborativeDoc(), core.string.Description) - description!: CollaborativeDoc | null + description!: MarkupBlobRef | null @Prop(TypeString(), core.string.Id) identifier?: string | undefined @@ -373,10 +373,6 @@ export class TDomainIndexConfiguration extends TDoc implements DomainIndexConfig @Model(core.class.TypeCollaborativeDoc, core.class.Type) export class TTypeCollaborativeDoc extends TType {} -@UX(core.string.CollaborativeDocVersion) -@Model(core.class.TypeCollaborativeDocVersion, core.class.Type) -export class TTypeCollaborativeDocVersion extends TType {} - @UX(core.string.Rank) @Model(core.class.TypeRank, core.class.Type) export class TTypeRank extends TType {} diff --git a/models/core/src/index.ts b/models/core/src/index.ts index 26445f9f9d..d741be243c 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -58,7 +58,6 @@ import { TTypeBlob, TTypeBoolean, TTypeCollaborativeDoc, - TTypeCollaborativeDocVersion, TTypeDate, TTypeFileSize, TTypeHyperlink, @@ -125,7 +124,6 @@ export function createModel (builder: Builder): void { TEnumOf, TTypeMarkup, TTypeCollaborativeDoc, - TTypeCollaborativeDocVersion, TArrOf, TRefTo, TTypeDate, diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 36a96bafd0..e5dfc1e3bc 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -13,19 +13,21 @@ // limitations under the License. // -import { saveCollaborativeDoc } from '@hcengineering/collaboration' +import { saveCollabJson, saveCollabYdoc, yDocFromBuffer } from '@hcengineering/collaboration' import core, { - collaborativeDocParse, coreId, DOMAIN_MODEL_TX, DOMAIN_SPACE, DOMAIN_STATUS, DOMAIN_TX, generateId, - makeCollaborativeDoc, + makeDocCollabId, + makeCollabJsonId, + makeCollabYdocId, MeasureMetricsContext, RateLimiter, type AnyAttribute, + type Blob, type Class, type Doc, type Domain, @@ -48,7 +50,6 @@ import { type MigrationUpgradeClient } from '@hcengineering/model' import { type StorageAdapter } from '@hcengineering/storage' -import { markupToYDoc } from '@hcengineering/text' async function migrateStatusesToModel (client: MigrationClient): Promise { // Move statuses to model: @@ -176,7 +177,7 @@ async function migrateCollaborativeContentToStorage (client: MigrationClient): P const iterator = await client.traverse(domain, query) try { - console.log('processing', _class) + ctx.info('processing', { _class }) await processMigrateContentFor(ctx, domain, attributes, client, storageAdapter, iterator) } finally { await iterator.close() @@ -204,9 +205,6 @@ async function processMigrateContentFor ( break } - const timestamp = Date.now() - const revisionId = `${timestamp}` - const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] for (const doc of docs) { @@ -214,8 +212,6 @@ async function processMigrateContentFor ( const update: MigrateUpdate = {} for (const attribute of attributes) { - const collaborativeDoc = makeCollaborativeDoc(doc._id, attribute.name, revisionId) - const value = hierarchy.isMixin(attribute.attributeOf) ? ((doc as any)[attribute.attributeOf]?.[attribute.name] as string) : ((doc as any)[attribute.name] as string) @@ -224,22 +220,20 @@ async function processMigrateContentFor ( ? `${attribute.attributeOf}.${attribute.name}` : attribute.name + const collabId = makeDocCollabId(doc, attribute.name) + const blobId = makeCollabJsonId(collabId) + if (value != null && value.startsWith('{')) { - const { documentId } = collaborativeDocParse(collaborativeDoc) - const blob = await storageAdapter.stat(ctx, client.workspaceId, documentId) - // only for documents not in storage - if (blob === undefined) { - try { - const ydoc = markupToYDoc(value, attribute.name) - await saveCollaborativeDoc(ctx, storageAdapter, client.workspaceId, collaborativeDoc, ydoc) - } catch (err) { - console.error('failed to process document', doc._class, doc._id, err) - } + try { + const buffer = Buffer.from(value) + await storageAdapter.put(ctx, client.workspaceId, blobId, buffer, 'application/json', buffer.length) + } catch (err) { + ctx.error('failed to process document', { _class: doc._class, _id: doc._id, err }) } - update[attributeName] = collaborativeDoc + update[attributeName] = blobId } else if (value == null || value === '') { - update[attributeName] = collaborativeDoc + update[attributeName] = null } } @@ -256,10 +250,153 @@ async function processMigrateContentFor ( } processed += docs.length - console.log('...processed', processed) + ctx.info('...processed', { count: processed }) } } +async function migrateCollaborativeDocsToJson (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('migrateCollaborativeDocsToJson', {}) + const storageAdapter = client.storageAdapter + + const hierarchy = client.hierarchy + const classes = hierarchy.getDescendants(core.class.Doc) + for (const _class of classes) { + const domain = hierarchy.findDomain(_class) + if (domain === undefined) continue + + const allAttributes = hierarchy.getAllAttributes(_class) + const attributes = Array.from(allAttributes.values()).filter((attribute) => { + return hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc) + }) + + if (attributes.length === 0) continue + if (hierarchy.isMixin(_class) && attributes.every((p) => p.attributeOf !== _class)) continue + + const query = hierarchy.isMixin(_class) ? { [_class]: { $exists: true } } : { _class } + + const iterator = await client.traverse(domain, query) + try { + ctx.info('processing', { _class }) + await processMigrateJsonForDomain(ctx, domain, attributes, client, storageAdapter, iterator) + } finally { + await iterator.close() + } + } +} + +async function processMigrateJsonForDomain ( + ctx: MeasureContext, + domain: Domain, + attributes: AnyAttribute[], + client: MigrationClient, + storageAdapter: StorageAdapter, + iterator: MigrationIterator +): Promise { + const rateLimiter = new RateLimiter(10) + + let processed = 0 + + while (true) { + const docs = await iterator.next(100) + if (docs === null || docs.length === 0) { + break + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + + for (const doc of docs) { + await rateLimiter.exec(async () => { + const update = await processMigrateJsonForDoc(ctx, doc, attributes, client, storageAdapter) + if (Object.keys(update).length > 0) { + operations.push({ filter: { _id: doc._id }, update }) + } + }) + } + + await rateLimiter.waitProcessing() + + if (operations.length > 0) { + await client.bulk(domain, operations) + } + + processed += docs.length + ctx.info('...processed', { count: processed }) + } +} + +async function processMigrateJsonForDoc ( + ctx: MeasureContext, + doc: Doc, + attributes: AnyAttribute[], + client: MigrationClient, + storageAdapter: StorageAdapter +): Promise> { + const { hierarchy, workspaceId } = client + + const update: MigrateUpdate = {} + + for (const attribute of attributes) { + const value = hierarchy.isMixin(attribute.attributeOf) + ? ((doc as any)[attribute.attributeOf]?.[attribute.name] as string) + : ((doc as any)[attribute.name] as string) + + if (value == null || value === '') { + continue + } + + const attributeName = hierarchy.isMixin(attribute.attributeOf) + ? `${attribute.attributeOf}.${attribute.name}` + : attribute.name + + // Name of existing ydoc document + // original value here looks like '65b7f82f4d422b89d4cbdd6f:HEAD:0' + // where the first part is the blob id + const currentYdocId = value.split(':')[0] as Ref + const collabId = makeDocCollabId(doc, attribute.name) + + if (value.startsWith('{')) { + // For some reason we have documents that are already markups + const jsonId = await saveCollabJson(ctx, storageAdapter, workspaceId, collabId, value) + update[attributeName] = jsonId + continue + } + + try { + const stat = await storageAdapter.stat(ctx, workspaceId, currentYdocId) + if (stat !== undefined) { + if (stat.contentType.includes('application/ydoc')) { + const buffer = await storageAdapter.read(ctx, workspaceId, currentYdocId) + const ydoc = yDocFromBuffer(Buffer.concat(buffer as any)) + + // If document id has changed, save it with new name to ensure we will be able to load it later + const ydocId = makeCollabYdocId(collabId) + if (ydocId !== currentYdocId) { + ctx.info('saving collaborative doc with new name', { collabId, ydocId, currentYdocId }) + await saveCollabYdoc(ctx, storageAdapter, workspaceId, collabId, ydoc) + // do not bother with deletion so we can restore content is something goes wrong + // await storageAdapter.remove(ctx, client.workspaceId, [currentYdocId]) + } + + // Save document as JSON and save blob Id + const jsonId = await saveCollabJson(ctx, storageAdapter, workspaceId, collabId, ydoc) + update[attributeName] = jsonId + } else { + // it is not ydoc, do nothing + continue + } + } else { + // document is empty, unset + const unset = update.$unset ?? {} + update.$unset = { ...unset, [attribute.name]: 1 } + } + } catch (err) { + ctx.warn('failed to process collaborative doc', { workspaceId, collabId, currentYdocId, err }) + } + } + + return update +} + export const coreOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, coreId, [ @@ -351,6 +488,10 @@ export const coreOperation: MigrateOperation = { DOMAIN_MODEL_TX ) } + }, + { + state: 'collaborative-docs-to-json', + func: migrateCollaborativeDocsToJson } ]) }, diff --git a/models/document/package.json b/models/document/package.json index 55c1b687ad..f671066983 100644 --- a/models/document/package.json +++ b/models/document/package.json @@ -28,6 +28,7 @@ }, "dependencies": { "@hcengineering/model-core": "^0.6.0", + "@hcengineering/model-activity": "^0.6.0", "@hcengineering/model-chunter": "^0.6.0", "@hcengineering/model-workbench": "^0.6.1", "@hcengineering/model-attachment": "^0.6.0", diff --git a/models/document/src/index.ts b/models/document/src/index.ts index 23d8f4d997..1e17d90d36 100644 --- a/models/document/src/index.ts +++ b/models/document/src/index.ts @@ -1,5 +1,5 @@ // -// Copyright © 2022, 2023 Hardcore Engineering Inc. +// Copyright © 2022, 2023, 2024 Hardcore Engineering Inc. // // Licensed under the Eclipse Public License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. You may @@ -14,7 +14,7 @@ // import activity from '@hcengineering/activity' -import type { Class, CollaborativeDoc, CollectionSize, Domain, Rank, Role, RolesAssignment } from '@hcengineering/core' +import type { Class, CollectionSize, MarkupBlobRef, Domain, Rank, Role, RolesAssignment } from '@hcengineering/core' import { Account, AccountRole, IndexKind, Ref } from '@hcengineering/core' import { type Document, @@ -32,8 +32,8 @@ import { Mixin, Model, Prop, + ReadOnly, TypeCollaborativeDoc, - TypeCollaborativeDocVersion, TypeNumber, TypeRef, TypeString, @@ -77,7 +77,7 @@ export class TDocument extends TDoc implements Document, Todoable { title!: string @Prop(TypeCollaborativeDoc(), document.string.Document) - content!: CollaborativeDoc + content!: MarkupBlobRef | null @Prop(TypeRef(document.class.Document), document.string.ParentDocument) parent!: Ref @@ -137,8 +137,9 @@ export class TDocumentSnapshot extends TDoc implements DocumentSnapshot { @Index(IndexKind.FullText) title!: string - @Prop(TypeCollaborativeDocVersion(), document.string.Document) - content!: CollaborativeDoc + @Prop(TypeCollaborativeDoc(), document.string.Document) + @ReadOnly() + content!: MarkupBlobRef @Prop(TypeRef(document.class.Document), document.string.ParentDocument) parent!: Ref diff --git a/models/document/src/migration.ts b/models/document/src/migration.ts index e05caea6f9..6bc4efb4a4 100644 --- a/models/document/src/migration.ts +++ b/models/document/src/migration.ts @@ -13,8 +13,18 @@ // limitations under the License. // -import { DOMAIN_MODEL_TX, MeasureMetricsContext, SortingOrder, type CollaborativeDoc } from '@hcengineering/core' -import { type Document, type DocumentSnapshot, type Teamspace } from '@hcengineering/document' +import { + type Class, + type CollaborativeDoc, + type Doc, + type Ref, + DOMAIN_TX, + DOMAIN_MODEL_TX, + MeasureMetricsContext, + SortingOrder, + makeDocCollabId +} from '@hcengineering/core' +import { type DocumentSnapshot, type Document, type Teamspace } from '@hcengineering/document' import { migrateSpaceRanks, tryMigrate, @@ -24,11 +34,13 @@ import { type MigrationDocumentQuery, type MigrationUpgradeClient } from '@hcengineering/model' +import { DOMAIN_ACTIVITY } from '@hcengineering/model-activity' import core, { DOMAIN_SPACE } from '@hcengineering/model-core' +import { DOMAIN_NOTIFICATION } from '@hcengineering/notification' import { type Asset } from '@hcengineering/platform' import { makeRank } from '@hcengineering/rank' -import { loadCollaborativeDoc, saveCollaborativeDoc, yDocCopyXmlField } from '@hcengineering/collaboration' +import { loadCollabYdoc, saveCollabYdoc, yDocCopyXmlField } from '@hcengineering/collaboration' import document, { documentId, DOMAIN_DOCUMENT } from './index' async function migrateDocumentIcons (client: MigrationClient): Promise { @@ -197,23 +209,23 @@ async function renameFieldsRevert (client: MigrationClient): Promise { } ) - if (document.description.includes('%description:')) { - try { - const ydoc = await loadCollaborativeDoc(ctx, storage, client.workspaceId, document.description) - if (ydoc === undefined) { - continue - } + try { + const collabId = makeDocCollabId(document, 'content') - if (!ydoc.share.has('description') || ydoc.share.has('content')) { - continue - } - - yDocCopyXmlField(ydoc, 'description', 'content') - - await saveCollaborativeDoc(ctx, storage, client.workspaceId, document.description, ydoc) - } catch (err) { - ctx.error('error document content migration', { error: err, document: document.title }) + const ydoc = await loadCollabYdoc(ctx, storage, client.workspaceId, collabId) + if (ydoc === undefined) { + continue } + + if (!ydoc.share.has('description') || ydoc.share.has('content')) { + continue + } + + yDocCopyXmlField(ydoc, 'description', 'content') + + await saveCollabYdoc(ctx, storage, client.workspaceId, collabId, ydoc) + } catch (err) { + ctx.error('error document content migration', { error: err, document: document.title }) } } @@ -246,7 +258,9 @@ async function restoreContentField (client: MigrationClient): Promise { for (const document of documents) { try { - const ydoc = await loadCollaborativeDoc(ctx, storage, client.workspaceId, document.content) + const collabId = makeDocCollabId(document, 'content') + + const ydoc = await loadCollabYdoc(ctx, storage, client.workspaceId, collabId) if (ydoc === undefined) { ctx.error('document content not found', { document: document.title }) continue @@ -260,7 +274,7 @@ async function restoreContentField (client: MigrationClient): Promise { if (ydoc.share.has('')) { yDocCopyXmlField(ydoc, '', 'content') if (ydoc.share.has('content')) { - await saveCollaborativeDoc(ctx, storage, client.workspaceId, document.content, ydoc) + await saveCollabYdoc(ctx, storage, client.workspaceId, collabId, ydoc) } else { ctx.error('document content still not found', { document: document.title }) } @@ -281,6 +295,23 @@ async function migrateRanks (client: MigrationClient): Promise { } } +async function removeOldClasses (client: MigrationClient): Promise { + const classes = [ + 'document:class:DocumentContent', + 'document:class:DocumentSnapshot', + 'document:class:DocumentVersion', + 'document:class:DocumentRequest' + ] as Ref>[] + + for (const _class of classes) { + await client.deleteMany(DOMAIN_DOCUMENT, { _class }) + await client.deleteMany(DOMAIN_ACTIVITY, { attachedToClass: _class }) + await client.deleteMany(DOMAIN_ACTIVITY, { objectClass: _class }) + await client.deleteMany(DOMAIN_NOTIFICATION, { attachedToClass: _class }) + await client.deleteMany(DOMAIN_TX, { objectClass: _class }) + await client.deleteMany(DOMAIN_TX, { 'tx.objectClass': _class }) + } +} export const documentOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, documentId, [ @@ -321,6 +352,10 @@ export const documentOperation: MigrateOperation = { { state: 'migrateRanks', func: migrateRanks + }, + { + state: 'removeOldClasses', + func: removeOldClasses } ]) }, diff --git a/models/lead/src/migration.ts b/models/lead/src/migration.ts index 7f77fd49da..df581ebabf 100644 --- a/models/lead/src/migration.ts +++ b/models/lead/src/migration.ts @@ -13,14 +13,7 @@ // limitations under the License. // -import { - AccountRole, - DOMAIN_MODEL_TX, - makeCollaborativeDoc, - TxOperations, - type Ref, - type Status -} from '@hcengineering/core' +import { AccountRole, DOMAIN_MODEL_TX, TxOperations, type Ref, type Status } from '@hcengineering/core' import { leadId, type Lead } from '@hcengineering/lead' import { tryMigrate, @@ -213,44 +206,6 @@ export const leadOperation: MigrateOperation = { } } ) - const it = await client.traverse(DOMAIN_CONTACT, { - _class: contact.class.Organization, - description: { $exists: false } - }) - while (true) { - const docs = await it.next(50) - if (docs == null || docs.length === 0) { - break - } - await client.bulk( - DOMAIN_CONTACT, - docs.map((doc) => ({ - filter: { _id: doc._id }, - update: { $set: { description: makeCollaborativeDoc(doc._id, 'description') } } - })) - ) - } - const it2 = await client.traverse(DOMAIN_CONTACT, { [lead.mixin.Customer + '.customerDescription']: null }) - while (true) { - const docs = await it2.next(50) - if (docs == null || docs.length === 0) { - break - } - await client.bulk( - DOMAIN_CONTACT, - docs.map((doc) => ({ - filter: { _id: doc._id }, - update: { - $set: { - [lead.mixin.Customer + '.customerDescription']: makeCollaborativeDoc( - docs[0]._id, - 'customerDescription' - ) - } - } - })) - ) - } } } ]) diff --git a/models/lead/src/types.ts b/models/lead/src/types.ts index 7a32aeebbf..25b807ec0e 100644 --- a/models/lead/src/types.ts +++ b/models/lead/src/types.ts @@ -17,7 +17,7 @@ import type { Employee } from '@hcengineering/contact' import { Account, IndexKind, - type CollaborativeDoc, + type MarkupBlobRef, type Role, type RolesAssignment, type Ref, @@ -97,7 +97,7 @@ export class TCustomer extends TContact implements Customer { @Prop(TypeCollaborativeDoc(), lead.string.Description) @Index(IndexKind.FullText) - customerDescription!: CollaborativeDoc + customerDescription!: MarkupBlobRef | null } @Mixin(lead.mixin.DefaultFunnelTypeData, lead.class.Funnel) diff --git a/models/love/src/index.ts b/models/love/src/index.ts index 0467e74658..4b5183244c 100644 --- a/models/love/src/index.ts +++ b/models/love/src/index.ts @@ -16,15 +16,15 @@ import contact, { type Employee, type Person } from '@hcengineering/contact' import { AccountRole, - type CollaborativeDoc, type CollectionSize, - DateRangeMode, type Doc, type Domain, - DOMAIN_TRANSIENT, - IndexKind, + type MarkupBlobRef, type Ref, - type Timestamp + type Timestamp, + DOMAIN_TRANSIENT, + DateRangeMode, + IndexKind } from '@hcengineering/core' import { type DevicesPreference, @@ -91,7 +91,7 @@ export class TRoom extends TDoc implements Room { @Prop(TypeCollaborativeDoc(), core.string.Description) @Index(IndexKind.FullText) - description!: CollaborativeDoc + description!: MarkupBlobRef | null type!: RoomType @@ -212,7 +212,7 @@ export class TMeetingMinutes extends TAttachedDoc implements MeetingMinutes, Tod @Prop(TypeCollaborativeDoc(), core.string.Description) @Index(IndexKind.FullText) - description!: CollaborativeDoc + description!: MarkupBlobRef | null @Prop(TypeAny(love.component.MeetingMinutesStatusPresenter, love.string.Status), love.string.Status, { editor: love.component.MeetingMinutesStatusPresenter diff --git a/models/love/src/migration.ts b/models/love/src/migration.ts index be65d9912d..db983c7aa4 100644 --- a/models/love/src/migration.ts +++ b/models/love/src/migration.ts @@ -14,7 +14,7 @@ // import contact from '@hcengineering/contact' -import { type Space, TxOperations, type Ref, makeCollaborativeDoc } from '@hcengineering/core' +import { type Space, TxOperations, type Ref } from '@hcengineering/core' import drive from '@hcengineering/drive' import { MeetingStatus, @@ -23,8 +23,7 @@ import { createDefaultRooms, isOffice, loveId, - type Floor, - type Room + type Floor } from '@hcengineering/love' import { createDefaultSpace, @@ -90,7 +89,7 @@ async function createReception (client: MigrationUpgradeClient): Promise { language: 'en', startWithTranscription: false, startWithRecording: false, - description: makeCollaborativeDoc(love.ids.Reception, 'description') + description: null }, love.ids.Reception ) @@ -156,18 +155,6 @@ export const loveOperation: MigrateOperation = { await client.move(DOMAIN_LOVE, { _class: love.class.MeetingMinutes }, DOMAIN_MEETING_MINUTES) } }, - { - state: 'create-description-collaborative', - func: async (client) => { - const rooms = await client.find(DOMAIN_LOVE, { _class: { $in: [love.class.Room, love.class.Office] } }) - for (const room of rooms) { - const description = room.description - if (description == null) { - await client.update(DOMAIN_LOVE, room, { description: makeCollaborativeDoc(room._id, 'description') }) - } - } - } - }, { state: 'default-meeting-minutes-status', func: async (client) => { diff --git a/models/recruit/src/types.ts b/models/recruit/src/types.ts index c6d390be15..8d1c097744 100644 --- a/models/recruit/src/types.ts +++ b/models/recruit/src/types.ts @@ -17,8 +17,8 @@ import type { Employee, Organization } from '@hcengineering/contact' import { Account, IndexKind, - type CollaborativeDoc, type Collection, + type MarkupBlobRef, type Domain, type Markup, type Ref, @@ -68,7 +68,7 @@ import recruit from './plugin' export class TVacancy extends TProject implements Vacancy { @Prop(TypeCollaborativeDoc(), recruit.string.FullDescription) @Index(IndexKind.FullText) - fullDescription!: CollaborativeDoc + fullDescription!: MarkupBlobRef | null @Prop(TypeCollection(attachment.class.Attachment), attachment.string.Attachments, { shortLabel: attachment.string.Files diff --git a/models/test-management/src/types.ts b/models/test-management/src/types.ts index c0b98b211a..fc1ffcd1ef 100644 --- a/models/test-management/src/types.ts +++ b/models/test-management/src/types.ts @@ -40,7 +40,7 @@ import { type Timestamp, type Type, type CollectionSize, - type CollaborativeDoc, + type MarkupBlobRef, type Class } from '@hcengineering/core' import { @@ -163,7 +163,7 @@ export class TTestCase extends TAttachedDoc implements TestCase { @Prop(TypeCollaborativeDoc(), testManagement.string.FullDescription) @Index(IndexKind.FullText) - description!: CollaborativeDoc + description!: MarkupBlobRef | null @Prop(TypeTestCaseType(), testManagement.string.TestType) @ReadOnly() @@ -196,7 +196,7 @@ export class TTestRun extends TDoc implements TestRun { @Prop(TypeCollaborativeDoc(), testManagement.string.FullDescription) @Index(IndexKind.FullText) - description!: CollaborativeDoc + description!: MarkupBlobRef | null @Prop(TypeDate(DateRangeMode.DATETIME), testManagement.string.DueDate) dueDate?: Timestamp @@ -244,7 +244,7 @@ export class TTestResult extends TAttachedDoc implements TestResult { @Prop(TypeCollaborativeDoc(), testManagement.string.FullDescription) @Index(IndexKind.FullText) - description!: CollaborativeDoc + description!: MarkupBlobRef | null @Prop(TypeRef(testManagement.class.TestCase), testManagement.string.TestCase) testCase!: Ref diff --git a/models/tracker/src/types.ts b/models/tracker/src/types.ts index 70cadb279e..92d8ba2d13 100644 --- a/models/tracker/src/types.ts +++ b/models/tracker/src/types.ts @@ -19,7 +19,7 @@ import { DOMAIN_MODEL, DateRangeMode, IndexKind, - type CollaborativeDoc, + type MarkupBlobRef, type Domain, type Markup, type Ref, @@ -185,7 +185,7 @@ export class TIssue extends TTask implements Issue { @Prop(TypeCollaborativeDoc(), tracker.string.Description) @Index(IndexKind.FullText) - description!: CollaborativeDoc + description!: MarkupBlobRef | null @Prop(TypeRef(tracker.class.IssueStatus), tracker.string.Status, { _id: tracker.attribute.IssueStatus, diff --git a/models/view/src/index.ts b/models/view/src/index.ts index 7d21f3db1c..968100c190 100644 --- a/models/view/src/index.ts +++ b/models/view/src/index.ts @@ -514,10 +514,6 @@ export function createModel (builder: Builder): void { presenter: view.component.MarkupDiffPresenter }) - builder.mixin(core.class.TypeCollaborativeDocVersion, core.class.Class, view.mixin.InlineAttributEditor, { - editor: view.component.CollaborativeDocEditor - }) - classPresenter(builder, core.class.TypeBoolean, view.component.BooleanPresenter, view.component.BooleanEditor) classPresenter( builder, diff --git a/packages/api-client/src/client.ts b/packages/api-client/src/client.ts index a39c0bc8df..5ad7fe45b9 100644 --- a/packages/api-client/src/client.ts +++ b/packages/api-client/src/client.ts @@ -128,12 +128,12 @@ class PlatformClientImpl implements PlatformClient { await this.connection.close() } - private async processMarkup(id: Ref, data: WithMarkup): Promise { + private async processMarkup(_class: Ref>, id: Ref, data: WithMarkup): Promise { const result: any = {} for (const [key, value] of Object.entries(data)) { if (value instanceof MarkupContent) { - result[key] = this.markup.uploadMarkup(id, key, value.content, value.kind) + result[key] = this.markup.uploadMarkup(_class, id, key, value.content, value.kind) } else { result[key] = value } @@ -151,7 +151,7 @@ class PlatformClientImpl implements PlatformClient { id?: Ref ): Promise> { id ??= generateId() - const data = await this.processMarkup>(id, attributes) + const data = await this.processMarkup>(_class, id, attributes) return await this.client.createDoc(_class, space, data, id) } @@ -162,7 +162,7 @@ class PlatformClientImpl implements PlatformClient { operations: WithMarkup>, retrieve?: boolean ): Promise { - const update = await this.processMarkup>(objectId, operations) + const update = await this.processMarkup>(_class, objectId, operations) return await this.client.updateDoc(_class, space, objectId, update, retrieve) } @@ -182,7 +182,7 @@ class PlatformClientImpl implements PlatformClient { id?: Ref

): Promise> { id ??= generateId() - const data = await this.processMarkup>(id, attributes) + const data = await this.processMarkup>(_class, id, attributes) return await this.client.addCollection(_class, space, attachedTo, attachedToClass, collection, data, id) } @@ -196,7 +196,7 @@ class PlatformClientImpl implements PlatformClient { operations: WithMarkup>, retrieve?: boolean ): Promise> { - const update = await this.processMarkup>(objectId, operations) + const update = await this.processMarkup>(_class, objectId, operations) return await this.client.updateCollection( _class, space, @@ -229,7 +229,7 @@ class PlatformClientImpl implements PlatformClient { mixin: Ref>, attributes: WithMarkup> ): Promise { - const data = await this.processMarkup>(objectId, attributes) + const data = await this.processMarkup>(objectClass, objectId, attributes) return await this.client.createMixin(objectId, objectClass, objectSpace, mixin, data) } @@ -240,18 +240,30 @@ class PlatformClientImpl implements PlatformClient { mixin: Ref>, attributes: WithMarkup> ): Promise { - const update = await this.processMarkup>(objectId, attributes) + const update = await this.processMarkup>(objectClass, objectId, attributes) return await this.client.updateMixin(objectId, objectClass, objectSpace, mixin, update) } // Markup - async fetchMarkup (objectId: Ref, objectAttr: string, markup: MarkupRef, format: MarkupFormat): Promise { - return await this.markup.fetchMarkup(objectId, objectAttr, markup, format) + async fetchMarkup ( + objectClass: Ref>, + objectId: Ref, + objectAttr: string, + markup: MarkupRef, + format: MarkupFormat + ): Promise { + return await this.markup.fetchMarkup(objectClass, objectId, objectAttr, markup, format) } - async uploadMarkup (objectId: Ref, objectAttr: string, markup: string, format: MarkupFormat): Promise { - return await this.markup.uploadMarkup(objectId, objectAttr, markup, format) + async uploadMarkup ( + objectClass: Ref>, + objectId: Ref, + objectAttr: string, + markup: string, + format: MarkupFormat + ): Promise { + return await this.markup.uploadMarkup(objectClass, objectId, objectAttr, markup, format) } // AsyncDisposable diff --git a/packages/api-client/src/markup/client.ts b/packages/api-client/src/markup/client.ts index 7bc255899a..180e6fdfdf 100644 --- a/packages/api-client/src/markup/client.ts +++ b/packages/api-client/src/markup/client.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { type Doc, Markup, type Ref, concatLink, makeCollaborativeDoc } from '@hcengineering/core' +import { type Class, type Doc, type Markup, type Ref, concatLink, makeCollabId } from '@hcengineering/core' import { type CollaboratorClient, getClient } from '@hcengineering/collaborator-client' import { parseMessageMarkdown, jsonToMarkup, markupToHTML, markupToMarkdown, htmlToMarkup } from '@hcengineering/text' @@ -45,9 +45,15 @@ class MarkupOperationsImpl implements MarkupOperations { this.collaborator = getClient({ name: workspace }, token, config.COLLABORATOR_URL) } - async fetchMarkup (objectId: Ref, objectAttr: string, doc: MarkupRef, format: MarkupFormat): Promise { - const content = await this.collaborator.getContent(doc) - const markup = content[objectAttr] ?? '' + async fetchMarkup ( + objectClass: Ref>, + objectId: Ref, + objectAttr: string, + doc: MarkupRef, + format: MarkupFormat + ): Promise { + const collabId = makeCollabId(objectClass, objectId, objectAttr) + const markup = await this.collaborator.getMarkup(collabId, doc) switch (format) { case 'markup': @@ -61,7 +67,13 @@ class MarkupOperationsImpl implements MarkupOperations { } } - async uploadMarkup (objectId: Ref, objectAttr: string, value: string, format: MarkupFormat): Promise { + async uploadMarkup ( + objectClass: Ref>, + objectId: Ref, + objectAttr: string, + value: string, + format: MarkupFormat + ): Promise { let markup: Markup = '' switch (format) { @@ -78,8 +90,7 @@ class MarkupOperationsImpl implements MarkupOperations { throw new Error('Unknown content format') } - const doc = makeCollaborativeDoc(objectId, objectAttr) - await this.collaborator.updateContent(doc, { [objectAttr]: markup }) - return doc + const collabId = makeCollabId(objectClass, objectId, objectAttr) + return await this.collaborator.createMarkup(collabId, markup) } } diff --git a/packages/api-client/src/markup/types.ts b/packages/api-client/src/markup/types.ts index a641f9fd06..a4f6765689 100644 --- a/packages/api-client/src/markup/types.ts +++ b/packages/api-client/src/markup/types.ts @@ -13,10 +13,10 @@ // limitations under the License. // -import { type CollaborativeDoc, type Doc, type Ref } from '@hcengineering/core' +import { Class, type Blob, type Doc, type Ref } from '@hcengineering/core' /** @public */ -export type MarkupRef = CollaborativeDoc +export type MarkupRef = Ref /** @public */ export type MarkupFormat = 'markup' | 'html' | 'markdown' @@ -45,21 +45,35 @@ export function markdown (content: string): MarkupContent { export interface MarkupOperations { /** * Retrieves markup content for a specified document object + * * @param objectClass - Reference to the class of the document containing the markup * @param objectId - Reference to the document containing the markup * @param objectAttr - The attribute/field name where the markup is stored * @param id - Unique reference identifying the specific markup content * @param format - The format of the markup (e.g., HTML, Markdown, etc.) * @returns Promise containing the markup content as a string */ - fetchMarkup: (objectId: Ref, objectAttr: string, id: MarkupRef, format: MarkupFormat) => Promise + fetchMarkup: ( + objectClass: Ref>, + objectId: Ref, + objectAttr: string, + id: MarkupRef, + format: MarkupFormat + ) => Promise /** * Saves markup content for a document object + * @param objectClass - Reference to the class of the document where markup should be stored * @param objectId - Reference to the document where markup should be stored * @param objectAttr - The attribute/field name where markup should be saved * @param markup - The actual markup content to be uploaded * @param format - The format of the provided markup (e.g., HTML, Markdown, etc.) * @returns Promise containing a reference to the newly saved markup */ - uploadMarkup: (objectId: Ref, objectAttr: string, markup: string, format: MarkupFormat) => Promise + uploadMarkup: ( + objectClass: Ref>, + objectId: Ref, + objectAttr: string, + markup: string, + format: MarkupFormat + ) => Promise } diff --git a/packages/collaborator-client/src/__tests__/utils.test.ts b/packages/collaborator-client/src/__tests__/utils.test.ts index 00b407d3d6..992e39c2bd 100644 --- a/packages/collaborator-client/src/__tests__/utils.test.ts +++ b/packages/collaborator-client/src/__tests__/utils.test.ts @@ -13,26 +13,27 @@ // limitations under the License. // -import { CollaborativeDoc } from '@hcengineering/core' -import { DocumentId } from '../types' -import { formatDocumentId, parseDocumentId } from '../utils' +import core, { CollaborativeDoc, Doc, Ref } from '@hcengineering/core' +import { encodeDocumentId, decodeDocumentId } from '../utils' describe('utils', () => { - it('formatDocumentId', () => { - expect(formatDocumentId('ws1', 'doc1:HEAD:v1' as CollaborativeDoc)).toEqual('ws1://doc1:HEAD' as DocumentId) - expect(formatDocumentId('ws1', 'doc1:HEAD:v1#doc2:v2:v2' as CollaborativeDoc)).toEqual( - 'ws1://doc1:HEAD/doc2:v2' as DocumentId - ) + it('encodeDocumentId', () => { + const doc: CollaborativeDoc = { + objectClass: core.class.Card, + objectId: 'doc1' as Ref, + objectAttr: 'description' + } + expect(encodeDocumentId('ws1', doc)).toEqual('ws1|core:class:Card|doc1|description') }) - describe('parseDocumentId', () => { - expect(parseDocumentId('ws1://doc1:HEAD' as DocumentId)).toEqual({ + describe('decodeDocumentId', () => { + expect(decodeDocumentId('ws1|core:class:Card|doc1|description')).toEqual({ workspaceId: 'ws1', - collaborativeDoc: 'doc1:HEAD:HEAD' as CollaborativeDoc - }) - expect(parseDocumentId('ws1://doc1:HEAD/doc2:v2' as DocumentId)).toEqual({ - workspaceId: 'ws1', - collaborativeDoc: 'doc1:HEAD:HEAD#doc2:v2:v2' as CollaborativeDoc + documentId: { + objectClass: core.class.Card, + objectId: 'doc1' as Ref, + objectAttr: 'description' + } }) }) }) diff --git a/packages/collaborator-client/src/client.ts b/packages/collaborator-client/src/client.ts index c7ffa716a4..b7d34156a4 100644 --- a/packages/collaborator-client/src/client.ts +++ b/packages/collaborator-client/src/client.ts @@ -13,18 +13,29 @@ // limitations under the License. // -import { CollaborativeDoc, Markup, WorkspaceId, concatLink } from '@hcengineering/core' -import { formatDocumentId } from './utils' +import { Blob, CollaborativeDoc, Markup, MarkupBlobRef, Ref, WorkspaceId, concatLink } from '@hcengineering/core' +import { encodeDocumentId } from './utils' /** @public */ -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface GetContentRequest {} +export interface GetContentRequest { + source?: Ref +} /** @public */ export interface GetContentResponse { content: Record } +/** @public */ +export interface CreateContentRequest { + content: Record +} + +/** @public */ +export interface CreateContentResponse { + content: Record +} + /** @public */ export interface UpdateContentRequest { content: Record @@ -36,8 +47,9 @@ export interface UpdateContentResponse {} /** @public */ export interface CollaboratorClient { - getContent: (document: CollaborativeDoc) => Promise> - updateContent: (document: CollaborativeDoc, content: Record) => Promise + getMarkup: (document: CollaborativeDoc, source?: Ref | null) => Promise + createMarkup: (document: CollaborativeDoc, markup: Markup) => Promise + updateMarkup: (document: CollaborativeDoc, markup: Markup) => Promise copyContent: (source: CollaborativeDoc, target: CollaborativeDoc) => Promise } @@ -54,11 +66,11 @@ class CollaboratorClientImpl implements CollaboratorClient { private readonly collaboratorUrl: string ) {} - private async rpc (document: CollaborativeDoc, method: string, payload: any): Promise { + private async rpc(document: CollaborativeDoc, method: string, payload: P): Promise { const workspace = this.workspace.name - const documentId = formatDocumentId(workspace, document) + const documentId = encodeDocumentId(workspace, document) - const url = concatLink(this.collaboratorUrl, '/rpc') + const url = concatLink(this.collaboratorUrl, `/rpc/${encodeURIComponent(documentId)}`) const res = await fetch(url, { method: 'POST', @@ -66,7 +78,7 @@ class CollaboratorClientImpl implements CollaboratorClient { Authorization: 'Bearer ' + this.token, 'Content-Type': 'application/json' }, - body: JSON.stringify({ method, documentId, payload }) + body: JSON.stringify({ method, payload }) }) if (!res.ok) { @@ -79,33 +91,58 @@ class CollaboratorClientImpl implements CollaboratorClient { throw new Error(result.error) } - return result + return result as R } - async getContent (document: CollaborativeDoc): Promise> { + async getMarkup (document: CollaborativeDoc, source?: Ref | null): Promise { + const payload: GetContentRequest = { + source: source !== null ? source : undefined + } + const res = await retry( 3, async () => { - return (await this.rpc(document, 'getContent', {})) as GetContentResponse + return await this.rpc(document, 'getContent', payload) }, 50 ) - return res.content ?? {} + + return res.content[document.objectAttr] ?? '' } - async updateContent (document: CollaborativeDoc, content: Record): Promise { + async createMarkup (document: CollaborativeDoc, markup: Markup): Promise { + const content = { + [document.objectAttr]: markup + } + + const res = await retry( + 3, + async () => { + return await this.rpc(document, 'createContent', { content }) + }, + 50 + ) + + return res.content[document.objectAttr] + } + + async updateMarkup (document: CollaborativeDoc, markup: Markup): Promise { + const content = { + [document.objectAttr]: markup + } + await retry( 3, async () => { - await this.rpc(document, 'updateContent', { content }) + await this.rpc(document, 'updateContent', { content }) }, 50 ) } - async copyContent (source: CollaborativeDoc, target: CollaborativeDoc): Promise { - const content = await this.getContent(source) - await this.updateContent(target, content) + async copyContent (source: CollaborativeDoc, target: CollaborativeDoc, content?: Ref): Promise { + const markup = await this.getMarkup(source, content) + await this.updateMarkup(target, markup) } } diff --git a/packages/collaborator-client/src/index.ts b/packages/collaborator-client/src/index.ts index b487cf8270..70baf83124 100644 --- a/packages/collaborator-client/src/index.ts +++ b/packages/collaborator-client/src/index.ts @@ -14,5 +14,4 @@ // export * from './client' -export * from './types' export * from './utils' diff --git a/packages/collaborator-client/src/types.ts b/packages/collaborator-client/src/types.ts deleted file mode 100644 index 9169a5eadc..0000000000 --- a/packages/collaborator-client/src/types.ts +++ /dev/null @@ -1,20 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -/** @public */ -export type DocumentId = string & { __documentId: true } - -/** @public */ -export type PlatformDocumentId = string & { __platformDocId: true } diff --git a/packages/collaborator-client/src/utils.ts b/packages/collaborator-client/src/utils.ts index 216744b62e..f6225ba0aa 100644 --- a/packages/collaborator-client/src/utils.ts +++ b/packages/collaborator-client/src/utils.ts @@ -13,78 +13,26 @@ // limitations under the License. // -import { - Class, - CollaborativeDoc, - Doc, - Ref, - collaborativeDocChain, - collaborativeDocFormat, - collaborativeDocParse, - collaborativeDocUnchain -} from '@hcengineering/core' -import { DocumentId, PlatformDocumentId } from './types' +import { Class, CollaborativeDoc, Doc, Ref } from '@hcengineering/core' -/** - * Formats collaborative document as Hocuspocus document name. - * - * The document name is used for document identification on the server so should remain the same even - * when document is updated. Hence, we remove lastVersionId component from CollaborativeDoc. - * - * Example: - * workspace1://doc1:HEAD/doc2:v1 - * - * @public - */ -export function formatDocumentId (workspaceId: string, collaborativeDoc: CollaborativeDoc): DocumentId { - const path = collaborativeDocUnchain(collaborativeDoc) - .map((p) => { - const { documentId, versionId } = collaborativeDocParse(p) - return `${documentId}:${versionId}` - }) - .join('/') - - return `${workspaceId}://${path}` as DocumentId +/** @public */ +export function encodeDocumentId (workspaceId: string, documentId: CollaborativeDoc): string { + const { objectClass, objectId, objectAttr } = documentId + return [workspaceId, objectClass, objectId, objectAttr].join('|') } /** @public */ -export function parseDocumentId (documentId: DocumentId): { +export function decodeDocumentId (documentId: string): { workspaceId: string - collaborativeDoc: CollaborativeDoc + documentId: CollaborativeDoc } { - const [workspaceId, path] = documentId.split('://') - const segments = path.split('/') - - const collaborativeDocs = segments.map((p) => { - const [documentId, versionId] = p.split(':') - return collaborativeDocFormat({ documentId, versionId, lastVersionId: versionId }) - }) - + const [workspaceId, objectClass, objectId, objectAttr] = documentId.split('|') return { workspaceId, - collaborativeDoc: collaborativeDocChain(...collaborativeDocs) - } -} - -/** @public */ -export function formatPlatformDocumentId ( - objectClass: Ref>, - objectId: Ref, - objectAttr: string -): PlatformDocumentId { - return `${objectClass}/${objectId}/${objectAttr}` as PlatformDocumentId -} - -/** @public */ -export function parsePlatformDocumentId (platformDocumentId: PlatformDocumentId): { - objectClass: Ref> - objectId: Ref - objectAttr: string -} { - const [objectClass, objectId, objectAttr] = platformDocumentId.split('/') - return { - objectClass: objectClass as Ref>, - objectId: objectId as Ref, - objectAttr + documentId: { + objectClass: objectClass as Ref>, + objectId: objectId as Ref, + objectAttr + } } } diff --git a/packages/core/lang/en.json b/packages/core/lang/en.json index fa7f241872..f0a11de8cd 100644 --- a/packages/core/lang/en.json +++ b/packages/core/lang/en.json @@ -36,7 +36,7 @@ "Enum": "Enum", "Members": "Members", "Hyperlink": "URL", - "Collaborative": "Collaborative", + "MarkupBlobRef": "Collaborative", "Object": "Object", "System": "System", "CreatedBy": "Created by", diff --git a/packages/core/lang/es.json b/packages/core/lang/es.json index 563325862c..67acda9558 100644 --- a/packages/core/lang/es.json +++ b/packages/core/lang/es.json @@ -29,7 +29,7 @@ "Enum": "Enum.", "Members": "Miembros", "Hyperlink": "Enlace", - "Collaborative": "Colaborativo", + "MarkupBlobRef": "Colaborativo", "Object": "Objeto", "System": "Sistema", "CreatedBy": "Creado por", diff --git a/packages/core/lang/fr.json b/packages/core/lang/fr.json index 0e33492919..9fea9a9952 100644 --- a/packages/core/lang/fr.json +++ b/packages/core/lang/fr.json @@ -36,7 +36,7 @@ "Enum": "Énumération", "Members": "Membres", "Hyperlink": "URL", - "Collaborative": "Collaboratif", + "MarkupBlobRef": "Collaboratif", "Object": "Objet", "System": "Système", "CreatedBy": "Créé par", diff --git a/packages/core/lang/pt.json b/packages/core/lang/pt.json index 171e524cb6..d654320cf1 100644 --- a/packages/core/lang/pt.json +++ b/packages/core/lang/pt.json @@ -29,7 +29,7 @@ "Enum": "Enumeração", "Members": "Membros", "Hyperlink": "URL", - "Collaborative": "Colaborativo", + "MarkupBlobRef": "Colaborativo", "Object": "Objeto", "System": "Sistema", "CreatedBy": "Criado por", diff --git a/packages/core/lang/ru.json b/packages/core/lang/ru.json index c31dd56013..b887a212fa 100644 --- a/packages/core/lang/ru.json +++ b/packages/core/lang/ru.json @@ -36,7 +36,7 @@ "Enum": "Справочник", "Members": "Участники", "Hyperlink": "URL", - "Collaborative": "Коллаборативный", + "MarkupBlobRef": "Коллаборативный", "Object": "Объект", "System": "Система", "CreatedBy": "Создан", diff --git a/packages/core/lang/zh.json b/packages/core/lang/zh.json index d7370b90d2..11a95ea773 100644 --- a/packages/core/lang/zh.json +++ b/packages/core/lang/zh.json @@ -36,7 +36,7 @@ "Enum": "枚举", "Members": "成员", "Hyperlink": "URL", - "Collaborative": "协作", + "MarkupBlobRef": "协作", "Object": "对象", "System": "系统", "CreatedBy": "创建者", diff --git a/packages/core/src/__tests__/collaboration.test.ts b/packages/core/src/__tests__/collaboration.test.ts deleted file mode 100644 index ffc376a30a..0000000000 --- a/packages/core/src/__tests__/collaboration.test.ts +++ /dev/null @@ -1,204 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { - CollaborativeDoc, - collaborativeDocChain, - collaborativeDocUnchain, - collaborativeDocFormat, - collaborativeDocParse, - collaborativeDocFromCollaborativeDoc, - collaborativeDocFromLastVersion, - collaborativeDocWithVersion, - collaborativeDocWithLastVersion, - collaborativeDocWithSource -} from '../collaboration' - -describe('collaborative-doc', () => { - describe('collaborativeDocChain', () => { - it('chains one collaborative doc', async () => { - expect(collaborativeDocChain('doc1:v1:v1' as CollaborativeDoc)).toEqual('doc1:v1:v1' as CollaborativeDoc) - }) - it('chains multiple collaborative docs', async () => { - expect(collaborativeDocChain('doc1:v1:v1' as CollaborativeDoc, 'doc2:v2:v2' as CollaborativeDoc)).toEqual( - 'doc1:v1:v1#doc2:v2:v2' as CollaborativeDoc - ) - }) - it('chains multiple chained collaborative docs', async () => { - expect( - collaborativeDocChain('doc1:v1:v1#doc2:v2:v2' as CollaborativeDoc, 'doc3:v3:v3' as CollaborativeDoc) - ).toEqual('doc1:v1:v1#doc2:v2:v2#doc3:v3:v3' as CollaborativeDoc) - }) - }) - - describe('collaborativeDocUnchain', () => { - it('unchains one collaborative doc', async () => { - expect(collaborativeDocUnchain('doc1:v1:v1' as CollaborativeDoc)).toEqual(['doc1:v1:v1'] as CollaborativeDoc[]) - }) - it('unchains multiple collaborative docs', async () => { - expect(collaborativeDocUnchain('doc1:v1:v1#doc2:v2:v2' as CollaborativeDoc)).toEqual([ - 'doc1:v1:v1', - 'doc2:v2:v2' - ] as CollaborativeDoc[]) - }) - }) - - describe('collaborativeDocParse', () => { - it('parses collaborative doc id', async () => { - expect(collaborativeDocParse('documentId' as CollaborativeDoc)).toEqual({ - documentId: 'documentId', - versionId: 'HEAD', - lastVersionId: 'HEAD', - source: [] - }) - }) - it('parses collaborative doc id with versionId', async () => { - expect(collaborativeDocParse('documentId:main' as CollaborativeDoc)).toEqual({ - documentId: 'documentId', - versionId: 'main', - lastVersionId: 'main', - source: [] - }) - }) - it('parses collaborative doc id with versionId and lastVersionId', async () => { - expect(collaborativeDocParse('documentId:HEAD:0' as CollaborativeDoc)).toEqual({ - documentId: 'documentId', - versionId: 'HEAD', - lastVersionId: '0', - source: [] - }) - }) - it('parses collaborative doc id with versionId, lastVersionId, and source', async () => { - expect(collaborativeDocParse('documentId:HEAD:0#documentId1:main#documentId2:HEAD' as CollaborativeDoc)).toEqual({ - documentId: 'documentId', - versionId: 'HEAD', - lastVersionId: '0', - source: ['documentId1:main' as CollaborativeDoc, 'documentId2:HEAD' as CollaborativeDoc] - }) - }) - }) - - describe('collaborativeDocFormat', () => { - it('formats collaborative doc id', async () => { - expect( - collaborativeDocFormat({ - documentId: 'documentId', - versionId: 'HEAD', - lastVersionId: '0' - }) - ).toEqual('documentId:HEAD:0') - }) - it('formats collaborative doc id with sources', async () => { - expect( - collaborativeDocFormat({ - documentId: 'documentId', - versionId: 'HEAD', - lastVersionId: '0', - source: ['documentId1:main' as CollaborativeDoc, 'documentId2:HEAD' as CollaborativeDoc] - }) - ).toEqual('documentId:HEAD:0#documentId1:main#documentId2:HEAD') - }) - it('formats collaborative doc id with invalid characters', async () => { - expect( - collaborativeDocFormat({ - documentId: 'doc:id', - versionId: 'version#id', - lastVersionId: 'last:version#id' - }) - ).toEqual('doc%id:version%id:last%version%id') - }) - }) - - describe('collaborativeDocWithVersion', () => { - it('updates collaborative doc version id', async () => { - expect(collaborativeDocWithVersion('doc1:HEAD:HEAD' as CollaborativeDoc, 'v1')).toEqual('doc1:v1:v1') - expect(collaborativeDocWithVersion('doc1:HEAD:v1' as CollaborativeDoc, 'v2')).toEqual('doc1:v2:v2') - expect(collaborativeDocWithVersion('doc1:HEAD:v1#doc2:v1:v1' as CollaborativeDoc, 'v2')).toEqual( - 'doc1:v2:v2#doc2:v1:v1' - ) - }) - }) - - describe('collaborativeDocWithLastVersion', () => { - it('updates collaborative doc version id', async () => { - expect(collaborativeDocWithLastVersion('doc1:HEAD:HEAD' as CollaborativeDoc, 'v1')).toEqual('doc1:HEAD:v1') - expect(collaborativeDocWithLastVersion('doc1:HEAD:v1' as CollaborativeDoc, 'v2')).toEqual('doc1:HEAD:v2') - expect(collaborativeDocWithLastVersion('doc1:HEAD:v1#doc2:v1:v1' as CollaborativeDoc, 'v2')).toEqual( - 'doc1:HEAD:v2#doc2:v1:v1' - ) - expect(collaborativeDocWithLastVersion('doc1:v1:v1' as CollaborativeDoc, 'v2')).toEqual( - // cannot update last version for non HEAD - 'doc1:v1:v1' - ) - }) - }) - - describe('collaborativeDocWithSource', () => { - it('updates collaborative doc version id', async () => { - expect( - collaborativeDocWithSource('doc1:HEAD:HEAD' as CollaborativeDoc, 'doc2:v1:v1' as CollaborativeDoc) - ).toEqual('doc1:HEAD:HEAD#doc2:v1:v1' as CollaborativeDoc) - expect(collaborativeDocWithSource('doc1:v1:v1' as CollaborativeDoc, 'doc2:v2:v2' as CollaborativeDoc)).toEqual( - 'doc1:v1:v1#doc2:v2:v2' as CollaborativeDoc - ) - expect( - collaborativeDocWithSource('doc1:v1:v1' as CollaborativeDoc, 'doc2:v2:v2#doc3:v3:v3' as CollaborativeDoc) - ).toEqual('doc1:v1:v1#doc2:v2:v2#doc3:v3:v3' as CollaborativeDoc) - expect( - collaborativeDocWithSource('doc1:v1:v1#doc2:v2:v2' as CollaborativeDoc, 'doc3:v3:v3' as CollaborativeDoc) - ).toEqual('doc1:v1:v1#doc3:v3:v3' as CollaborativeDoc) - }) - }) - - describe('collaborativeDocFromLastVersion', () => { - it('returns valid collaborative doc id', async () => { - expect(collaborativeDocFromLastVersion('doc1:HEAD:HEAD#doc2:main:v2#doc3:main:v3' as CollaborativeDoc)).toEqual( - 'doc1:HEAD:HEAD#doc2:main:v2#doc3:main:v3' - ) - expect(collaborativeDocFromLastVersion('doc1:HEAD:v1#doc2:main:v2#doc3:main:v3' as CollaborativeDoc)).toEqual( - 'doc1:v1:v1#doc2:main:v2#doc3:main:v3' - ) - expect(collaborativeDocFromLastVersion('doc1:v1:v1#doc2:main:v2#doc3:main:v3' as CollaborativeDoc)).toEqual( - 'doc1:v1:v1#doc2:main:v2#doc3:main:v3' - ) - expect(collaborativeDocFromLastVersion('doc1:HEAD:v1' as CollaborativeDoc)).toEqual('doc1:v1:v1') - }) - }) - - describe('collaborativeDocFromCollaborativeDoc', () => { - it('returns valid collaborative doc id', async () => { - expect( - collaborativeDocFromCollaborativeDoc( - 'doc1:HEAD:HEAD' as CollaborativeDoc, - 'doc2:HEAD:v2#doc3:v3:v3' as CollaborativeDoc - ) - ).toEqual('doc1:HEAD:HEAD#doc2:v2:v2#doc3:v3:v3') - - expect( - collaborativeDocFromCollaborativeDoc( - 'doc1:HEAD:HEAD' as CollaborativeDoc, - 'doc2:v2:v2#doc3:v3:v3' as CollaborativeDoc - ) - ).toEqual('doc1:HEAD:HEAD#doc2:v2:v2#doc3:v3:v3') - - expect( - collaborativeDocFromCollaborativeDoc( - 'doc1:HEAD:HEAD' as CollaborativeDoc, - 'doc2:HEAD:HEAD#doc3:v3:v3' as CollaborativeDoc - ) - ).toEqual('doc1:HEAD:HEAD#doc2:HEAD:HEAD#doc3:v3:v3') - }) - }) -}) diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index 739ecebc11..e3f8043059 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -15,7 +15,6 @@ // import type { Asset, IntlString, Plugin } from '@hcengineering/platform' -import { CollaborativeDoc } from './collaboration' import type { DocumentQuery } from './storage' /** @@ -55,6 +54,13 @@ export type CollectionSize = T[]['length'] */ export type Rank = string +/** + * @public + * + * Reference to blob containing snapshot of collaborative doc. + */ +export type MarkupBlobRef = Ref + /** * @public */ @@ -76,7 +82,7 @@ export interface Doc extends Obj { export interface Card extends Doc { title: string - description?: CollaborativeDoc | null + description?: MarkupBlobRef | null identifier?: string parent?: Ref | null } diff --git a/packages/core/src/collaboration.ts b/packages/core/src/collaboration.ts index 77177ba966..1042ef561c 100644 --- a/packages/core/src/collaboration.ts +++ b/packages/core/src/collaboration.ts @@ -13,175 +13,41 @@ // limitations under the License. // -import { Doc, Ref } from './classes' - -/** - * Identifier of the collaborative document holding collaborative content. - * - * Format: - * {documentId}:{versionId}:{lastVersionId} - * {documentId}:{versionId} - * - * Where: - * - documentId is an identifier of the document in storage - * - versionId is an identifier of the document version, HEAD for latest editable version - * - lastVersionId is an identifier of the latest available version - * - * The collaborative document may contain one or more such sections chained with # (hash): - * collaborativeDocId#collaborativeDocId#collaborativeDocId#... - * - * When collaborative document does not exist, it will be initialized from the first existing - * document in the list. - * - * @public - * */ -export type CollaborativeDoc = string & { __collaborativeDoc: true } +import type { Class, MarkupBlobRef, Doc, Ref } from './classes' /** @public */ -export type CollaborativeDocVersion = string | typeof CollaborativeDocVersionHead +export interface CollaborativeDoc { + objectClass: Ref> + objectId: Ref + objectAttr: string +} /** @public */ -export const CollaborativeDocVersionHead = 'HEAD' - -/** @public */ -export function makeCollaborativeDoc ( - objectId: Ref, - objectAttr?: string | undefined, - versionId?: string | undefined +export function makeCollabId ( + objectClass: Ref>, + objectId: Ref, + objectAttr: Extract | string ): CollaborativeDoc { - const storageDocumentId = objectAttr !== undefined && objectAttr !== '' ? `${objectId}%${objectAttr}` : `${objectId}` - return collaborativeDocFormat({ - documentId: storageDocumentId, - versionId: CollaborativeDocVersionHead, - lastVersionId: versionId ?? '0' - }) + return { objectClass, objectId, objectAttr } } /** @public */ -export interface CollaborativeDocData { - // Id of the document in object storage - documentId: string - // Id of the document version - // HEAD version represents the editable last document version - // Otherwise, it is a readonly version - versionId: CollaborativeDocVersion - // For HEAD versionId it is the latest available document version - // Otherwise, it is the same value as versionId - lastVersionId: string - - source?: CollaborativeDoc[] -} - -/** - * Merge several collaborative docs into single collaborative doc train. - * - * @public - */ -export function collaborativeDocChain (...docs: CollaborativeDoc[]): CollaborativeDoc { - return docs.join('#') as CollaborativeDoc -} - -/** - * Split collaborative doc train into separate collaborative docs. - * - * @public - */ -export function collaborativeDocUnchain (doc: CollaborativeDoc): CollaborativeDoc[] { - return doc.split('#') as CollaborativeDoc[] +export function makeDocCollabId ( + doc: T, + objectAttr: Extract | string +): CollaborativeDoc { + return makeCollabId(doc._class, doc._id, objectAttr) } /** @public */ -export function collaborativeDocParse (doc: CollaborativeDoc): CollaborativeDocData { - const [first, ...other] = collaborativeDocUnchain(doc) - const [documentId, versionId, lastVersionId] = first.split(':') - return { - documentId, - versionId: versionId ?? CollaborativeDocVersionHead, - lastVersionId: lastVersionId ?? versionId ?? CollaborativeDocVersionHead, - source: other - } +export function makeCollabYdocId (doc: CollaborativeDoc): MarkupBlobRef { + const { objectId, objectAttr } = doc + return `${objectId}%${objectAttr}` as MarkupBlobRef } -const sanitize = (value: string): string => value.replace(/[:#]/g, '%') - /** @public */ -export function collaborativeDocFormat ({ - documentId, - versionId, - lastVersionId, - source -}: CollaborativeDocData): CollaborativeDoc { - const parts = [sanitize(documentId), sanitize(versionId), sanitize(lastVersionId)] - const collaborativeDoc = parts.join(':') as CollaborativeDoc - return collaborativeDocChain(collaborativeDoc, ...(source ?? [])) -} - -/** - * Updates versionId component in the collaborative document. - * Both versionId and lastVersionId will refer to the same collaborative document version. - * - * When versionId is not HEAD, the document will represent a readonly document version (snapshot). - * - * @public - */ -export function collaborativeDocWithVersion (collaborativeDoc: CollaborativeDoc, versionId: string): CollaborativeDoc { - const { documentId, source } = collaborativeDocParse(collaborativeDoc) - return collaborativeDocFormat({ documentId, versionId, lastVersionId: versionId, source }) -} - -/** - * Updates lastVersionId component in the collaborative document. - * - * When document versionId is HEAD, the function is no-op. - * - * @public - */ -export function collaborativeDocWithLastVersion ( - collaborativeDoc: CollaborativeDoc, - lastVersionId: string -): CollaborativeDoc { - const { documentId, versionId, source } = collaborativeDocParse(collaborativeDoc) - return versionId === CollaborativeDocVersionHead - ? collaborativeDocFormat({ documentId, versionId, lastVersionId, source }) - : collaborativeDoc -} - -/** - * Replaces source component in the collaborative document. - * - * @public - */ -export function collaborativeDocWithSource ( - collaborativeDoc: CollaborativeDoc, - source: CollaborativeDoc -): CollaborativeDoc { - const { documentId, versionId, lastVersionId } = collaborativeDocParse(collaborativeDoc) - return collaborativeDocFormat({ documentId, versionId, lastVersionId, source: [source] }) -} - -/** - * Creates collaborative document that refers to the last version from the source collaborative document. - * - * @public - */ -export function collaborativeDocFromLastVersion (collaborativeDoc: CollaborativeDoc): CollaborativeDoc { - const { documentId, lastVersionId, source } = collaborativeDocParse(collaborativeDoc) - return collaborativeDocFormat({ - documentId, - versionId: lastVersionId, - lastVersionId, - source - }) -} - -/** - * Creates collaborative document that refers to the last version from the source collaborative document. - * - * @public - */ -export function collaborativeDocFromCollaborativeDoc ( - collaborativeDoc: CollaborativeDoc, - sourceCollaborativeDoc: CollaborativeDoc -): CollaborativeDoc { - return collaborativeDocWithSource(collaborativeDoc, collaborativeDocFromLastVersion(sourceCollaborativeDoc)) +export function makeCollabJsonId (doc: CollaborativeDoc): MarkupBlobRef { + const timestamp = Date.now() + const { objectId, objectAttr } = doc + return [objectId, objectAttr, timestamp].join('-') as MarkupBlobRef } diff --git a/packages/core/src/component.ts b/packages/core/src/component.ts index 4ed7db7dff..398c937502 100644 --- a/packages/core/src/component.ts +++ b/packages/core/src/component.ts @@ -23,6 +23,7 @@ import type { Blob, Card, Class, + MarkupBlobRef, Collection, Configuration, ConfigurationElement, @@ -54,7 +55,6 @@ import type { TypedSpace, UserStatus } from './classes' -import { CollaborativeDoc } from './collaboration' import { Status, StatusCategory } from './status' import type { Tx, @@ -120,8 +120,7 @@ export default plugin(coreId, { TypeBoolean: '' as Ref>>, TypeTimestamp: '' as Ref>>, TypeDate: '' as Ref>>, - TypeCollaborativeDoc: '' as Ref>>, - TypeCollaborativeDocVersion: '' as Ref>>, + TypeCollaborativeDoc: '' as Ref>>, RefTo: '' as Ref>>, ArrOf: '' as Ref>>, Enum: '' as Ref>, @@ -185,9 +184,8 @@ export default plugin(coreId, { String: '' as IntlString, Record: '' as IntlString, Markup: '' as IntlString, - Collaborative: '' as IntlString, CollaborativeDoc: '' as IntlString, - CollaborativeDocVersion: '' as IntlString, + MarkupBlobRef: '' as IntlString, Number: '' as IntlString, Boolean: '' as IntlString, Timestamp: '' as IntlString, diff --git a/packages/model/src/dsl.ts b/packages/model/src/dsl.ts index 430aeaea72..5741db1fb9 100644 --- a/packages/model/src/dsl.ts +++ b/packages/model/src/dsl.ts @@ -20,7 +20,7 @@ import core, { Class, Classifier, ClassifierKind, - CollaborativeDoc, + MarkupBlobRef, Data, DateRangeMode, Doc, @@ -500,15 +500,8 @@ export function ArrOf> (type: Type): TypeAr /** * @public */ -export function TypeCollaborativeDoc (): Type { - return { _class: core.class.TypeCollaborativeDoc, label: core.string.CollaborativeDoc } -} - -/** - * @public - */ -export function TypeCollaborativeDocVersion (): Type { - return { _class: core.class.TypeCollaborativeDocVersion, label: core.string.CollaborativeDocVersion } +export function TypeCollaborativeDoc (): Type { + return { _class: core.class.TypeCollaborativeDoc, label: core.string.MarkupBlobRef } } /** diff --git a/packages/presentation/src/collaborator.ts b/packages/presentation/src/collaborator.ts index 712a90d0b0..8b3ad682e2 100644 --- a/packages/presentation/src/collaborator.ts +++ b/packages/presentation/src/collaborator.ts @@ -14,13 +14,12 @@ // import { type CollaboratorClient, getClient as getCollaborator } from '@hcengineering/collaborator-client' -import { type CollaborativeDoc, type Markup, getWorkspaceId } from '@hcengineering/core' +import { type Blob, type CollaborativeDoc, type Markup, type Ref, getWorkspaceId } from '@hcengineering/core' import { getMetadata } from '@hcengineering/platform' import presentation from './plugin' -/** @public */ -export function getCollaboratorClient (): CollaboratorClient { +function getClient (): CollaboratorClient { const workspaceId = getWorkspaceId(getMetadata(presentation.metadata.WorkspaceId) ?? '') const token = getMetadata(presentation.metadata.Token) ?? '' const collaboratorURL = getMetadata(presentation.metadata.CollaboratorUrl) ?? '' @@ -29,19 +28,25 @@ export function getCollaboratorClient (): CollaboratorClient { } /** @public */ -export async function getMarkup (collaborativeDoc: CollaborativeDoc): Promise> { - const client = getCollaboratorClient() - return await client.getContent(collaborativeDoc) +export async function getMarkup (doc: CollaborativeDoc, source: Ref | null | undefined): Promise { + const client = getClient() + return await client.getMarkup(doc, source) } /** @public */ -export async function updateMarkup (collaborativeDoc: CollaborativeDoc, content: Record): Promise { - const client = getCollaboratorClient() - await client.updateContent(collaborativeDoc, content) +export async function createMarkup (doc: CollaborativeDoc, markup: Markup): Promise> { + const client = getClient() + return await client.createMarkup(doc, markup) } /** @public */ -export async function copyDocument (source: CollaborativeDoc, target: CollaborativeDoc): Promise { - const client = getCollaboratorClient() +export async function updateMarkup (doc: CollaborativeDoc, markup: Markup): Promise { + const client = getClient() + await client.updateMarkup(doc, markup) +} + +/** @public */ +export async function copyMarkup (source: CollaborativeDoc, target: CollaborativeDoc): Promise { + const client = getClient() await client.copyContent(source, target) } diff --git a/packages/presentation/src/file.ts b/packages/presentation/src/file.ts index 5d1fa55756..a49b011d4a 100644 --- a/packages/presentation/src/file.ts +++ b/packages/presentation/src/file.ts @@ -173,17 +173,18 @@ export function getFileUrl (file: string, filename?: string): string { /** * @public */ -export async function uploadFile (file: File): Promise> { - const id = generateFileId() - const params = getFileUploadParams(id, file) +export async function uploadFile (file: File, uuid?: Ref): Promise> { + uuid ??= generateFileId() as Ref + + const params = getFileUploadParams(uuid, file) if (params.method === 'signed-url') { - await uploadFileWithSignedUrl(file, id, params.url) + await uploadFileWithSignedUrl(file, uuid, params.url) } else { - await uploadFileWithFormData(file, id, params.url) + await uploadFileWithFormData(file, uuid, params.url) } - return id as Ref + return uuid } /** diff --git a/plugins/bitrix/src/hr.ts b/plugins/bitrix/src/hr.ts index 1ed9d9faa7..f36c7605dc 100644 --- a/plugins/bitrix/src/hr.ts +++ b/plugins/bitrix/src/hr.ts @@ -8,8 +8,7 @@ import core, { SortingOrder, Status, TxOperations, - generateId, - makeCollaborativeDoc + generateId } from '@hcengineering/core' import recruit, { Applicant, Vacancy } from '@hcengineering/recruit' import task, { ProjectType, makeRank } from '@hcengineering/task' @@ -41,7 +40,7 @@ export async function createVacancy ( { name, description: type.shortDescription ?? '', - fullDescription: makeCollaborativeDoc(id, 'fullDescription'), + fullDescription: null, private: false, archived: false, company, diff --git a/plugins/contact-resources/src/components/CreateOrganization.svelte b/plugins/contact-resources/src/components/CreateOrganization.svelte index 36f8e07f88..4818633fb9 100644 --- a/plugins/contact-resources/src/components/CreateOrganization.svelte +++ b/plugins/contact-resources/src/components/CreateOrganization.svelte @@ -20,13 +20,13 @@ AttachedData, fillDefaults, generateId, - makeCollaborativeDoc, + makeCollabId, Ref, TxOperations, WithLookup } from '@hcengineering/core' - import { Card, getClient, InlineAttributeBar, updateMarkup } from '@hcengineering/presentation' - import { EmptyMarkup } from '@hcengineering/text' + import { Card, createMarkup, getClient, InlineAttributeBar } from '@hcengineering/presentation' + import { EmptyMarkup, isEmptyMarkup } from '@hcengineering/text' import { Button, createFocusManager, EditBox, FocusHandler, IconAttachment, IconInfo, Label } from '@hcengineering/ui' import { createEventDispatcher } from 'svelte' @@ -46,7 +46,6 @@ const object: Organization = { name: '', - description: makeCollaborativeDoc(id, 'description'), attachments: 0 } as unknown as Organization @@ -59,8 +58,11 @@ fillDefaults(hierarchy, object, contact.class.Organization) async function createOrganization (): Promise { + if (!isEmptyMarkup(description)) { + const target = makeCollabId(contact.class.Organization, id, 'description') + object.description = await createMarkup(target, description) + } const op = client.apply() - await updateMarkup(object.description, { description }) await op.createDoc(contact.class.Organization, contact.space.Contacts, object, id) await descriptionBox.createAttachments(id, op) diff --git a/plugins/contact/src/index.ts b/plugins/contact/src/index.ts index bb6d5f077b..b19b6ff83d 100644 --- a/plugins/contact/src/index.ts +++ b/plugins/contact/src/index.ts @@ -18,13 +18,13 @@ import { Account, AttachedDoc, Class, - CollaborativeDoc, Doc, Ref, Space, Timestamp, UXObject, type Blob, + type MarkupBlobRef, type Data, type WithLookup } from '@hcengineering/core' @@ -137,7 +137,7 @@ export interface Member extends AttachedDoc { */ export interface Organization extends Contact { members: number - description: CollaborativeDoc + description: MarkupBlobRef | null } /** diff --git a/plugins/controlled-documents-resources/src/components/CreateDocument.svelte b/plugins/controlled-documents-resources/src/components/CreateDocument.svelte index 310275ad53..7b60de7287 100644 --- a/plugins/controlled-documents-resources/src/components/CreateDocument.svelte +++ b/plugins/controlled-documents-resources/src/components/CreateDocument.svelte @@ -17,17 +17,8 @@ -{#if $controlledDocument !== null && collaborativeDoc !== undefined} +{#if $controlledDocument !== null && attribute !== undefined} - + {#if $editorMode === 'comparing'} {:else} diff --git a/plugins/controlled-documents-resources/src/components/create-doc/QmsDocumentWizard.svelte b/plugins/controlled-documents-resources/src/components/create-doc/QmsDocumentWizard.svelte index eebd20c597..ae43692bd9 100644 --- a/plugins/controlled-documents-resources/src/components/create-doc/QmsDocumentWizard.svelte +++ b/plugins/controlled-documents-resources/src/components/create-doc/QmsDocumentWizard.svelte @@ -18,13 +18,12 @@ import { generateId, getCurrentAccount, - makeCollaborativeDoc, type AttachedData, type Class, type Data, type Ref } from '@hcengineering/core' - import { MessageBox, copyDocument, getClient } from '@hcengineering/presentation' + import { MessageBox, getClient } from '@hcengineering/presentation' import { AnySvelteComponent, addNotification, @@ -43,10 +42,10 @@ type DocumentTemplate, DocumentState, createChangeControl, - createControlledDocFromTemplate, DEFAULT_PERIODIC_REVIEW_INTERVAL } from '@hcengineering/controlled-documents' + import { createControlledDocFromTemplate } from '../../docutils' import documents from '../../plugin' import { getProjectDocumentLink } from '../../navigation' import InfoStep from './steps/InfoStep.svelte' @@ -122,7 +121,7 @@ state: DocumentState.Draft, snapshots: 0, changeControl: ccRecordId, - content: makeCollaborativeDoc(generateId()), + content: null, requests: 0, reviewers: [], @@ -161,8 +160,7 @@ _space, $locationStep.project, $locationStep.parent, - documents.class.ControlledDocument, - copyDocument + documents.class.ControlledDocument ) if (!success) { diff --git a/plugins/controlled-documents-resources/src/components/create-doc/QmsTemplateWizard.svelte b/plugins/controlled-documents-resources/src/components/create-doc/QmsTemplateWizard.svelte index 32b89c7aa8..1c7a2d6560 100644 --- a/plugins/controlled-documents-resources/src/components/create-doc/QmsTemplateWizard.svelte +++ b/plugins/controlled-documents-resources/src/components/create-doc/QmsTemplateWizard.svelte @@ -32,8 +32,7 @@ type Ref, type Mixin, generateId, - getCurrentAccount, - makeCollaborativeDoc + getCurrentAccount } from '@hcengineering/core' import { MessageBox, getClient } from '@hcengineering/presentation' import { @@ -118,7 +117,7 @@ state: DocumentState.Draft, snapshots: 0, changeControl: ccRecordId, - content: makeCollaborativeDoc(generateId()), + content: null, requests: 0, reviewers: [], diff --git a/plugins/controlled-documents-resources/src/components/document/DocumentDiffViewer.svelte b/plugins/controlled-documents-resources/src/components/document/DocumentDiffViewer.svelte index 802da8264e..af125cfc9f 100644 --- a/plugins/controlled-documents-resources/src/components/document/DocumentDiffViewer.svelte +++ b/plugins/controlled-documents-resources/src/components/document/DocumentDiffViewer.svelte @@ -1,7 +1,7 @@ -{#if $controlledDocument && collaborativeDoc} +{#if $controlledDocument && attribute} {#if headings.length > 0} @@ -270,14 +264,10 @@ | undefined, + documentId: Ref, + spec: AttachedData, + space: Ref, + project: Ref | undefined, + parent: Ref | undefined, + docClass: Ref> = documents.class.ControlledDocument +): Promise<{ seqNumber: number, success: boolean }> { + const result = await controlledDocFromTemplate(client, templateId, documentId, spec, space, project, parent, docClass) + + if (result.success && templateId !== undefined) { + const source = makeCollabId(documents.mixin.DocumentTemplate, templateId, 'content') + const target = makeCollabId(docClass, documentId, 'content') + try { + await copyMarkup(source, target) + } catch (err) { + await setPlatformStatus(unknownError(err)) + return { ...result, success: false } + } + } + + return result +} + export async function createNewDraftForControlledDoc ( client: TxOperations, document: ControlledDocument, @@ -65,18 +94,6 @@ export async function createNewDraftForControlledDoc ( ops.notMatch(documents.class.Document, notMatchQuery) - const collaborativeDoc = getCollaborativeDocForDocument( - `DOC-${document.prefix}`, - document.seqNumber, - document.major, - document.minor, - true - ) - - if (document.content !== undefined) { - await copyDocument(document.content, collaborativeDoc) - } - // Create new change control for new version const newCCId = generateId() const newCCSpec: Data = { @@ -111,7 +128,7 @@ export async function createNewDraftForControlledDoc ( labels: 0, state: DocumentState.Draft, plannedEffectiveDate: 0, - content: collaborativeDoc + content: document.content } const meta = await client.findOne(documents.class.ProjectMeta, { @@ -170,16 +187,6 @@ export async function createNewDraftForControlledDoc ( } export async function createDocumentSnapshotAndEdit (client: TxOperations, document: ControlledDocument): Promise { - const collaborativeDoc = getCollaborativeDocForDocument( - `DOC-${document.prefix}`, - document.seqNumber, - document.major, - document.minor, - true - ) - - await copyDocument(document.content, collaborativeDoc) - const language = get(themeStore).language const namePrefix = await translate(documents.string.DraftRevision, {}, language) const name = `${namePrefix} ${(document.snapshots ?? 0) + 1}` @@ -197,7 +204,7 @@ export async function createDocumentSnapshotAndEdit (client: TxOperations, docum name, state: document.state, controlledState: document.controlledState, - content: collaborativeDoc + content: document.content }, newSnapshotId ) @@ -205,6 +212,10 @@ export async function createDocumentSnapshotAndEdit (client: TxOperations, docum await op.commit() await client.update(document, { controlledState: undefined }) + + const source = makeDocCollabId(document, 'content') + const target = makeCollabId(documents.class.ControlledDocumentSnapshot, newSnapshotId, 'content') + await copyMarkup(source, target) } export function getDocumentTrainingClass (hierarchy: Hierarchy): Class { diff --git a/plugins/controlled-documents/src/docutils.ts b/plugins/controlled-documents/src/docutils.ts index 29e2815475..1e7afaad44 100644 --- a/plugins/controlled-documents/src/docutils.ts +++ b/plugins/controlled-documents/src/docutils.ts @@ -14,17 +14,7 @@ // import { type Employee } from '@hcengineering/contact' -import { - type AttachedData, - type Class, - type CollaborativeDoc, - type Doc, - type Ref, - type TxOperations, - Mixin, - generateId, - makeCollaborativeDoc -} from '@hcengineering/core' +import { type AttachedData, type Class, type Ref, type TxOperations, Blob, Mixin } from '@hcengineering/core' import { type Document, type DocumentTemplate, @@ -40,7 +30,6 @@ import { import documents from './plugin' import { TEMPLATE_PREFIX } from './utils' -import { setPlatformStatus, unknownError } from '@hcengineering/platform' async function getParentPath (client: TxOperations, parent: Ref): Promise>> { const parentDocObj = await client.findOne(documents.class.ProjectDocument, { @@ -72,8 +61,7 @@ export async function createControlledDocFromTemplate ( space: Ref, project: Ref | undefined, parent: Ref | undefined, - docClass: Ref> = documents.class.ControlledDocument, - copyContent: (source: CollaborativeDoc, target: CollaborativeDoc) => Promise = async () => {} + docClass: Ref> = documents.class.ControlledDocument ): Promise<{ seqNumber: number, success: boolean }> { if (templateId == null) { return { seqNumber: -1, success: false } @@ -101,8 +89,6 @@ export async function createControlledDocFromTemplate ( const seqNumber = template.sequence + 1 const prefix = template.docPrefix - const _copyContent = (doc: CollaborativeDoc): Promise => copyContent(template.content, doc) - return await createControlledDoc( client, templateId, @@ -114,7 +100,7 @@ export async function createControlledDocFromTemplate ( seqNumber, path, docClass, - _copyContent + template.content ) } @@ -129,12 +115,10 @@ async function createControlledDoc ( seqNumber: number, path: Ref[] = [], docClass: Ref> = documents.class.ControlledDocument, - copyContent: (doc: CollaborativeDoc) => Promise + content: Ref | null ): Promise<{ seqNumber: number, success: boolean }> { const projectId = project ?? documents.ids.NoProject - const collaborativeDoc = getCollaborativeDocForDocument(`DOC-${prefix}`, seqNumber, 0, 1) - const ops = client.apply() ops.notMatch(documents.class.Document, { @@ -184,22 +168,12 @@ async function createControlledDoc ( seqNumber, prefix, state: DocumentState.Draft, - content: collaborativeDoc + content }, documentId ) const success = await ops.commit() - - if (success.result) { - try { - await copyContent(collaborativeDoc) - } catch (err) { - await setPlatformStatus(unknownError(err)) - return { seqNumber, success: false } - } - } - return { seqNumber, success: success.result } } @@ -228,7 +202,6 @@ export async function createDocumentTemplate ( true ) const seqNumber = (incResult as any).object.sequence as number - const collaborativeDocId = getCollaborativeDocForDocument('TPL-DOC', seqNumber, 0, 1) const code = spec.code === '' ? `${TEMPLATE_PREFIX}-${seqNumber}` : spec.code let path: Array> = [] @@ -292,7 +265,7 @@ export async function createDocumentTemplate ( prefix: TEMPLATE_PREFIX, author, owner: author, - content: collaborativeDocId + content: null }, templateId ) @@ -306,19 +279,3 @@ export async function createDocumentTemplate ( return { seqNumber, success: success.result } } - -export function getCollaborativeDocForDocument ( - prefix: string, - seqNumber: number, - major: number, - minor: number, - next: boolean = false -): CollaborativeDoc { - if (prefix.endsWith('-')) { - prefix = prefix.substring(0, prefix.length - 1) - } - - return makeCollaborativeDoc( - (`${prefix}-${seqNumber}-${major}.${minor}${next ? '.next' : ''}-` + generateId()) as Ref - ) -} diff --git a/plugins/controlled-documents/src/types.ts b/plugins/controlled-documents/src/types.ts index 31119266a6..4ec4a756ac 100644 --- a/plugins/controlled-documents/src/types.ts +++ b/plugins/controlled-documents/src/types.ts @@ -5,10 +5,10 @@ import { Attachment } from '@hcengineering/attachment' import { ChatMessage } from '@hcengineering/chunter' import { Employee } from '@hcengineering/contact' import { - CollaborativeDoc, type AttachedDoc, type Class, type CollectionSize, + type MarkupBlobRef, type Doc, type Markup, type Ref, @@ -122,7 +122,7 @@ export interface Document extends Doc { author?: Ref // Employee who created/released the document owner?: Ref // Employee responsible for working on the document state: DocumentState - content: CollaborativeDoc + content: MarkupBlobRef | null labels?: CollectionSize // A collection of attached tags(labels) abstract?: string commentSequence: number // Used to enumerate the comments across revisions of the working copy of the document @@ -138,7 +138,7 @@ export interface Document extends Doc { */ export interface DocumentSnapshot extends AttachedDoc { name?: string - content: CollaborativeDoc + content: MarkupBlobRef | null state?: DocumentState } diff --git a/plugins/document-resources/src/components/DocumentEditor.svelte b/plugins/document-resources/src/components/DocumentEditor.svelte index afc26cc971..547023115e 100644 --- a/plugins/document-resources/src/components/DocumentEditor.svelte +++ b/plugins/document-resources/src/components/DocumentEditor.svelte @@ -16,8 +16,9 @@ --> = { title, - content: makeCollaborativeDoc(id, 'content'), + content: null, attachments: 0, embeddings: 0, labels: 0, diff --git a/plugins/document/src/types.ts b/plugins/document/src/types.ts index 8b650bcc49..767afdbb45 100644 --- a/plugins/document/src/types.ts +++ b/plugins/document/src/types.ts @@ -14,7 +14,7 @@ // import { Attachment } from '@hcengineering/attachment' -import { Account, Class, CollaborativeDoc, Doc, Rank, Ref, TypedSpace } from '@hcengineering/core' +import { Account, Class, MarkupBlobRef, Doc, Rank, Ref, TypedSpace } from '@hcengineering/core' import { Preference } from '@hcengineering/preference' import { IconProps } from '@hcengineering/view' @@ -24,7 +24,7 @@ export interface Teamspace extends TypedSpace, IconProps {} /** @public */ export interface Document extends Doc, IconProps { title: string - content: CollaborativeDoc + content: MarkupBlobRef | null parent: Ref space: Ref @@ -43,7 +43,7 @@ export interface Document extends Doc, IconProps { /** @public */ export interface DocumentSnapshot extends Doc { title: string - content: CollaborativeDoc + content: MarkupBlobRef parent: Ref } diff --git a/plugins/lead-resources/src/components/CreateCustomer.svelte b/plugins/lead-resources/src/components/CreateCustomer.svelte index b218fc30be..f2ef95ccdd 100644 --- a/plugins/lead-resources/src/components/CreateCustomer.svelte +++ b/plugins/lead-resources/src/components/CreateCustomer.svelte @@ -25,12 +25,12 @@ Ref, WithLookup, generateId, - makeCollaborativeDoc + makeCollabId } from '@hcengineering/core' import { Customer, LeadEvents } from '@hcengineering/lead' - import { Card, getClient, InlineAttributeBar, updateMarkup } from '@hcengineering/presentation' + import { Card, createMarkup, getClient, InlineAttributeBar } from '@hcengineering/presentation' import { StyledTextBox } from '@hcengineering/text-editor-resources' - import { EmptyMarkup } from '@hcengineering/text' + import { EmptyMarkup, isEmptyMarkup } from '@hcengineering/text' import { Button, createFocusManager, @@ -85,14 +85,17 @@ } if (client.getHierarchy().isDerived(targetClass._id, contact.class.Organization)) { - ;(candidate as Organization).description = makeCollaborativeDoc(customerId, 'description') + ;(candidate as Organization).description = null } const candidateData: MixinData = { - customerDescription: makeCollaborativeDoc(customerId, 'customerDescription') + customerDescription: null } - await updateMarkup(candidateData.customerDescription, { customerDescription: description }) + if (!isEmptyMarkup(description)) { + const collabId = makeCollabId(lead.mixin.Customer, customerId, 'customerDescription') + candidateData.customerDescription = await createMarkup(collabId, description) + } const id = await client.createDoc(targetClass._id, contact.space.Contacts, { ...candidate, ...object }, customerId) await client.createMixin( diff --git a/plugins/lead/src/index.ts b/plugins/lead/src/index.ts index d9fc1f0fee..f30be8eb82 100644 --- a/plugins/lead/src/index.ts +++ b/plugins/lead/src/index.ts @@ -15,7 +15,7 @@ // import type { Contact } from '@hcengineering/contact' -import type { Attribute, Class, CollaborativeDoc, Doc, Markup, Ref, Status, Timestamp } from '@hcengineering/core' +import type { Attribute, Class, MarkupBlobRef, Doc, Markup, Ref, Status, Timestamp } from '@hcengineering/core' import { Mixin } from '@hcengineering/core' import type { Asset, IntlString, Plugin } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' @@ -36,7 +36,7 @@ export interface Funnel extends Project { export interface Customer extends Contact { leads?: number - customerDescription: CollaborativeDoc + customerDescription: MarkupBlobRef | null } /** diff --git a/plugins/love-resources/src/components/AddRoomPopup.svelte b/plugins/love-resources/src/components/AddRoomPopup.svelte index 4d61e0b1fd..719eacc0c3 100644 --- a/plugins/love-resources/src/components/AddRoomPopup.svelte +++ b/plugins/love-resources/src/components/AddRoomPopup.svelte @@ -1,5 +1,5 @@ -{#key _documentId} +{#key documentId} {/key} diff --git a/plugins/text-editor-resources/src/components/CollaborativeAttributeBox.svelte b/plugins/text-editor-resources/src/components/CollaborativeAttributeBox.svelte index 18cd027ba9..3742b98e3a 100644 --- a/plugins/text-editor-resources/src/components/CollaborativeAttributeBox.svelte +++ b/plugins/text-editor-resources/src/components/CollaborativeAttributeBox.svelte @@ -15,7 +15,7 @@ -{#if collaborativeDoc != null} - -{/if} + diff --git a/plugins/text-editor-resources/src/components/CollaborativeTextEditor.svelte b/plugins/text-editor-resources/src/components/CollaborativeTextEditor.svelte index aaa4a66110..dfce9ff788 100644 --- a/plugins/text-editor-resources/src/components/CollaborativeTextEditor.svelte +++ b/plugins/text-editor-resources/src/components/CollaborativeTextEditor.svelte @@ -16,9 +16,16 @@ --> {#if issue} @@ -92,11 +96,11 @@

- {#if issue.description} + {#if descriptionKey}
limit} style:max-height={`${limit}px`}>
(cHeight = element.clientHeight)}> {#key issue._id} - + {/key}
diff --git a/plugins/tracker/src/index.ts b/plugins/tracker/src/index.ts index 3ba06ba655..bec155e700 100644 --- a/plugins/tracker/src/index.ts +++ b/plugins/tracker/src/index.ts @@ -18,7 +18,7 @@ import { AttachedDoc, Attribute, Class, - CollaborativeDoc, + MarkupBlobRef, CollectionSize, Data, Doc, @@ -183,7 +183,7 @@ export interface Milestone extends Doc { export interface Issue extends Task { attachedTo: Ref title: string - description: CollaborativeDoc + description: MarkupBlobRef | null status: Ref priority: IssuePriority diff --git a/server-plugins/activity-resources/src/references.ts b/server-plugins/activity-resources/src/references.ts index 3c4d12fa45..f97426cb8a 100644 --- a/server-plugins/activity-resources/src/references.ts +++ b/server-plugins/activity-resources/src/references.ts @@ -14,12 +14,12 @@ // import activity, { ActivityMessage, ActivityReference, UserMentionInfo } from '@hcengineering/activity' -import { loadCollaborativeDoc, yDocToBuffer } from '@hcengineering/collaboration' +import { loadCollabJson } from '@hcengineering/collaboration' import contact, { Employee, Person, PersonAccount } from '@hcengineering/contact' import core, { Account, + Blob, Class, - CollaborativeDoc, Data, Doc, generateId, @@ -416,21 +416,17 @@ async function getCreateReferencesTxes ( refs.push(...attrReferences) } else if (attr.type._class === core.class.TypeCollaborativeDoc) { - const collaborativeDoc = (createdDoc as any)[attr.name] as CollaborativeDoc - try { - const ydoc = await loadCollaborativeDoc(ctx, storage, control.workspace, collaborativeDoc) - if (ydoc !== undefined) { - const attrReferences = getReferencesData( - srcDocId, - srcDocClass, - attachedDocId, - attachedDocClass, - yDocToBuffer(ydoc) - ) - refs.push(...attrReferences) + const blobId = (createdDoc as any)[attr.name] as Ref + if (blobId != null) { + try { + const markup = await loadCollabJson(ctx, storage, control.workspace, blobId) + if (markup !== undefined) { + const attrReferences = getReferencesData(srcDocId, srcDocClass, attachedDocId, attachedDocClass, markup) + refs.push(...attrReferences) + } + } catch { + // do nothing, the collaborative doc does not sem to exist yet } - } catch { - // do nothing, the collaborative doc does not sem to exist yet } } } @@ -469,17 +465,13 @@ async function getUpdateReferencesTxes ( } else if (attr.type._class === core.class.TypeCollaborativeDoc) { hasReferenceAttrs = true try { - const collaborativeDoc = (updatedDoc as any)[attr.name] as CollaborativeDoc - const ydoc = await loadCollaborativeDoc(ctx, storage, control.workspace, collaborativeDoc) - if (ydoc !== undefined) { - const attrReferences = getReferencesData( - srcDocId, - srcDocClass, - attachedDocId, - attachedDocClass, - yDocToBuffer(ydoc) - ) - references.push(...attrReferences) + const blobId = (updatedDoc as any)[attr.name] as Ref + if (blobId != null) { + const markup = await loadCollabJson(ctx, storage, control.workspace, blobId) + if (markup !== undefined) { + const attrReferences = getReferencesData(srcDocId, srcDocClass, attachedDocId, attachedDocClass, markup) + references.push(...attrReferences) + } } } catch { // do nothing, the collaborative doc does not sem to exist yet diff --git a/server-plugins/collaboration-resources/src/index.ts b/server-plugins/collaboration-resources/src/index.ts index bcf19cbfdd..18cc1c7ce0 100644 --- a/server-plugins/collaboration-resources/src/index.ts +++ b/server-plugins/collaboration-resources/src/index.ts @@ -13,9 +13,9 @@ // limitations under the License. // -import { removeCollaborativeDoc } from '@hcengineering/collaboration' import type { CollaborativeDoc, Doc, Tx, TxRemoveDoc } from '@hcengineering/core' -import core from '@hcengineering/core' +import core, { makeCollabId } from '@hcengineering/core' +import { removeCollabYdoc } from '@hcengineering/collaboration' import { type TriggerControl } from '@hcengineering/server-core' /** @@ -39,17 +39,16 @@ export async function OnDelete ( const attributes = hierarchy.getAllAttributes(rmTx.objectClass) for (const attribute of attributes.values()) { if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc)) { - const value = (doc as any)[attribute.name] as CollaborativeDoc - if (value !== undefined) { - toDelete.push(value) - } + toDelete.push(makeCollabId(doc._class, doc._id, attribute.name)) } } // TODO This is not accurate way to delete collaborative document // Even though we are deleting it here, the document can be currently in use by someone else // and when editing session ends, the collborator service will recreate the document again - await removeCollaborativeDoc(storageAdapter, workspace, toDelete, ctx) + if (toDelete.length > 0) { + await removeCollabYdoc(storageAdapter, workspace, toDelete, ctx) + } } return [] } diff --git a/server/collaboration/package.json b/server/collaboration/package.json index e2a6c20abf..b591ce67a3 100644 --- a/server/collaboration/package.json +++ b/server/collaboration/package.json @@ -40,6 +40,7 @@ "dependencies": { "@hcengineering/core": "^0.6.32", "@hcengineering/server-core": "^0.6.1", + "@hcengineering/text": "^0.6.5", "base64-js": "^1.5.1", "yjs": "^13.6.19" } diff --git a/server/collaboration/src/history/__tests__/branch.test.ts b/server/collaboration/src/history/__tests__/branch.test.ts deleted file mode 100644 index ee92d9bb1a..0000000000 --- a/server/collaboration/src/history/__tests__/branch.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Doc as YDoc, encodeStateAsUpdate, encodeStateVector } from 'yjs' - -import { yDocBranch, yDocBranchWithGC } from '../branch' -import { generateId } from '@hcengineering/core' - -describe('branch', () => { - describe('yDocBranch', () => { - it('branches document without gc', async () => { - const source = new YDoc({ guid: generateId(), gc: false }) - - applyGarbageCollectableChanges(source) - - const target = yDocBranch(source) - - expect(target.gc).toBeFalsy() - - // ensure target data - const sourceData = source.getArray('data') - const targetData = target.getArray('data') - expect(targetData.toArray()).toEqual(expect.arrayContaining(sourceData.toArray())) - - // ensure target state - const sourceState = encodeStateVector(source) - const targetState = encodeStateVector(target) - expect(targetState).toEqual(sourceState) - - // ensure target updates the same as source regardless of gc - const sourceUpdate = encodeStateAsUpdate(source) - const targetUpdate = encodeStateAsUpdate(target) - expect(targetUpdate).toEqual(sourceUpdate) - }) - - it('branches document state with gc', async () => { - const source = new YDoc({ guid: generateId(), gc: true }) - - applyGarbageCollectableChanges(source) - - const target = yDocBranch(source) - - expect(target.gc).toBeTruthy() - - // ensure target data - const sourceData = source.getArray('data') - const targetData = target.getArray('data') - expect(targetData.toArray()).toEqual(expect.arrayContaining(sourceData.toArray())) - - // ensure target state - const sourceState = encodeStateVector(source) - const targetState = encodeStateVector(target) - expect(targetState).toEqual(sourceState) - - // ensure target updates the same as source regardless of gc - const sourceUpdate = encodeStateAsUpdate(source) - const targetUpdate = encodeStateAsUpdate(target) - expect(targetUpdate).toEqual(sourceUpdate) - }) - }) - - describe('yDocBranchWithGC', () => { - it('branches document state without gc', async () => { - const source = new YDoc({ guid: generateId(), gc: false }) - - applyGarbageCollectableChanges(source) - - const target = yDocBranchWithGC(source) - - expect(target.gc).toBeFalsy() - - // ensure target data - const sourceData = source.getArray('data') - const targetData = target.getArray('data') - expect(targetData.toArray()).toEqual(expect.arrayContaining(sourceData.toArray())) - - // ensure target state - const sourceState = encodeStateVector(source) - const targetState = encodeStateVector(target) - expect(targetState).toEqual(sourceState) - - // ensure target updates different because source is not gc-ed - const sourceUpdate = encodeStateAsUpdate(source) - const targetUpdate = encodeStateAsUpdate(target) - expect(targetUpdate).not.toEqual(sourceUpdate) - }) - - it('branches document state with gc', async () => { - const source = new YDoc({ guid: generateId(), gc: true }) - - applyGarbageCollectableChanges(source) - - const target = yDocBranchWithGC(source) - - expect(target.gc).toBeTruthy() - - // ensure target data - const sourceData = source.getArray('data') - const targetData = target.getArray('data') - expect(targetData.toArray()).toEqual(expect.arrayContaining(sourceData.toArray())) - - // ensure target state - const sourceState = encodeStateVector(source) - const targetState = encodeStateVector(target) - expect(targetState).toEqual(sourceState) - - // ensure target updates the same because source is gc-ed - const sourceUpdate = encodeStateAsUpdate(source) - const targetUpdate = encodeStateAsUpdate(target) - expect(targetUpdate).toEqual(sourceUpdate) - }) - }) - - function applyGarbageCollectableChanges (ydoc: YDoc): void { - const sourceData = ydoc.getArray('data') - sourceData.insert(0, ['a']) - sourceData.insert(1, [1, 2]) - sourceData.delete(0, 1) - sourceData.insert(2, [3]) - } -}) diff --git a/server/collaboration/src/history/__tests__/history.test.ts b/server/collaboration/src/history/__tests__/history.test.ts deleted file mode 100644 index 957b048e78..0000000000 --- a/server/collaboration/src/history/__tests__/history.test.ts +++ /dev/null @@ -1,159 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Account, Ref, generateId } from '@hcengineering/core' -import { Doc as YDoc, encodeStateAsUpdate } from 'yjs' - -import { YDocVersion, addVersion, deleteVersion, getVersion, getVersionData, listVersions } from '../history' - -const HISTORY = 'history' -const UPDATES = 'updates' - -describe('history', () => { - let ydoc: YDoc - - beforeEach(() => { - ydoc = new YDoc({ guid: generateId() }) - }) - - it('addVersion should append new version', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - const update = encodeStateAsUpdate(ydoc) - - addVersion(ydoc, version, update) - - const history = ydoc.getArray(HISTORY) - const updates = ydoc.getMap(UPDATES) - - expect(history.length).toEqual(1) - expect(updates.size).toEqual(1) - - expect(history.get(0)).toEqual(version) - expect(updates.get(versionId)).toBeDefined() - }) - - it('addVersion should raise an error when a version already exists', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - const update = encodeStateAsUpdate(ydoc) - - addVersion(ydoc, version, update) - expect(() => { - addVersion(ydoc, version, update) - }).toThrow() - - const history = ydoc.getArray(HISTORY) - const updates = ydoc.getMap(UPDATES) - - expect(history.length).toEqual(1) - expect(updates.size).toEqual(1) - - expect(history.get(0)).toEqual(version) - expect(updates.get(versionId)).toBeDefined() - }) - - it('getVersion should get existing version data', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - const update = encodeStateAsUpdate(ydoc) - - addVersion(ydoc, yDocVersion(generateId()), encodeStateAsUpdate(ydoc)) - addVersion(ydoc, yDocVersion(generateId()), encodeStateAsUpdate(ydoc)) - addVersion(ydoc, version, update) - - const history = ydoc.getArray(HISTORY) - const updates = ydoc.getMap(UPDATES) - - expect(history.length).toEqual(3) - expect(updates.size).toEqual(3) - - expect(getVersion(ydoc, versionId)).toEqual(version) - }) - - it('getVersion should return undefined for unknown version', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - addVersion(ydoc, version, encodeStateAsUpdate(ydoc)) - - expect(getVersion(ydoc, generateId())).toBeUndefined() - }) - - it('listVersions should return existing versions', async () => { - const version1 = yDocVersion(generateId()) - const version2 = yDocVersion(generateId()) - - addVersion(ydoc, version1, encodeStateAsUpdate(ydoc)) - addVersion(ydoc, version2, encodeStateAsUpdate(ydoc)) - - expect(listVersions(ydoc)).toEqual(expect.arrayContaining([version1, version2])) - }) - - it('listVersions should return empty list when no versions', async () => { - expect(listVersions(ydoc)).toEqual([]) - }) - - it('getVersionData should get existing version data', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - const update = encodeStateAsUpdate(ydoc) - - addVersion(ydoc, version, update) - addVersion(ydoc, yDocVersion(generateId()), encodeStateAsUpdate(ydoc)) - addVersion(ydoc, yDocVersion(generateId()), encodeStateAsUpdate(ydoc)) - - const history = ydoc.getArray(HISTORY) - const updates = ydoc.getMap(UPDATES) - - expect(history.length).toEqual(3) - expect(updates.size).toEqual(3) - - expect(getVersionData(ydoc, versionId)).toEqual(update) - }) - - it('getVersionData should return undefined for unknown version', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - addVersion(ydoc, version, encodeStateAsUpdate(ydoc)) - - expect(getVersionData(ydoc, generateId())).toBeUndefined() - }) - - it('deleteVersion should delete existing version', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - addVersion(ydoc, version, encodeStateAsUpdate(ydoc)) - - deleteVersion(ydoc, versionId) - - const history = ydoc.getArray(HISTORY) - const updates = ydoc.getMap(UPDATES) - - expect(history.length).toEqual(0) - expect(updates.size).toEqual(0) - - expect(getVersion(ydoc, versionId)).toEqual(undefined) - expect(getVersionData(ydoc, versionId)).toEqual(undefined) - }) -}) - -function yDocVersion (versionId: string): YDocVersion { - return { - versionId, - name: versionId, - createdBy: 'unit test' as Ref, - createdOn: Date.now() - } -} diff --git a/server/collaboration/src/history/__tests__/snapshot.test.ts b/server/collaboration/src/history/__tests__/snapshot.test.ts deleted file mode 100644 index 6c36917352..0000000000 --- a/server/collaboration/src/history/__tests__/snapshot.test.ts +++ /dev/null @@ -1,109 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Account, Ref, generateId } from '@hcengineering/core' -import { Doc as YDoc } from 'yjs' - -import { YDocVersion } from '../history' -import { createYdocSnapshot, restoreYdocSnapshot } from '../snapshot' - -const HISTORY = 'history' -const UPDATES = 'updates' - -describe('snapshot', () => { - let yContent: YDoc - let yHistory: YDoc - - beforeEach(() => { - yContent = new YDoc({ guid: generateId(), gc: false }) - yHistory = new YDoc({ guid: generateId() }) - }) - - it('createYdocSnapshot appends new version', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - - createYdocSnapshot(yContent, yHistory, version) - - const history = yHistory.getArray(HISTORY) - const updates = yHistory.getMap(UPDATES) - - expect(history.length).toEqual(1) - expect(updates.size).toEqual(1) - - expect(history.get(0)).toEqual(version) - expect(updates.get(versionId)).toBeDefined() - }) - - it('restoreYdocSnapshot restores existing version', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - - const data = yContent.getArray('data') - data.insert(0, [1, 2, 3]) - expect(data.toArray()).toEqual(expect.arrayContaining([1, 2, 3])) - - createYdocSnapshot(yContent, yHistory, version) - - data.delete(1, 1) - expect(data.toArray()).toEqual(expect.arrayContaining([1, 3])) - - const yRestore = restoreYdocSnapshot(yContent, yHistory, versionId) - - // assert the restored doc has not been changed - expect(yRestore).toBeDefined() - expect(yRestore?.getArray('data').toArray()).toEqual(expect.arrayContaining([1, 2, 3])) - - // assert the original doc has not been changed - expect(yContent.getArray('data').toArray()).toEqual(expect.arrayContaining([1, 3])) - }) - - it('restoreYdocSnapshot throws an error when gc is enabled', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - - yContent = new YDoc({ guid: generateId(), gc: true }) - createYdocSnapshot(yContent, yHistory, version) - expect(() => restoreYdocSnapshot(yContent, yHistory, versionId)).toThrow() - }) - - it('restoreYdocSnapshot does not restore version that does not exist', async () => { - const versionId = generateId() - - const yRestore = restoreYdocSnapshot(yContent, yHistory, versionId) - expect(yRestore).toBeUndefined() - }) - - it('restoreYdocSnapshot restored document has gc enabled', async () => { - const versionId = generateId() - const version = yDocVersion(versionId) - - createYdocSnapshot(yContent, yHistory, version) - const yRestore = restoreYdocSnapshot(yContent, yHistory, versionId) - - // so far we don't care whether gc is enabled or not in the restore - // but we need to ensure we understand that it is enabled - expect(yRestore?.gc).toEqual(true) - }) -}) - -function yDocVersion (versionId: string): YDocVersion { - return { - versionId, - name: versionId, - createdBy: 'unit test' as Ref, - createdOn: Date.now() - } -} diff --git a/server/collaboration/src/history/branch.ts b/server/collaboration/src/history/branch.ts deleted file mode 100644 index 275cf4a31d..0000000000 --- a/server/collaboration/src/history/branch.ts +++ /dev/null @@ -1,54 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { generateId } from '@hcengineering/core' -import { Doc as YDoc, applyUpdate, encodeStateAsUpdate } from 'yjs' - -/** - * Branch (copy) document content as is. - * - * If the source document has gc parameter enabled, then garbage - * collection will be performed. The result document will have the same - * gc parameter value as the source document. - * - * @public - * */ -export function yDocBranch (source: YDoc): YDoc { - const target = new YDoc({ guid: generateId(), gc: source.gc }) - - const update = encodeStateAsUpdate(source) - applyUpdate(target, update) - - return target -} - -/** - * Branch (copy) document content with garbage collecting while applying update. - * - * Garbage collection will be performed regardless of the gc parameter - * in the source document. The result document will have the same gc - * parameter value as the source document. - * - * @public - * */ -export function yDocBranchWithGC (source: YDoc): YDoc { - const target = new YDoc({ guid: generateId(), gc: source.gc }) - - const gc = new YDoc({ guid: generateId(), gc: true }) - applyUpdate(gc, encodeStateAsUpdate(source)) - applyUpdate(target, encodeStateAsUpdate(gc)) - - return target -} diff --git a/server/collaboration/src/history/history.ts b/server/collaboration/src/history/history.ts deleted file mode 100644 index 7feaebb3ed..0000000000 --- a/server/collaboration/src/history/history.ts +++ /dev/null @@ -1,110 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Account, Ref, Timestamp } from '@hcengineering/core' -import { fromByteArray, toByteArray } from 'base64-js' -import { Array as YArray, Doc as YDoc, Map as YMap } from 'yjs' - -/** - * This module provides utils for document version storage based on YDoc - * - * At the top level the history document contains two fields: - * 1. history - * An array containing version ids in creation order - * 2. updates - * A map containing version data keyed by version id - * - * { - * "history": [ - * { "versionId": "version1", ... }, - * { "versionId": "version2", ... }, - * ... - * ], - * "updates": { - * "version1": ... version data as ydoc update base64 encoded ..., - * "version2": ... version data as ydoc update base64 encoded ..., - * ... - * } - * } - */ - -/** @public */ -export interface YDocVersion { - versionId: string - name: string - - createdBy: Ref - createdOn: Timestamp -} - -const HISTORY = 'history' -const UPDATES = 'updates' - -function getHistory (ydoc: YDoc): YArray { - return ydoc.getArray(HISTORY) -} - -function getUpdates (ydoc: YDoc): YMap { - return ydoc.getMap(UPDATES) -} - -/** @public */ -export function addVersion (ydoc: YDoc, version: YDocVersion, update: Uint8Array): void { - const history = getHistory(ydoc) - const updates = getUpdates(ydoc) - - const { versionId } = version - - if (updates.has(versionId)) { - throw Error('history item already exists') - } - - ydoc.transact((tr) => { - history.push([version]) - updates.set(versionId, fromByteArray(update)) - }) -} - -/** @public */ -export function getVersion (ydoc: YDoc, versionId: string): YDocVersion | undefined { - const history = getHistory(ydoc) - return history.toArray().find((p) => p.versionId === versionId) -} - -/** @public */ -export function listVersions (ydoc: YDoc): YDocVersion[] { - return getHistory(ydoc).toArray() -} - -/** @public */ -export function getVersionData (ydoc: YDoc, versionId: string): Uint8Array | undefined { - const updates = getUpdates(ydoc) - const update = updates.get(versionId) - return update !== undefined ? toByteArray(update) : undefined -} - -/** @public */ -export function deleteVersion (ydoc: YDoc, versionId: string): void { - const history = getHistory(ydoc) - const updates = getUpdates(ydoc) - - ydoc.transact((tr) => { - const index = history.toArray().findIndex((p) => p.versionId === versionId) - if (index !== -1) { - history.delete(index, 1) - } - updates.delete(versionId) - }) -} diff --git a/server/collaboration/src/history/snapshot.ts b/server/collaboration/src/history/snapshot.ts deleted file mode 100644 index ac0d273af3..0000000000 --- a/server/collaboration/src/history/snapshot.ts +++ /dev/null @@ -1,36 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Doc as YDoc } from 'yjs' -import * as Y from 'yjs' - -import { YDocVersion, addVersion, getVersionData } from './history' - -/** @public */ -export function createYdocSnapshot (yContent: YDoc, yHistory: YDoc, version: YDocVersion): void { - const snapshot = Y.snapshot(yContent) - const update = Y.encodeSnapshot(snapshot) - - addVersion(yHistory, version, update) -} - -/** @public */ -export function restoreYdocSnapshot (yContent: YDoc, yHistory: YDoc, versionId: string): YDoc | undefined { - const update = getVersionData(yHistory, versionId) - if (update !== undefined) { - const snapshot = Y.decodeSnapshot(update) - return Y.createDocFromSnapshot(yContent, snapshot) - } -} diff --git a/server/collaboration/src/index.ts b/server/collaboration/src/index.ts index 223512db04..88cac91502 100644 --- a/server/collaboration/src/index.ts +++ b/server/collaboration/src/index.ts @@ -13,9 +13,5 @@ // limitations under the License. // -export * from './history/branch' -export * from './history/history' -export * from './history/snapshot' -export * from './utils/collaborative-doc' -export * from './utils/storage' -export * from './utils/ydoc' +export * from './storage' +export * from './ydoc' diff --git a/server/collaboration/src/storage.ts b/server/collaboration/src/storage.ts new file mode 100644 index 0000000000..2d195ef3c8 --- /dev/null +++ b/server/collaboration/src/storage.ts @@ -0,0 +1,127 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type Blob, + type CollaborativeDoc, + type Ref, + type WorkspaceId, + Markup, + MeasureContext, + generateId, + makeCollabJsonId, + makeCollabYdocId +} from '@hcengineering/core' +import { StorageAdapter } from '@hcengineering/server-core' +import { yDocToMarkup } from '@hcengineering/text' +import { Doc as YDoc } from 'yjs' + +import { yDocFromBuffer, yDocToBuffer } from './ydoc' + +/** @public */ +export async function loadCollabYdoc ( + ctx: MeasureContext, + storageAdapter: StorageAdapter, + workspace: WorkspaceId, + doc: CollaborativeDoc +): Promise { + const blobId = makeCollabYdocId(doc) + + const blob = await storageAdapter.stat(ctx, workspace, blobId) + if (blob === undefined) { + return undefined + } + + if (!blob.contentType.includes('application/ydoc')) { + ctx.error('invalid content type', { contentType: blob.contentType }) + return undefined + } + + // no need to apply gc because we load existing document + // it is either already gc-ed, or gc not needed and it is disabled + const ydoc = new YDoc({ guid: generateId(), gc: false }) + + const buffer = await storageAdapter.read(ctx, workspace, blobId) + return yDocFromBuffer(Buffer.concat(buffer as any), ydoc) +} + +/** @public */ +export async function saveCollabYdoc ( + ctx: MeasureContext, + storageAdapter: StorageAdapter, + workspace: WorkspaceId, + doc: CollaborativeDoc, + ydoc: YDoc +): Promise> { + const blobId = makeCollabYdocId(doc) + + const buffer = yDocToBuffer(ydoc) + await storageAdapter.put(ctx, workspace, blobId, buffer, 'application/ydoc', buffer.length) + + return blobId +} + +/** @public */ +export async function removeCollabYdoc ( + storageAdapter: StorageAdapter, + workspace: WorkspaceId, + collaborativeDocs: CollaborativeDoc[], + ctx: MeasureContext +): Promise { + const toRemove: string[] = collaborativeDocs.map(makeCollabYdocId) + if (toRemove.length > 0) { + await ctx.with('remove', {}, async () => { + await storageAdapter.remove(ctx, workspace, toRemove) + }) + } +} + +/** @public */ +export async function loadCollabJson ( + ctx: MeasureContext, + storageAdapter: StorageAdapter, + workspace: WorkspaceId, + blobId: Ref +): Promise { + const blob = await storageAdapter.stat(ctx, workspace, blobId) + if (blob === undefined) { + return undefined + } + + if (!blob.contentType.includes('application/json')) { + ctx.error('invalid content type', { contentType: blob.contentType }) + return undefined + } + + const buffer = await storageAdapter.read(ctx, workspace, blobId) + return Buffer.concat(buffer as any).toString() +} + +/** @public */ +export async function saveCollabJson ( + ctx: MeasureContext, + storageAdapter: StorageAdapter, + workspace: WorkspaceId, + doc: CollaborativeDoc, + content: Markup | YDoc +): Promise> { + const blobId = makeCollabJsonId(doc) + + const markup = typeof content === 'string' ? content : yDocToMarkup(content, doc.objectAttr) + const buffer = Buffer.from(markup) + await storageAdapter.put(ctx, workspace, blobId, buffer, 'application/json', buffer.length) + + return blobId +} diff --git a/server/collaboration/src/utils/__tests__/collaborative-doc.test.ts b/server/collaboration/src/utils/__tests__/collaborative-doc.test.ts deleted file mode 100644 index 87e8e542c7..0000000000 --- a/server/collaboration/src/utils/__tests__/collaborative-doc.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { collaborativeDocFormat } from '@hcengineering/core' -import { collaborativeHistoryDocId, isEditableDoc, isEditableDocVersion } from '../collaborative-doc' - -describe('collaborative-doc', () => { - describe('collaborativeHistoryDocId', () => { - it('returns valid history doc id', async () => { - expect(collaborativeHistoryDocId('documentId')).toEqual('documentId#history') - }) - - it('returns valid history doc id for history doc id', async () => { - expect(collaborativeHistoryDocId('documentId#history')).toEqual('documentId#history') - }) - }) - - describe('isEditableDoc', () => { - it('returns true for HEAD version', async () => { - const doc = collaborativeDocFormat({ - documentId: 'example', - versionId: 'HEAD', - lastVersionId: '0' - }) - expect(isEditableDoc(doc)).toBeTruthy() - }) - - it('returns false for other versions', async () => { - const doc = collaborativeDocFormat({ - documentId: 'example', - versionId: 'main', - lastVersionId: '0' - }) - expect(isEditableDoc(doc)).toBeFalsy() - }) - }) - - describe('isEditableDocVersion', () => { - it('returns true for HEAD version', async () => { - expect(isEditableDocVersion('HEAD')).toBeTruthy() - }) - - it('returns false for other versions', async () => { - expect(isEditableDocVersion('')).toBeFalsy() - expect(isEditableDocVersion('main')).toBeFalsy() - expect(isEditableDocVersion('head')).toBeFalsy() - }) - }) -}) diff --git a/server/collaboration/src/utils/__tests__/ydoc.test.ts b/server/collaboration/src/utils/__tests__/ydoc.test.ts deleted file mode 100644 index 6dc0acaeb4..0000000000 --- a/server/collaboration/src/utils/__tests__/ydoc.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Doc as YDoc, XmlElement as YXmlElement, XmlText as YXmlText, encodeStateVector } from 'yjs' - -import { clone, yDocCopyXmlField, yDocFromBuffer, yDocToBuffer } from '../ydoc' -import { generateId } from '@hcengineering/core' - -describe('ydoc', () => { - it('yDocFromBuffer converts ydoc to a buffer', async () => { - const ydoc = new YDoc({ guid: generateId() }) - const buffer = yDocToBuffer(ydoc) - - expect(buffer).toBeDefined() - }) - - it('yDocFromBuffer converts buffer to a ydoc', async () => { - const source = new YDoc({ guid: generateId() }) - source.getArray('data').insert(0, [1, 2]) - - const buffer = yDocToBuffer(source) - - const target = yDocFromBuffer(buffer, new YDoc({ guid: generateId() })) - expect(target).toBeDefined() - expect(encodeStateVector(target)).toEqual(encodeStateVector(source)) - }) - - describe('yDocCopyXmlField', () => { - it('copies into new field', async () => { - const ydoc = new YDoc() - - const source = ydoc.getXmlFragment('source') - source.insertAfter(null, [new YXmlElement('p'), new YXmlText('foo'), new YXmlElement('p')]) - expect(ydoc.share.has('target')).toBeFalsy() - - yDocCopyXmlField(ydoc, 'source', 'target') - const target = ydoc.getXmlFragment('target') - - expect(ydoc.share.has('target')).toBeTruthy() - expect(target.toJSON()).toEqual(source.toJSON()) - }) - - it('copies into existing field', async () => { - const ydoc = new YDoc() - - const source = ydoc.getXmlFragment('source') - const target = ydoc.getXmlFragment('target') - - source.insertAfter(null, [new YXmlElement('p'), new YXmlText('foo'), new YXmlElement('p')]) - target.insertAfter(null, [new YXmlText('bar')]) - expect(target.toJSON()).not.toEqual(source.toJSON()) - - yDocCopyXmlField(ydoc, 'source', 'target') - expect(ydoc.share.has('target')).toBeTruthy() - expect(target.toJSON()).toEqual(source.toJSON()) - }) - }) - - it('clones YXmlElement', () => { - const ydoc = new YDoc() - const source = ydoc.getXmlElement('source') - const target = ydoc.getXmlFragment('target') - - const src = new YXmlElement('paragraph') - src.setAttribute('class', 'text') - src.setAttribute('size', 1024 as any) - src.insert(0, [new YXmlText('foo')]) - source.insert(0, [src]) - - const dst = clone(src) - target.insert(0, [dst]) - - expect(src.toJSON()).toEqual(dst.toJSON()) - }) -}) diff --git a/server/collaboration/src/utils/collaborative-doc.ts b/server/collaboration/src/utils/collaborative-doc.ts deleted file mode 100644 index f6d6f9cd29..0000000000 --- a/server/collaboration/src/utils/collaborative-doc.ts +++ /dev/null @@ -1,168 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { - CollaborativeDoc, - CollaborativeDocVersion, - CollaborativeDocVersionHead, - MeasureContext, - WorkspaceId, - collaborativeDocParse, - collaborativeDocUnchain, - generateId -} from '@hcengineering/core' -import { Doc as YDoc } from 'yjs' - -import { StorageAdapter } from '@hcengineering/server-core' -import { restoreYdocSnapshot } from '../history/snapshot' -import { yDocFromStorage, yDocToStorage } from './storage' - -/** @public */ -export function collaborativeHistoryDocId (id: string): string { - const suffix = '#history' - return id.endsWith(suffix) ? id : id + suffix -} - -async function loadCollaborativeDocVersion ( - ctx: MeasureContext, - storageAdapter: StorageAdapter, - workspace: WorkspaceId, - documentId: string, - versionId: string -): Promise { - const yContent = await ctx.with('yDocFromStorage', { type: 'content' }, (ctx) => { - return yDocFromStorage(ctx, storageAdapter, workspace, documentId, new YDoc({ guid: generateId(), gc: false })) - }) - - // the document does not exist - if (yContent === undefined) { - return undefined - } - - if (versionId === 'HEAD') { - return yContent - } - - const historyDocumentId = collaborativeHistoryDocId(documentId) - const yHistory = await ctx.with('yDocFromStorage', { type: 'history' }, (ctx) => { - return yDocFromStorage(ctx, storageAdapter, workspace, historyDocumentId, new YDoc({ guid: generateId() })) - }) - - // the history document does not exist - if (yHistory === undefined) { - return undefined - } - - return await ctx.with('restoreYdocSnapshot', {}, () => { - return restoreYdocSnapshot(yContent, yHistory, versionId) - }) -} - -/** @public */ -export async function loadCollaborativeDoc ( - ctx: MeasureContext, - storageAdapter: StorageAdapter, - workspace: WorkspaceId, - collaborativeDoc: CollaborativeDoc -): Promise { - const sources = collaborativeDocUnchain(collaborativeDoc) - - return await ctx.with('loadCollaborativeDoc', { type: 'content' }, async (ctx) => { - for (const source of sources) { - const { documentId, versionId } = collaborativeDocParse(source) - - const ydoc = await loadCollaborativeDocVersion(ctx, storageAdapter, workspace, documentId, versionId) - - if (ydoc !== undefined) { - return ydoc - } - } - return undefined - }) -} - -/** @public */ -export async function saveCollaborativeDoc ( - ctx: MeasureContext, - storageAdapter: StorageAdapter, - workspace: WorkspaceId, - collaborativeDoc: CollaborativeDoc, - ydoc: YDoc -): Promise { - const { documentId, versionId } = collaborativeDocParse(collaborativeDoc) - await saveCollaborativeDocVersion(ctx, storageAdapter, workspace, documentId, versionId, ydoc) -} - -/** @public */ -export async function saveCollaborativeDocVersion ( - ctx: MeasureContext, - storageAdapter: StorageAdapter, - workspace: WorkspaceId, - documentId: string, - versionId: CollaborativeDocVersion, - ydoc: YDoc -): Promise { - await ctx.with('saveCollaborativeDoc', {}, async (ctx) => { - if (versionId === 'HEAD') { - await ctx.with('yDocToStorage', {}, () => yDocToStorage(ctx, storageAdapter, workspace, documentId, ydoc)) - } else { - console.warn('Cannot save non HEAD document version', documentId, versionId) - } - }) -} - -/** @public */ -export async function removeCollaborativeDoc ( - storageAdapter: StorageAdapter, - workspace: WorkspaceId, - collaborativeDocs: CollaborativeDoc[], - ctx: MeasureContext -): Promise { - await ctx.with('removeollaborativeDoc', {}, async (ctx) => { - const toRemove: string[] = [] - for (const collaborativeDoc of collaborativeDocs) { - const { documentId, versionId } = collaborativeDocParse(collaborativeDoc) - if (versionId === CollaborativeDocVersionHead) { - toRemove.push(documentId, collaborativeHistoryDocId(documentId)) - } else { - console.warn('Cannot remove non HEAD document version', documentId, versionId) - } - } - if (toRemove.length > 0) { - await ctx.with('remove', {}, () => storageAdapter.remove(ctx, workspace, toRemove)) - } - }) -} - -/** @public */ -export function isEditableDoc (id: CollaborativeDoc): boolean { - const { versionId } = collaborativeDocParse(id) - return isEditableDocVersion(versionId) -} - -/** @public */ -export function isReadonlyDoc (id: CollaborativeDoc): boolean { - return !isEditableDoc(id) -} - -/** @public */ -export function isEditableDocVersion (version: CollaborativeDocVersion): boolean { - return version === CollaborativeDocVersionHead -} - -/** @public */ -export function isReadonlyDocVersion (version: CollaborativeDocVersion): boolean { - return !isEditableDocVersion(version) -} diff --git a/server/collaboration/src/utils/storage.ts b/server/collaboration/src/utils/storage.ts deleted file mode 100644 index be2cb7976d..0000000000 --- a/server/collaboration/src/utils/storage.ts +++ /dev/null @@ -1,57 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { generateId, MeasureContext, WorkspaceId } from '@hcengineering/core' -import { StorageAdapter } from '@hcengineering/server-core' -import { Doc as YDoc } from 'yjs' - -import { yDocFromBuffer, yDocToBuffer } from './ydoc' - -/** @public */ -export async function yDocFromStorage ( - ctx: MeasureContext, - storageAdapter: StorageAdapter, - workspace: WorkspaceId, - documentId: string, - ydoc?: YDoc -): Promise { - // stat the object to ensure it exists, because read will throw an error in this case - try { - const buffer = await storageAdapter.read(ctx, workspace, documentId) - - // no need to apply gc because we load existing document - // it is either already gc-ed, or gc not needed and it is disabled - ydoc ??= new YDoc({ guid: generateId(), gc: false }) - - return yDocFromBuffer(Buffer.concat(buffer as any), ydoc) - } catch (err: any) { - if (err.code === 'NoSuchKey') { - return undefined - } - throw err - } -} - -/** @public */ -export async function yDocToStorage ( - ctx: MeasureContext, - storageAdapter: StorageAdapter, - workspace: WorkspaceId, - documentId: string, - ydoc: YDoc -): Promise { - const buffer = yDocToBuffer(ydoc) - await storageAdapter.put(ctx, workspace, documentId, buffer, 'application/ydoc', buffer.length) -} diff --git a/server/collaboration/src/utils/ydoc.ts b/server/collaboration/src/ydoc.ts similarity index 88% rename from server/collaboration/src/utils/ydoc.ts rename to server/collaboration/src/ydoc.ts index c6824fb4fb..b20cd907e8 100644 --- a/server/collaboration/src/utils/ydoc.ts +++ b/server/collaboration/src/ydoc.ts @@ -13,6 +13,7 @@ // limitations under the License. // +import { generateId } from '@hcengineering/core' import { AbstractType as YAbstractType, Doc as YDoc, @@ -24,7 +25,8 @@ import { export { XmlElement as YXmlElement, XmlText as YXmlText, AbstractType as YAbstractType } from 'yjs' /** @public */ -export function yDocFromBuffer (buffer: Buffer, ydoc: YDoc): YDoc { +export function yDocFromBuffer (buffer: Buffer, ydoc?: YDoc): YDoc { + ydoc ??= new YDoc({ guid: generateId(), gc: false }) try { const uint8arr = new Uint8Array(buffer) applyUpdate(ydoc, uint8arr) @@ -59,7 +61,7 @@ export function yDocCopyXmlField (ydoc: YDoc, source: string, target: string): v * @param src YXmlElement * @returns YXmlElement */ -export function clone (src: YXmlElement): YXmlElement { +export function yXmlElementClone (src: YXmlElement): YXmlElement { const el = new YXmlElement(src.nodeName) const attrs = src.getAttributes() @@ -72,7 +74,7 @@ export function clone (src: YXmlElement): YXmlElement { src .toArray() .map((item) => - item instanceof YAbstractType ? (item instanceof YXmlElement ? clone(item) : item.clone()) : item + item instanceof YAbstractType ? (item instanceof YXmlElement ? yXmlElementClone(item) : item.clone()) : item ) as any ) diff --git a/server/collaborator/src/context.ts b/server/collaborator/src/context.ts index b0de5441ef..c1775f9ba4 100644 --- a/server/collaborator/src/context.ts +++ b/server/collaborator/src/context.ts @@ -13,8 +13,7 @@ // limitations under the License. // -import { type DocumentId, type PlatformDocumentId } from '@hcengineering/collaborator-client' -import { WorkspaceId, generateId } from '@hcengineering/core' +import { Blob, Ref, WorkspaceId, generateId } from '@hcengineering/core' import { decodeToken } from '@hcengineering/server-token' import { onAuthenticatePayload } from '@hocuspocus/server' import { ClientFactory, simpleClientFactory } from './platform' @@ -24,8 +23,7 @@ export interface Context { workspaceId: WorkspaceId clientFactory: ClientFactory - initialContentId?: DocumentId - platformDocumentId?: PlatformDocumentId + content?: Ref } interface WithContext { @@ -42,14 +40,12 @@ export function buildContext (data: onAuthenticatePayload): Context { const connectionId = context.connectionId ?? generateId() const decodedToken = decodeToken(data.token) - const initialContentId = (data.requestParameters.get('initialContentId') as DocumentId) ?? undefined - const platformDocumentId = (data.requestParameters.get('platformDocumentId') as PlatformDocumentId) ?? undefined + const content = (data.requestParameters.get('content') as Ref) ?? undefined return { connectionId, workspaceId: decodedToken.workspace, clientFactory: simpleClientFactory(decodedToken), - initialContentId, - platformDocumentId + content } } diff --git a/server/collaborator/src/extensions/authentication.ts b/server/collaborator/src/extensions/authentication.ts index 2d7081133b..757b0d4422 100644 --- a/server/collaborator/src/extensions/authentication.ts +++ b/server/collaborator/src/extensions/authentication.ts @@ -13,8 +13,7 @@ // limitations under the License. // -import { DocumentId, parseDocumentId } from '@hcengineering/collaborator-client' -import { isReadonlyDoc } from '@hcengineering/collaboration' +import { decodeDocumentId } from '@hcengineering/collaborator-client' import { MeasureContext } from '@hcengineering/core' import { decodeToken } from '@hcengineering/server-token' import { Extension, onAuthenticatePayload } from '@hocuspocus/server' @@ -35,7 +34,7 @@ export class AuthenticationExtension implements Extension { async onAuthenticate (data: onAuthenticatePayload): Promise { const ctx = this.configuration.ctx - const { workspaceId, collaborativeDoc } = parseDocumentId(data.documentName as DocumentId) + const { workspaceId } = decodeDocumentId(data.documentName) return await ctx.with('authenticate', { workspaceId }, async () => { const token = decodeToken(data.token) @@ -50,8 +49,6 @@ export class AuthenticationExtension implements Extension { throw new Error('documentName must include workspace id') } - data.connection.readOnly = isReadonlyDoc(collaborativeDoc) - return buildContext(data) }) } diff --git a/server/collaborator/src/extensions/storage.ts b/server/collaborator/src/extensions/storage.ts index db49e51148..279e4568b3 100644 --- a/server/collaborator/src/extensions/storage.ts +++ b/server/collaborator/src/extensions/storage.ts @@ -13,7 +13,6 @@ // limitations under the License. // -import { DocumentId } from '@hcengineering/collaborator-client' import { type Markup, MeasureContext } from '@hcengineering/core' import { Document, @@ -130,7 +129,7 @@ export class StorageExtension implements Extension { try { return await ctx.with('load-document', {}, (ctx) => { - return adapter.loadDocument(ctx, documentName as DocumentId, context) + return adapter.loadDocument(ctx, documentName, context) }) } catch (err) { ctx.error('failed to load document', { documentName, error: err }) @@ -146,7 +145,7 @@ export class StorageExtension implements Extension { const currMarkup = this.configuration.transformer.fromYdoc(document) await ctx.with('save-document', {}, (ctx) => - adapter.saveDocument(ctx, documentName as DocumentId, document, context, { + adapter.saveDocument(ctx, documentName, document, context, { prev: prevMarkup, curr: currMarkup }) diff --git a/server/collaborator/src/rpc/methods/createContent.ts b/server/collaborator/src/rpc/methods/createContent.ts new file mode 100644 index 0000000000..97ad6e4ce7 --- /dev/null +++ b/server/collaborator/src/rpc/methods/createContent.ts @@ -0,0 +1,49 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + type CreateContentRequest, + type CreateContentResponse, + decodeDocumentId +} from '@hcengineering/collaborator-client' +import { saveCollabJson } from '@hcengineering/collaboration' +import { type Blob, type Ref, MeasureContext } from '@hcengineering/core' +import { Context } from '../../context' +import { RpcMethodParams } from '../rpc' + +export async function createContent ( + ctx: MeasureContext, + context: Context, + documentName: string, + payload: CreateContentRequest, + params: RpcMethodParams +): Promise { + const { content } = payload + const { hocuspocus, storageAdapter } = params + + if (hocuspocus.documents.has(documentName) || hocuspocus.loadingDocuments.has(documentName)) { + throw new Error(`Document ${documentName} already exists`) + } + + const { documentId, workspaceId } = decodeDocumentId(documentName) + + const result: Record> = {} + for (const [field, markup] of Object.entries(content)) { + const blob = await saveCollabJson(ctx, storageAdapter, { name: workspaceId }, documentId, markup) + result[field] = blob + } + + return { content: result } +} diff --git a/server/collaborator/src/rpc/methods/getContent.ts b/server/collaborator/src/rpc/methods/getContent.ts index 90aeb92cdf..8899141834 100644 --- a/server/collaborator/src/rpc/methods/getContent.ts +++ b/server/collaborator/src/rpc/methods/getContent.ts @@ -21,14 +21,17 @@ import { RpcMethodParams } from '../rpc' export async function getContent ( ctx: MeasureContext, context: Context, - documentId: string, + documentName: string, payload: GetContentRequest, params: RpcMethodParams ): Promise { const { hocuspocus, transformer } = params + const { source } = payload + + context = { ...context, content: source } const connection = await ctx.with('connect', {}, () => { - return hocuspocus.openDirectConnection(documentId, context) + return hocuspocus.openDirectConnection(documentName, context) }) try { diff --git a/server/collaborator/src/rpc/methods/index.ts b/server/collaborator/src/rpc/methods/index.ts index 6c69a0a2f3..2a01f16280 100644 --- a/server/collaborator/src/rpc/methods/index.ts +++ b/server/collaborator/src/rpc/methods/index.ts @@ -14,10 +14,12 @@ // import { getContent } from './getContent' +import { createContent } from './createContent' import { updateContent } from './updateContent' import { RpcMethod } from '../rpc' export const methods: Record = { getContent, + createContent, updateContent } diff --git a/server/collaborator/src/rpc/methods/updateContent.ts b/server/collaborator/src/rpc/methods/updateContent.ts index 1334f0265c..87dc350e78 100644 --- a/server/collaborator/src/rpc/methods/updateContent.ts +++ b/server/collaborator/src/rpc/methods/updateContent.ts @@ -22,7 +22,7 @@ import { RpcMethodParams } from '../rpc' export async function updateContent ( ctx: MeasureContext, context: Context, - documentId: string, + documentName: string, payload: UpdateContentRequest, params: RpcMethodParams ): Promise { @@ -41,7 +41,7 @@ export async function updateContent ( }) const connection = await ctx.with('connect', {}, () => { - return hocuspocus.openDirectConnection(documentId, context) + return hocuspocus.openDirectConnection(documentName, context) }) try { diff --git a/server/collaborator/src/server.ts b/server/collaborator/src/server.ts index 8e7422ebc0..e3afeefbc2 100644 --- a/server/collaborator/src/server.ts +++ b/server/collaborator/src/server.ts @@ -136,24 +136,24 @@ export async function start (ctx: MeasureContext, config: Config, storageAdapter }) // eslint-disable-next-line @typescript-eslint/no-misused-promises - app.post('/rpc', async (req, res) => { + app.post('/rpc/:id', async (req, res) => { const authHeader = req.headers.authorization if (authHeader === undefined) { res.status(403).send({ error: 'Unauthorized' }) return } - const request = req.body as RpcRequest - - const documentId = request.documentId + const documentId = req.params.id if (documentId === undefined || documentId === '') { const response: RpcErrorResponse = { - error: 'Missing documentId' + error: 'Missing document id' } res.status(400).send(response) return } + const request = req.body as RpcRequest + const method = methods[request.method] if (method === undefined) { const response: RpcErrorResponse = { diff --git a/server/collaborator/src/storage/adapter.ts b/server/collaborator/src/storage/adapter.ts index 04573e7296..8b5fb44a7d 100644 --- a/server/collaborator/src/storage/adapter.ts +++ b/server/collaborator/src/storage/adapter.ts @@ -13,16 +13,15 @@ // limitations under the License. // -import { DocumentId } from '@hcengineering/collaborator-client' import { MeasureContext } from '@hcengineering/core' import { Doc as YDoc } from 'yjs' import { Context } from '../context' export interface CollabStorageAdapter { - loadDocument: (ctx: MeasureContext, documentId: DocumentId, context: Context) => Promise + loadDocument: (ctx: MeasureContext, documentId: string, context: Context) => Promise saveDocument: ( ctx: MeasureContext, - documentId: DocumentId, + documentId: string, document: YDoc, context: Context, markup: { diff --git a/server/collaborator/src/storage/platform.ts b/server/collaborator/src/storage/platform.ts index 3179ae5e5d..d9d7884479 100644 --- a/server/collaborator/src/storage/platform.ts +++ b/server/collaborator/src/storage/platform.ts @@ -14,61 +14,63 @@ // import activity, { DocUpdateMessage } from '@hcengineering/activity' -import { loadCollaborativeDoc, saveCollaborativeDoc } from '@hcengineering/collaboration' -import { - DocumentId, - PlatformDocumentId, - parseDocumentId, - parsePlatformDocumentId -} from '@hcengineering/collaborator-client' -import core, { - AttachedData, - CollaborativeDoc, - MeasureContext, - TxOperations, - collaborativeDocWithLastVersion -} from '@hcengineering/core' +import { loadCollabJson, loadCollabYdoc, saveCollabJson, saveCollabYdoc } from '@hcengineering/collaboration' +import { decodeDocumentId } from '@hcengineering/collaborator-client' +import core, { AttachedData, MeasureContext, TxOperations } from '@hcengineering/core' import { StorageAdapter } from '@hcengineering/server-core' +import { markupToYDocNoSchema, areEqualMarkups } from '@hcengineering/text' import { Doc as YDoc } from 'yjs' import { Context } from '../context' -import { areEqualMarkups } from '@hcengineering/text' import { CollabStorageAdapter } from './adapter' export class PlatformStorageAdapter implements CollabStorageAdapter { constructor (private readonly storage: StorageAdapter) {} - async loadDocument (ctx: MeasureContext, documentId: DocumentId, context: Context): Promise { + async loadDocument (ctx: MeasureContext, documentName: string, context: Context): Promise { + const { content, workspaceId } = context + const { documentId } = decodeDocumentId(documentName) + // try to load document content try { - ctx.info('load document content', { documentId }) - const ydoc = await this.loadDocumentFromStorage(ctx, documentId, context) + ctx.info('load document content', { documentName }) + + const ydoc = await ctx.with('loadCollabYdoc', {}, (ctx) => { + return withRetry(ctx, 5, () => { + return loadCollabYdoc(ctx, this.storage, context.workspaceId, documentId) + }) + }) if (ydoc !== undefined) { return ydoc } } catch (err) { - ctx.error('failed to load document content', { documentId, error: err }) + ctx.error('failed to load document content', { documentName, error: err }) throw err } // then try to load from inital content - const { initialContentId } = context - if (initialContentId !== undefined && initialContentId.length > 0) { + if (content !== undefined) { try { - ctx.info('load document initial content', { documentId, initialContentId }) - const ydoc = await this.loadDocumentFromStorage(ctx, initialContentId, context) + ctx.info('load document initial content', { documentName, content }) + + const markup = await ctx.with('loadCollabJson', {}, (ctx) => { + return withRetry(ctx, 5, () => { + return loadCollabJson(ctx, this.storage, workspaceId, content) + }) + }) + if (markup !== undefined) { + const ydoc = markupToYDocNoSchema(markup, documentId.objectAttr) + + // if document was loaded from the initial content or storage we need to save + // it to ensure the next time we load it from the ydoc document + await saveCollabYdoc(ctx, this.storage, workspaceId, documentId, ydoc) - // if document was loaded from the initial content or storage we need to save - // it to ensure the next time we load it from the ydoc document - if (ydoc !== undefined) { - ctx.info('save document content', { documentId, initialContentId }) - await this.saveDocumentToStorage(ctx, documentId, ydoc, context) return ydoc } } catch (err) { - ctx.error('failed to load initial document content', { documentId, initialContentId, error: err }) + ctx.error('failed to load initial document content', { documentName, content, error: err }) throw err } } @@ -79,7 +81,7 @@ export class PlatformStorageAdapter implements CollabStorageAdapter { async saveDocument ( ctx: MeasureContext, - documentId: DocumentId, + documentName: string, document: YDoc, context: Context, markup: { @@ -88,72 +90,45 @@ export class PlatformStorageAdapter implements CollabStorageAdapter { } ): Promise { const { clientFactory } = context + const { documentId } = decodeDocumentId(documentName) const client = await ctx.with('connect', {}, () => clientFactory()) try { try { - ctx.info('save document content', { documentId }) - await this.saveDocumentToStorage(ctx, documentId, document, context) + ctx.info('save document ydoc content', { documentName }) + await ctx.with('saveCollabYdoc', {}, (ctx) => { + return withRetry(ctx, 5, () => { + return saveCollabYdoc(ctx, this.storage, context.workspaceId, documentId, document) + }) + }) } catch (err) { - ctx.error('failed to save document', { documentId, error: err }) + ctx.error('failed to save document ydoc content', { documentName, error: err }) // raise an error if failed to save document to storage // this will prevent document from being unloaded from memory throw err } - const { platformDocumentId } = context - if (platformDocumentId !== undefined) { - ctx.info('save document content to platform', { documentId, platformDocumentId }) - await ctx.with('save-to-platform', {}, (ctx) => - this.saveDocumentToPlatform(ctx, client, documentId, platformDocumentId, markup) - ) - } + ctx.info('save document content to platform', { documentName }) + await ctx.with('save-to-platform', {}, (ctx) => { + return this.saveDocumentToPlatform(ctx, client, documentName, markup) + }) } finally { await client.close() } } - async loadDocumentFromStorage ( - ctx: MeasureContext, - documentId: DocumentId, - context: Context - ): Promise { - const { collaborativeDoc } = parseDocumentId(documentId) - - return await ctx.with('load-document', {}, (ctx) => - withRetry(ctx, 5, async () => { - return await loadCollaborativeDoc(ctx, this.storage, context.workspaceId, collaborativeDoc) - }) - ) - } - - async saveDocumentToStorage ( - ctx: MeasureContext, - documentId: DocumentId, - document: YDoc, - context: Context - ): Promise { - const { collaborativeDoc } = parseDocumentId(documentId) - - await ctx.with('save-document', {}, (ctx) => - withRetry(ctx, 5, async () => { - await saveCollaborativeDoc(ctx, this.storage, context.workspaceId, collaborativeDoc, document) - }) - ) - } - async saveDocumentToPlatform ( ctx: MeasureContext, client: Omit, documentName: string, - platformDocumentId: PlatformDocumentId, markup: { prev: Record curr: Record } ): Promise { - const { objectClass, objectId, objectAttr } = parsePlatformDocumentId(platformDocumentId) + const { documentId, workspaceId } = decodeDocumentId(documentName) + const { objectAttr, objectClass, objectId } = documentId const currMarkup = markup.curr[objectAttr] const prevMarkup = markup.prev[objectAttr] @@ -184,10 +159,13 @@ export class PlatformStorageAdapter implements CollabStorageAdapter { return } - const collaborativeDoc = (current as any)[objectAttr] as CollaborativeDoc - const newCollaborativeDoc = collaborativeDocWithLastVersion(collaborativeDoc, `${Date.now()}`) + const blobId = await ctx.with('saveCollabJson', {}, (ctx) => { + return withRetry(ctx, 5, () => { + return saveCollabJson(ctx, this.storage, { name: workspaceId }, documentId, markup.curr[objectAttr]) + }) + }) - await ctx.with('update', {}, () => client.diffUpdate(current, { [objectAttr]: newCollaborativeDoc })) + await ctx.with('update', {}, () => client.diffUpdate(current, { [objectAttr]: blobId })) await ctx.with('activity', {}, () => { const data: AttachedData = { diff --git a/server/collaborator/src/types.ts b/server/collaborator/src/types.ts deleted file mode 100644 index 6171defa96..0000000000 --- a/server/collaborator/src/types.ts +++ /dev/null @@ -1,31 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import type { Class, Doc, Domain, Ref } from '@hcengineering/core' - -/** @public */ -export interface DocumentId { - workspaceId: string - documentId: string - versionId: string -} - -/** @public */ -export interface PlatformDocumentId { - objectDomain: Domain - objectClass: Ref> - objectId: Ref - objectAttr: string -} diff --git a/server/indexer/src/indexer/indexer.ts b/server/indexer/src/indexer/indexer.ts index cb576d29f3..041b35b807 100644 --- a/server/indexer/src/indexer/indexer.ts +++ b/server/indexer/src/indexer/indexer.ts @@ -21,7 +21,6 @@ import core, { type AttachedDoc, type Blob, type Class, - type CollaborativeDoc, DOMAIN_DOC_INDEX_STATE, DOMAIN_MIGRATION, type Doc, @@ -41,7 +40,6 @@ import core, { TxFactory, type WithLookup, type WorkspaceIdWithUrl, - collaborativeDocParse, coreId, docKey, generateId, @@ -63,7 +61,7 @@ import type { StorageAdapter } from '@hcengineering/server-core' import { RateLimiter, SessionDataImpl } from '@hcengineering/server-core' -import { jsonToText, markupToJSON, pmNodeToText, yDocContentToNodes } from '@hcengineering/text' +import { jsonToText, markupToJSON, markupToText } from '@hcengineering/text' import { findSearchPresenter, updateDocWithPresenter } from '../mapper' import { type FullTextPipeline } from './types' import { createIndexedDoc, createStateDoc, getContent } from './utils' @@ -780,14 +778,12 @@ export class FullTextIndexPipeline implements FullTextPipeline { v: { value: any, attr: AnyAttribute }, indexedDoc: IndexedDoc ): Promise { - const collaborativeDoc = v.value as CollaborativeDoc - if (collaborativeDoc !== undefined && collaborativeDoc !== '') { - const { documentId } = collaborativeDocParse(collaborativeDoc) - + const value = v.value as Ref + if (value !== undefined && value !== '') { try { - const readable = await this.storageAdapter?.read(ctx, this.workspace, documentId) - const nodes = yDocContentToNodes(Buffer.concat(readable as any)) - let textContent = nodes.map(pmNodeToText).join('\n') + const readable = await this.storageAdapter?.read(ctx, this.workspace, value) + const markup = Buffer.concat(readable as any).toString() + let textContent = markupToText(markup) textContent = textContent .split(/ +|\t+|\f+/) .filter((it) => it) @@ -798,7 +794,7 @@ export class FullTextIndexPipeline implements FullTextPipeline { indexedDoc.fulltextSummary += '\n' + textContent } catch (err: any) { Analytics.handleError(err) - ctx.error('failed to handle blob', { _id: documentId, workspace: this.workspace.name }) + ctx.error('failed to handle blob', { _id: value, workspace: this.workspace.name }) } } } diff --git a/server/tool/src/initializer.ts b/server/tool/src/initializer.ts index e2782444aa..250115c633 100644 --- a/server/tool/src/initializer.ts +++ b/server/tool/src/initializer.ts @@ -1,11 +1,11 @@ -import { saveCollaborativeDoc } from '@hcengineering/collaboration' +import { saveCollabJson } from '@hcengineering/collaboration' import core, { AttachedDoc, Class, - CollaborativeDoc, Data, Doc, generateId, + makeCollabId, MeasureContext, Mixin, Ref, @@ -16,7 +16,7 @@ import core, { import { ModelLogger } from '@hcengineering/model' import { makeRank } from '@hcengineering/rank' import type { StorageAdapter } from '@hcengineering/server-core' -import { jsonToYDocNoSchema, parseMessageMarkdown } from '@hcengineering/text' +import { jsonToMarkup, parseMessageMarkdown } from '@hcengineering/text' import { v4 as uuid } from 'uuid' const fieldRegexp = /\${\S+?}/ @@ -199,7 +199,7 @@ export class WorkspaceInitializer { if (step.collabFields !== undefined) { for (const field of step.collabFields) { if ((data as any)[field] !== undefined) { - const res = await this.createCollab((data as any)[field], field, _id) + const res = await this.createCollab((data as any)[field], step._class, _id, field) ;(data as any)[field] = res } } @@ -265,15 +265,18 @@ export class WorkspaceInitializer { return data } - private async createCollab (data: string, field: string, _id: Ref): Promise { - const id = `${_id}%${field}` - const collabId = `${id}:HEAD:0` as CollaborativeDoc + private async createCollab ( + data: string, + objectClass: Ref>, + objectId: Ref, + objectAttr: string + ): Promise { + const doc = makeCollabId(objectClass, objectId, objectAttr) const json = parseMessageMarkdown(data ?? '', this.imageUrl) - const yDoc = jsonToYDocNoSchema(json, field) + const markup = jsonToMarkup(json) - await saveCollaborativeDoc(this.ctx, this.storageAdapter, this.wsUrl, collabId, yDoc) - return collabId + return await saveCollabJson(this.ctx, this.storageAdapter, this.wsUrl, doc, markup) } private async fillProps | Props>( diff --git a/services/github/pod-github/src/sync/issueBase.ts b/services/github/pod-github/src/sync/issueBase.ts index 08ba502e24..c767ad9a9c 100644 --- a/services/github/pod-github/src/sync/issueBase.ts +++ b/services/github/pod-github/src/sync/issueBase.ts @@ -14,7 +14,6 @@ import core, { Account, AttachedDoc, Class, - CollaborativeDoc, Doc, DocumentUpdate, Markup, @@ -22,7 +21,8 @@ import core, { Ref, Space, Status, - TxOperations + TxOperations, + makeDocCollabId } from '@hcengineering/core' import github, { DocSyncInfo, @@ -74,8 +74,8 @@ import { /** * @public */ -export type WithMarkup = { - [P in keyof T]: T[P] extends CollaborativeDoc ? Markup : T[P] +export type WithMarkup = Omit & { + description: Markup } /** @@ -259,7 +259,7 @@ export abstract class IssueSyncManagerBase { } let needProjectRefresh = false - const update: DocumentUpdate & Record = {} + const update: DocumentUpdate> & Record = {} let structure = integration.projectStructure.get(target.target._id) @@ -372,7 +372,8 @@ export abstract class IssueSyncManagerBase { !areEqualMarkups(update.description, syncData.current?.description ?? '') ) { try { - await this.collaborator.updateContent(doc.description, { description: update.description }) + const collabId = makeDocCollabId(doc, 'description') + await this.collaborator.updateMarkup(collabId, update.description) } catch (err: any) { Analytics.handleError(err) this.ctx.error(err) @@ -772,7 +773,7 @@ export abstract class IssueSyncManagerBase { } if (update.description !== undefined) { - if (areEqualMarkups(update.description, existingIssue.description)) { + if (update.description === existingIssue.description) { delete update.description } } @@ -924,8 +925,10 @@ export abstract class IssueSyncManagerBase { workspace: this.provider.getWorkspaceId().name }) try { - issueData.description = update.description - await this.collaborator.updateContent(existingIssue.description, { description: update.description }) + const description = update.description as Markup + issueData.description = description + const collabId = makeDocCollabId(existingIssue, 'description') + await this.collaborator.updateMarkup(collabId, description) } catch (err: any) { Analytics.handleError(err) this.ctx.error('error during description update', err) diff --git a/services/github/pod-github/src/sync/issues.ts b/services/github/pod-github/src/sync/issues.ts index bc389a98a3..a2bc87b74c 100644 --- a/services/github/pod-github/src/sync/issues.ts +++ b/services/github/pod-github/src/sync/issues.ts @@ -19,7 +19,9 @@ import core, { TxOperations, cutObjectArray, generateId, - makeCollaborativeDoc + makeDocCollabId, + makeCollabJsonId, + makeCollabId } from '@hcengineering/core' import github, { DocSyncInfo, @@ -239,7 +241,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan case 'assigned': case 'unassigned': { const assignees = await this.getAssigneesI(event.issue) - const update: DocumentUpdate = { + const update: IssueUpdate = { assignee: assignees?.[0]?.person ?? null } await this.handleUpdate(externalData as IssueExternalData, derivedClient, update, account, prj, false) @@ -255,7 +257,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan } const type = await this.provider.getTaskTypeOf(prj.type, tracker.class.Issue) const statuses = await this.provider.getStatuses(type?._id) - const update: DocumentUpdate = { + const update: IssueUpdate = { status: ( await guessStatus( { @@ -380,8 +382,8 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan } const description = await this.ctx.withLog('query collaborative description', {}, async () => { - const content = await this.collaborator.getContent((existing as Issue).description) - return content.description ?? '' + const collabId = makeDocCollabId(existing, 'description') + return await this.collaborator.getMarkup(collabId, (existing as Issue).description) }) this.ctx.info('create github issue', { @@ -624,8 +626,8 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan 'query collaborative description', {}, async () => { - const content = await this.collaborator.getContent((existing as Issue).description) - return content.description ?? '' + const collabId = makeDocCollabId(existing, 'description') + return await this.collaborator.getMarkup(collabId, (existing as Issue).description) }, { url: issueExternal.url } ) @@ -908,9 +910,12 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan const { description, ...update } = issueData + const collabId = makeCollabId(tracker.class.Issue, issueId, 'description') + const contentId = makeCollabJsonId(collabId) + const value: AttachedData = { ...update, - description: makeCollaborativeDoc(issueId, 'description'), + description: contentId, kind: taskType, component: null, milestone: null, @@ -930,7 +935,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan identifier: `${prj.identifier}-${number}` } - await this.collaborator.updateContent(value.description, { description }) + await this.collaborator.updateMarkup(collabId, description) await this.client.addCollection( tracker.class.Issue, diff --git a/services/github/pod-github/src/sync/pullrequests.ts b/services/github/pod-github/src/sync/pullrequests.ts index 731dc79403..33b8b77749 100644 --- a/services/github/pod-github/src/sync/pullrequests.ts +++ b/services/github/pod-github/src/sync/pullrequests.ts @@ -15,7 +15,8 @@ import core, { WithLookup, cutObjectArray, generateId, - makeCollaborativeDoc + makeCollabId, + makeDocCollabId } from '@hcengineering/core' import github, { DocSyncInfo, @@ -217,12 +218,12 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS break } case 'review_requested': { - const update: DocumentUpdate = {} + const update: GithubPullRequestUpdate = {} await this.handleUpdate(externalData, derivedClient, update, account, prj, true) break } case 'review_request_removed': { - const update: DocumentUpdate = {} + const update: GithubPullRequestUpdate = {} await this.handleUpdate(externalData, derivedClient, update, account, prj, true) break } @@ -234,7 +235,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS case 'assigned': case 'unassigned': { const assignees = await this.getAssignees(externalData) - const update: DocumentUpdate = { + const update: GithubPullRequestUpdate = { assignee: assignees?.[0]?.person ?? null } await this.handleUpdate(externalData, derivedClient, update, account, prj, true) @@ -247,7 +248,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS const isMerged = event.pull_request?.merged_at !== null - const update: DocumentUpdate = { + const update: GithubPullRequestUpdate = { draft: externalData.isDraft, head: externalData.headRef, base: externalData.baseRef, @@ -594,8 +595,8 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS 'query collaborative pull request description', {}, async () => { - const content = await this.collaborator.getContent((existing as any).description) - return content.description + const collabId = makeDocCollabId(existing, 'description') + return await this.collaborator.getMarkup(collabId, (existing as GithubPullRequest).description) }, { url: pullRequestExternal.url } ) @@ -998,7 +999,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS async performIssueFieldsUpdate ( info: DocSyncInfo, - existing: Issue, + existing: WithMarkup, platformUpdate: DocumentUpdate, issueData: Pick, 'title' | 'description' | 'assignee' | 'status' | 'remainingTime' | 'component'>, container: ContainerFocus, @@ -1180,7 +1181,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS const number = project.sequence const value: AttachedData = { ...data, - description: makeCollaborativeDoc(prId, 'description'), + description: null, kind: taskType, component: null, milestone: null, @@ -1203,7 +1204,8 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS reviews: 0 } - await this.collaborator.updateContent(value.description, { description }) + const collabId = makeCollabId(github.class.GithubPullRequest, prId, 'description') + await this.collaborator.updateMarkup(collabId, description) await client.addCollection( github.class.GithubPullRequest,