From ea0819e086881ad202a15c8f9329d51a2a1be4e5 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 25 Sep 2024 20:21:23 +0700 Subject: [PATCH] feat: restore lost markup tool (#6724) Signed-off-by: Alexander Onnikov --- dev/tool/src/index.ts | 34 ++++++++++- dev/tool/src/markup.ts | 133 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 164 insertions(+), 3 deletions(-) diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 314ba72f37..714fc15d6d 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -112,7 +112,7 @@ import { } from './clean' import { changeConfiguration } from './configuration' import { moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db' -import { fixJsonMarkup, migrateMarkup } from './markup' +import { fixJsonMarkup, migrateMarkup, restoreLostMarkup } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { importNotion } from './notion' import { fixAccountEmails, renameAccount } from './renameAccount' @@ -1273,6 +1273,38 @@ export function devTool ( }) }) + program.command('show-lost-markup ').action(async (workspace: string, cmd: any) => { + const { mongodbUri } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + await withStorage(mongodbUri, async (adapter) => { + try { + const workspaceId = getWorkspaceId(workspace) + const token = generateToken(systemAccountEmail, workspaceId) + const endpoint = await getTransactorEndpoint(token) + await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'show' }) + } catch (err: any) { + console.error(err) + } + }) + }) + }) + + program.command('restore-lost-markup ').action(async (workspace: string, cmd: any) => { + const { mongodbUri } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + await withStorage(mongodbUri, async (adapter) => { + try { + const workspaceId = getWorkspaceId(workspace) + const token = generateToken(systemAccountEmail, workspaceId) + const endpoint = await getTransactorEndpoint(token) + await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'restore' }) + } catch (err: any) { + console.error(err) + } + }) + }) + }) + program.command('fix-bw-workspace ').action(async (workspace: string) => { const { mongodbUri } = prepareTools() await withStorage(mongodbUri, async (adapter) => { diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index 837a0cb7f9..13143d5906 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -3,20 +3,28 @@ import core, { type AnyAttribute, type Class, type Client as CoreClient, + type CollaborativeDoc, type Doc, + type DocIndexState, type Domain, type Hierarchy, + type Markup, type MeasureContext, type Ref, + type TxMixin, + type TxCreateDoc, + type TxUpdateDoc, type WorkspaceId, RateLimiter, collaborativeDocParse, - makeCollaborativeDoc + makeCollaborativeDoc, + SortingOrder, + TxProcessor } from '@hcengineering/core' import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo' import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core' import { connect } from '@hcengineering/server-tool' -import { jsonToText, markupToYDoc } from '@hcengineering/text' +import { isEmptyMarkup, jsonToText, markupToYDoc } from '@hcengineering/text' import { type Db, type FindCursor, type MongoClient } from 'mongodb' export async function fixJsonMarkup ( @@ -212,3 +220,124 @@ async function processMigrateMarkupFor ( console.log('processed', processed) } + +export async function restoreLostMarkup ( + ctx: MeasureContext, + workspaceId: WorkspaceId, + transactorUrl: string, + storageAdapter: StorageAdapter, + { command }: { command: 'show' | 'restore' } +): Promise { + const connection = (await connect(transactorUrl, workspaceId, undefined, { + mode: 'backup' + })) as unknown as CoreClient + + try { + const hierarchy = connection.getHierarchy() + const classes = hierarchy.getDescendants(core.class.Doc) + + for (const _class of classes) { + const isAttachedDoc = hierarchy.isDerived(_class, core.class.AttachedDoc) + + const attributes = hierarchy.getAllAttributes(_class) + const attrs = Array.from(attributes.values()).filter((p) => p.type._class === core.class.TypeCollaborativeDoc) + + // ignore classes with no collaborative attributes + if (attrs.length === 0) continue + + const docs = await connection.findAll(_class, { _class }) + for (const doc of docs) { + for (const attr of attrs) { + const value = hierarchy.isMixin(attr.attributeOf) + ? ((doc as any)[attr.attributeOf]?.[attr.name] as CollaborativeDoc) + : ((doc as any)[attr.name] as CollaborativeDoc) + + if (value == null || value === '') continue + + const { documentId } = collaborativeDocParse(value) + const stat = await storageAdapter.stat(ctx, workspaceId, documentId) + if (stat !== undefined) continue + + const query = isAttachedDoc + ? { + 'tx.objectId': doc._id, + 'tx._class': { $in: [core.class.TxCreateDoc, core.class.TxUpdateDoc] } + } + : { + objectId: doc._id + } + + let restored = false + + // try to restore by txes + // we need last tx that modified the attribute + + const txes = await connection.findAll(isAttachedDoc ? core.class.TxCollectionCUD : core.class.TxCUD, query, { + sort: { modifiedOn: SortingOrder.Descending } + }) + for (const tx of txes) { + const innerTx = TxProcessor.extractTx(tx) + + let markup: string | undefined + if (innerTx._class === core.class.TxMixin) { + const mixinTx = innerTx as TxMixin + markup = (mixinTx.attributes as any)[attr.name] + } else if (innerTx._class === core.class.TxCreateDoc) { + const createTx = innerTx as TxCreateDoc + markup = (createTx.attributes as any)[attr.name] + } else if (innerTx._class === core.class.TxUpdateDoc) { + const updateTex = innerTx as TxUpdateDoc + markup = (updateTex.operations as any)[attr.name] + } else { + continue + } + + if (markup === undefined || !markup.startsWith('{')) continue + if (isEmptyMarkup(markup)) continue + + console.log(doc._class, doc._id, attr.name, markup) + if (command === 'restore') { + const ydoc = markupToYDoc(markup, attr.name) + await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx) + } + restored = true + break + } + + if (restored) continue + + // try to restore by doc index state + const docIndexState = await connection.findOne(core.class.DocIndexState, { + _id: doc._id as Ref + }) + if (docIndexState !== undefined) { + // document:class:Document%content#content#base64 + const attrName = `${doc._class}%${attr.name}#content#base64` + const base64: string | undefined = docIndexState.attributes[attrName] + if (base64 !== undefined) { + const text = Buffer.from(base64, 'base64').toString() + if (text !== '') { + const markup: Markup = JSON.stringify({ + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text, marks: [] }] + } + ] + }) + console.log(doc._class, doc._id, attr.name, markup) + if (command === 'restore') { + const ydoc = markupToYDoc(markup, attr.name) + await saveCollaborativeDoc(storageAdapter, workspaceId, value, ydoc, ctx) + } + } + } + } + } + } + } + } finally { + await connection.close() + } +}