Cockroach (#6334)

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

dev/tool/src/__start.ts

@@ -73,6 +73,7 @@ addLocation(serverAiBotId, () => import('@hcengineering/server-ai-bot-resources'
 
 function prepareTools (): {
   mongodbUri: string
+  dbUrl: string | undefined
   txes: Tx[]
   version: Data<Version>
   migrateOperations: [string, MigrateOperation][]

dev/tool/src/benchmark.ts

@@ -14,11 +14,14 @@
 //
 
 import core, {
+  AccountRole,
   MeasureMetricsContext,
   RateLimiter,
   TxOperations,
   concatLink,
   generateId,
+  getWorkspaceId,
+  makeCollaborativeDoc,
   metricsToString,
   newMetrics,
   systemAccountEmail,
@@ -40,6 +43,10 @@ import os from 'os'
 import { Worker, isMainThread, parentPort } from 'worker_threads'
 import { CSVWriter } from './csv'
 
+import { AvatarType, type PersonAccount } from '@hcengineering/contact'
+import contact from '@hcengineering/model-contact'
+import recruit from '@hcengineering/model-recruit'
+import { type Vacancy } from '@hcengineering/recruit'
 import { WebSocket } from 'ws'
 
 interface StartMessage {
@@ -503,3 +510,117 @@ export async function stressBenchmark (transactor: string, mode: StressBenchmark
     }
   }
 }
+
+export async function testFindAll (endpoint: string, workspace: string, email: string): Promise<void> {
+  const connection = await connect(endpoint, getWorkspaceId(workspace), email)
+  try {
+    const client = new TxOperations(connection, core.account.System)
+    const start = Date.now()
+    const res = await client.findAll(
+      recruit.class.Applicant,
+      {},
+      {
+        lookup: {
+          attachedTo: recruit.mixin.Candidate,
+          space: recruit.class.Vacancy
+        }
+      }
+    )
+    console.log('Find all', res.length, 'time', Date.now() - start)
+  } finally {
+    await connection.close()
+  }
+}
+
+export async function generateWorkspaceData (
+  endpoint: string,
+  workspace: string,
+  parallel: boolean,
+  user: string
+): Promise<void> {
+  const connection = await connect(endpoint, getWorkspaceId(workspace))
+  const client = new TxOperations(connection, core.account.System)
+  try {
+    const acc = await client.findOne(contact.class.PersonAccount, { email: user })
+    if (acc == null) {
+      throw new Error('User not found')
+    }
+    const employees: Ref<PersonAccount>[] = [acc._id]
+    const start = Date.now()
+    for (let i = 0; i < 100; i++) {
+      const acc = await generateEmployee(client)
+      employees.push(acc)
+    }
+    if (parallel) {
+      const promises: Promise<void>[] = []
+      for (let i = 0; i < 10; i++) {
+        promises.push(generateVacancy(client, employees))
+      }
+      await Promise.all(promises)
+    } else {
+      for (let i = 0; i < 10; i++) {
+        await generateVacancy(client, employees)
+      }
+    }
+    console.log('Generate', Date.now() - start)
+  } finally {
+    await connection.close()
+  }
+}
+
+export async function generateEmployee (client: TxOperations): Promise<Ref<PersonAccount>> {
+  const personId = await client.createDoc(contact.class.Person, contact.space.Contacts, {
+    name: generateId().toString(),
+    city: '',
+    avatarType: AvatarType.COLOR
+  })
+  await client.createMixin(personId, contact.class.Person, contact.space.Contacts, contact.mixin.Employee, {
+    active: true
+  })
+  const acc = await client.createDoc(contact.class.PersonAccount, core.space.Model, {
+    person: personId,
+    role: AccountRole.User,
+    email: personId
+  })
+  return acc
+}
+
+async function generateVacancy (client: TxOperations, members: Ref<PersonAccount>[]): Promise<void> {
+  // generate vacancies
+  const _id = generateId<Vacancy>()
+  await client.createDoc(
+    recruit.class.Vacancy,
+    core.space.Space,
+    {
+      name: generateId().toString(),
+      number: 0,
+      fullDescription: makeCollaborativeDoc(_id, 'fullDescription'),
+      type: recruit.template.DefaultVacancy,
+      description: '',
+      private: false,
+      members,
+      archived: false
+    },
+    _id
+  )
+  for (let i = 0; i < 100; i++) {
+    // generate candidate
+    const personId = await client.createDoc(contact.class.Person, contact.space.Contacts, {
+      name: generateId().toString(),
+      city: '',
+      avatarType: AvatarType.COLOR
+    })
+    await client.createMixin(personId, contact.class.Person, contact.space.Contacts, recruit.mixin.Candidate, {})
+    // generate applicants
+    await client.addCollection(recruit.class.Applicant, _id, personId, recruit.mixin.Candidate, 'applications', {
+      status: recruit.taskTypeStatus.Backlog,
+      number: i + 1,
+      identifier: `APP-${i + 1}`,
+      assignee: null,
+      rank: '',
+      startDate: null,
+      dueDate: null,
+      kind: recruit.taskTypes.Applicant
+    })
+  }
+}

dev/tool/src/db.ts

@@ -0,0 +1,68 @@
+import { type Doc, type WorkspaceId } from '@hcengineering/core'
+import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
+import { convertDoc, createTable, getDBClient, retryTxn, translateDomain } from '@hcengineering/postgres'
+
+export async function moveFromMongoToPG (
+  mongoUrl: string,
+  dbUrl: string | undefined,
+  workspaces: WorkspaceId[]
+): Promise<void> {
+  if (dbUrl === undefined) {
+    throw new Error('dbUrl is required')
+  }
+  const client = getMongoClient(mongoUrl)
+  const mongo = await client.getClient()
+  const pg = getDBClient(dbUrl)
+  const pgClient = await pg.getClient()
+
+  for (let index = 0; index < workspaces.length; index++) {
+    const ws = workspaces[index]
+    try {
+      const mongoDB = getWorkspaceDB(mongo, ws)
+      const collections = await mongoDB.collections()
+      await createTable(
+        pgClient,
+        collections.map((c) => c.collectionName)
+      )
+      for (const collection of collections) {
+        const cursor = collection.find()
+        const domain = translateDomain(collection.collectionName)
+        while (true) {
+          const doc = (await cursor.next()) as Doc | null
+          if (doc === null) break
+          try {
+            const converted = convertDoc(doc, ws.name)
+            await retryTxn(pgClient, async (client) => {
+              await client.query(
+                `INSERT INTO ${domain} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
+                [
+                  converted._id,
+                  converted.workspaceId,
+                  converted._class,
+                  converted.createdBy,
+                  converted.modifiedBy,
+                  converted.modifiedOn,
+                  converted.createdOn,
+                  converted.space,
+                  converted.attachedTo,
+                  converted.data
+                ]
+              )
+            })
+          } catch (err) {
+            console.log('error when move doc', doc._id, doc._class, err)
+            continue
+          }
+        }
+      }
+      if (index % 100 === 0) {
+        console.log('Move workspace', index, workspaces.length)
+      }
+    } catch (err) {
+      console.log('Error when move workspace', ws.name, err)
+      throw err
+    }
+  }
+  pg.close()
+  client.close()
+}

dev/tool/src/index.ts

@@ -19,6 +19,7 @@ import accountPlugin, {
   assignWorkspace,
   confirmEmail,
   createAcc,
+  createWorkspace as createWorkspaceRecord,
   dropAccount,
   dropWorkspace,
   dropWorkspaceFull,
@@ -32,10 +33,8 @@ import accountPlugin, {
   setAccountAdmin,
   setRole,
   updateWorkspace,
-  createWorkspace as createWorkspaceRecord,
   type Workspace
 } from '@hcengineering/account'
-import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
 import { setMetadata } from '@hcengineering/platform'
 import {
   backup,
@@ -54,8 +53,10 @@ import serverClientPlugin, {
   login,
   selectWorkspace
 } from '@hcengineering/server-client'
+import { getServerPipeline } from '@hcengineering/server-pipeline'
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
 import toolPlugin, { connect, FileModelLogger } from '@hcengineering/server-tool'
+import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
 import path from 'path'
 
 import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
@@ -66,6 +67,8 @@ import { diffWorkspace, recreateElastic, updateField } from './workspace'
 
 import core, {
   AccountRole,
+  concatLink,
+  generateId,
   getWorkspaceId,
   MeasureMetricsContext,
   metricsToString,
@@ -79,7 +82,7 @@ import core, {
   type Tx,
   type Version,
   type WorkspaceId,
-  concatLink
+  type WorkspaceIdWithUrl
 } from '@hcengineering/core'
 import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
 import contact from '@hcengineering/model-contact'
@@ -87,7 +90,14 @@ import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
 import type { StorageAdapter, StorageAdapterEx } from '@hcengineering/server-core'
 import { deepEqual } from 'fast-equals'
 import { createWriteStream, readFileSync } from 'fs'
-import { benchmark, benchmarkWorker, stressBenchmark, type StressBenchmarkMode } from './benchmark'
+import {
+  benchmark,
+  benchmarkWorker,
+  generateWorkspaceData,
+  stressBenchmark,
+  testFindAll,
+  type StressBenchmarkMode
+} from './benchmark'
 import {
   cleanArchivedSpaces,
   cleanRemovedTransactions,
@@ -101,11 +111,12 @@ import {
   restoreRecruitingTaskTypes
 } from './clean'
 import { changeConfiguration } from './configuration'
+import { moveFromMongoToPG } from './db'
 import { fixJsonMarkup, migrateMarkup } from './markup'
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
+import { importNotion } from './notion'
 import { fixAccountEmails, renameAccount } from './renameAccount'
 import { moveFiles, syncFiles } from './storage'
-import { importNotion } from './notion'
 
 const colorConstants = {
   colorRed: '\u001b[31m',
@@ -125,6 +136,7 @@ const colorConstants = {
 export function devTool (
   prepareTools: () => {
     mongodbUri: string
+    dbUrl: string | undefined
     txes: Tx[]
     version: Data<Version>
     migrateOperations: [string, MigrateOperation][]
@@ -1470,7 +1482,7 @@ export function devTool (
     .option('-w, --workspace <workspace>', 'Selected workspace only', '')
     .option('-c, --concurrency <concurrency>', 'Number of documents being processed concurrently', '10')
     .action(async (cmd: { workspace: string, concurrency: string }) => {
-      const { mongodbUri } = prepareTools()
+      const { mongodbUri, dbUrl } = prepareTools()
       await withDatabase(mongodbUri, async (db, client) => {
         await withStorage(mongodbUri, async (adapter) => {
           const workspaces = await listWorkspacesPure(db)
@@ -1482,8 +1494,15 @@ export function devTool (
 
             const wsId = getWorkspaceId(workspace.workspace)
             console.log('processing workspace', workspace.workspace, index, workspaces.length)
+            const wsUrl: WorkspaceIdWithUrl = {
+              name: workspace.workspace,
+              workspaceName: workspace.workspaceName ?? '',
+              workspaceUrl: workspace.workspaceUrl ?? ''
+            }
 
-            await migrateMarkup(toolCtx, adapter, wsId, client, mongodbUri, parseInt(cmd.concurrency))
+            const { pipeline } = await getServerPipeline(toolCtx, mongodbUri, dbUrl, wsUrl)
+
+            await migrateMarkup(toolCtx, adapter, wsId, client, pipeline, parseInt(cmd.concurrency))
 
             console.log('...done', workspace.workspace)
             index++
@@ -1502,6 +1521,57 @@ export function devTool (
       })
     })
 
+  program.command('move-to-pg').action(async () => {
+    const { mongodbUri, dbUrl } = prepareTools()
+    await withDatabase(mongodbUri, async (db) => {
+      const workspaces = await listWorkspacesRaw(db)
+      await moveFromMongoToPG(
+        mongodbUri,
+        dbUrl,
+        workspaces.map((it) => getWorkspaceId(it.workspace))
+      )
+    })
+  })
+
+  program
+    .command('perfomance')
+    .option('-p, --parallel', '', false)
+    .action(async (cmd: { parallel: boolean }) => {
+      const { mongodbUri, txes, version, migrateOperations } = prepareTools()
+      await withDatabase(mongodbUri, async (db) => {
+        const email = generateId()
+        const ws = generateId()
+        const wsid = getWorkspaceId(ws)
+        const start = new Date()
+        const measureCtx = new MeasureMetricsContext('create-workspace', {})
+        const wsInfo = await createWorkspaceRecord(measureCtx, db, null, email, ws, ws)
+
+        // update the record so it's not taken by one of the workers for the next 60 seconds
+        await updateWorkspace(db, wsInfo, {
+          mode: 'creating',
+          progress: 0,
+          lastProcessingTime: Date.now() + 1000 * 60
+        })
+
+        await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations)
+
+        await updateWorkspace(db, wsInfo, {
+          mode: 'active',
+          progress: 100,
+          disabled: false,
+          version
+        })
+        await createAcc(toolCtx, db, null, email, '1234', '', '', true)
+        await assignWorkspace(toolCtx, db, null, email, ws, AccountRole.User)
+        console.log('Workspace created in', new Date().getTime() - start.getTime(), 'ms')
+        const token = generateToken(systemAccountEmail, wsid)
+        const endpoint = await getTransactorEndpoint(token, 'external')
+        await generateWorkspaceData(endpoint, ws, cmd.parallel, email)
+        await testFindAll(endpoint, ws, email)
+        await dropWorkspace(toolCtx, db, null, ws)
+      })
+    })
+
   extendProgram?.(program)
 
   program.parse(process.argv)

dev/tool/src/markup.ts

@@ -14,8 +14,8 @@ import core, {
   makeCollaborativeDoc
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
-import { type StorageAdapter } from '@hcengineering/server-core'
-import { connect, fetchModelFromMongo } from '@hcengineering/server-tool'
+import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core'
+import { connect, fetchModel } from '@hcengineering/server-tool'
 import { jsonToText, markupToYDoc } from '@hcengineering/text'
 import { type Db, type FindCursor, type MongoClient } from 'mongodb'
 
@@ -120,10 +120,10 @@ export async function migrateMarkup (
   storageAdapter: StorageAdapter,
   workspaceId: WorkspaceId,
   client: MongoClient,
-  mongodbUri: string,
+  pipeline: Pipeline,
   concurrency: number
 ): Promise<void> {
-  const { hierarchy } = await fetchModelFromMongo(ctx, mongodbUri, workspaceId)
+  const { hierarchy } = await fetchModel(ctx, pipeline)
 
   const workspaceDb = client.db(workspaceId.name)