Merge remote-tracking branch 'origin/develop' into staging-new

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>

dev/tool/package.json

@@ -61,6 +61,11 @@
   "dependencies": {
     "@elastic/elasticsearch": "^7.17.14",
     "@hcengineering/account": "^0.7.0",
+    "@hcengineering/communication": "^0.7.0",
+    "@hcengineering/chat": "^0.7.0",
+    "@hcengineering/card": "^0.7.0",
+    "@hcengineering/rank": "^0.7.17",
+    "@hcengineering/text-markdown": "^0.7.18",
     "@hcengineering/account-service": "^0.7.0",
     "@hcengineering/workspace-service": "^0.7.0",
     "@hcengineering/attachment": "^0.7.0",

dev/tool/src/index.ts

@@ -135,9 +135,9 @@ import { sendTransactorEvent } from '@hcengineering/server-tool'
 import { existsSync } from 'fs'
 import { mkdir, writeFile } from 'fs/promises'
 import { dirname } from 'path'
-import { migrateWorkspaceMessages } from './communication'
 import { restoreMarkupRefs } from './markup'
 import { restoreGithubIntegrations } from './restoreGithub'
+import { migrateWorkspaceChat } from './communication'
 
 const colorConstants = {
   colorRed: '\u001b[31m',
@@ -2853,15 +2853,16 @@ export function devTool (
     })
 
   program
-    .command('migrate-communication-to-hulylake <workspace>')
-    .description('Migrate communication messages to hulylake')
-    .option('-c, --card <card>', 'Card to migrate')
-    .action(async (workspace: WorkspaceUuid, cmd: { card?: CardID }) => {
-      const { dbUrl } = prepareTools()
+    .command('migrate-chat-to-communication')
+    .description('Migrate old chat to new communication')
+    .option('-w, --workspace <workspace>', 'Workspace to migrate')
+    .action(async (cmd: { workspace?: WorkspaceUuid }) => {
+      const { dbUrl, txes } = prepareTools()
       const hulylakeUrl = process.env.HULYLAKE_URL ?? ''
 
+      const workspace = cmd.workspace
       console.log('Workspace', workspace)
-      console.log('Card', cmd.card)
+
       if (hulylakeUrl === '') {
         throw new Error('HULYLAKE_URL should be specified')
       }
@@ -2873,31 +2874,73 @@ export function devTool (
       const dbClient = await db.getClient()
       const accountClient = getAccountClient(token)
       const personUuidBySocialId = new Map<PersonId, PersonUuid>()
+      const storageConfig = storageConfigFromEnv()
+      const storage: StorageAdapter = buildStorageFromConfig(storageConfig)
 
       await withAccountDatabase(async (accountDb) => {
-        const ws = (await accountDb.workspace.find({ uuid: workspace }))[0]
-        if (ws === undefined) {
-          throw new Error(`Workspace ${workspace} not found`)
-        }
-        try {
-          const hulylake = getHulylakeClient(hulylakeUrl, ws.uuid, token)
-          console.log('start workspace migration', ws.name)
-          await migrateWorkspaceMessages(
-            toolCtx.newChild(ws.name, {}),
-            ws,
-            cmd.card,
-            dbClient,
-            hulylake,
-            accountClient,
-            personUuidBySocialId
-          )
-          console.log('done workspace migration', ws.name)
-        } catch (err: any) {
-          console.error('failed to migrate workspace', ws.name)
-          console.error(err)
+        const workspaces =
+          workspace != null
+            ? await accountDb.workspaceStatus.find({ workspaceUuid: workspace })
+            : await accountDb.workspaceStatus.find({}, { lastVisit: 'descending' })
+        for (const wss of workspaces) {
+          try {
+            const ws = await accountDb.workspace.findOne({ uuid: wss.workspaceUuid })
+            if (ws == null) continue
+            const hulylake = getHulylakeClient(hulylakeUrl, ws.uuid, token)
+
+            let pipeline: Pipeline | undefined
+            try {
+              pipeline = await createBackupPipeline(toolCtx, dbUrl, txes, {
+                externalStorage: storage,
+                usePassedCtx: true
+              })(
+                toolCtx,
+                {
+                  uuid: ws.uuid,
+                  url: ws.url ?? '',
+                  dataId: ws.dataId
+                },
+                createEmptyBroadcastOps(),
+                null
+              )
+            } catch (e) {
+              pipeline = undefined
+            }
+            if (pipeline === undefined) {
+              toolCtx.error('failed to migrate, pipeline is undefined', { ws })
+              return
+            }
+            const client = pipeline.context.lowLevelStorage
+
+            if (client == null) {
+              toolCtx.error('failed to migrate, lowLevelStorage is undefined', { ws })
+              return
+            }
+
+            console.log('------------start workspace migration', ws.name)
+            const s = Date.now()
+            await migrateWorkspaceChat(
+              toolCtx.newChild(ws.name, {}),
+              ws,
+              dbClient,
+              client,
+              pipeline.context.hierarchy,
+              hulylake,
+              accountClient,
+              personUuidBySocialId
+            )
+            const e = Date.now()
+            console.log('---------------done workspace migration', ws.name, ((e - s) / 1000 / 60).toFixed(2), 'minutes')
+            await pipeline.close()
+          } catch (err: any) {
+            console.error('failed to migrate workspace', wss.workspaceUuid)
+            console.error(err)
+          }
         }
         db.close()
       }, dbUrl)
+
+      console.log('done')
     })
 
   program

models/chunter/src/types.ts

@@ -22,7 +22,8 @@ import {
   TypeMarkup,
   TypeRef,
   TypeString,
-  UX
+  UX,
+  Hidden
 } from '@hcengineering/model'
 import core, { TClass, TDoc, TSpace } from '@hcengineering/model-core'
 import type {
@@ -42,6 +43,7 @@ import {
   DOMAIN_MODEL,
   IndexKind,
   type Ref,
+  type Space,
   type Timestamp
 } from '@hcengineering/core'
 import contact, { type ChannelProvider as SocialChannelProvider, type Person } from '@hcengineering/contact'
@@ -59,6 +61,15 @@ export const DOMAIN_CHUNTER = 'chunter' as Domain
 export class TChunterSpace extends TSpace implements ChunterSpace {
   @Prop(PropCollection(activity.class.ActivityMessage), chunter.string.Messages)
     messages?: number
+
+  @Hidden()
+    __migratedToCard?: {
+    card: Ref<Doc>
+    space: Ref<Space>
+  }
+
+  @Hidden()
+    __migratedUntil?: Timestamp
 }
 
 @Model(chunter.class.Channel, chunter.class.ChunterSpace)

plugins/chunter/src/index.ts

@@ -28,6 +28,12 @@ import { Widget, WidgetTab } from '@hcengineering/workbench'
  */
 export interface ChunterSpace extends Space {
   messages?: number
+
+  __migratedToCard?: {
+    card?: Ref<Doc>
+    space?: Ref<Space>
+  }
+  __migratedUntil?: Timestamp
 }
 
 /**

server/backup/src/backup.ts

@@ -100,6 +100,7 @@ export async function backup (
     progress?: (progress: number) => Promise<void>
     token?: string
     fullVerify?: boolean
+    forceCompact?: boolean
     keepSnapshots: number
     msg?: Record<string, any>
   } = {
@@ -146,6 +147,7 @@ export async function backup (
   const tmpRoot = mkdtempSync('huly')
 
   const forcedFullCheck = '4'
+  const forcedCompact = '1'
 
   try {
     let backupInfo: BackupInfo = {
@@ -155,6 +157,7 @@ export async function backup (
       domainHashes: {},
       migrations: {
         zeroCheckSize: true, // Assume already checked for new backups
+        forcedCompact, // Force backup compaction
         forcedFullCheck // A force to full recheck.
       },
       dataSize: 0,
@@ -202,12 +205,18 @@ export async function backup (
     }
 
     let fullCheck = options.fullVerify === true
+    let forceCompact = options.forceCompact === true
 
     if (backupInfo.migrations.forcedFullCheck !== forcedFullCheck) {
       // We have forced full check to be performed.
       fullCheck = true
     }
-    if (backupInfo.snapshots.length > options.keepSnapshots) {
+    if (backupInfo.migrations.forcedCompact !== forcedCompact) {
+      // We have forced compaction to be performed.
+      forceCompact = true
+    }
+
+    if (backupInfo.snapshots.length > options.keepSnapshots || forceCompact) {
       // We need to perform compaction
       ctx.warn('Compacting backup')
       await compactBackup(ctx, storage, true, {
@@ -1176,6 +1185,7 @@ export async function backup (
     if (!canceled() && domainChanges > 0) {
       backupInfo.lastTxId = lastTx?._id ?? '0' // We could store last tx, since full backup is complete
       backupInfo.migrations.forcedFullCheck = forcedFullCheck
+      backupInfo.migrations.forcedCompaction = forcedCompact
       backupInfo.dataSize = result.dataSize
       backupInfo.blobsSize = result.blobsSize
       backupInfo.backupSize = result.backupSize