UBERF-6161: Storage configuration (#5109)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/tool/src/clean.ts

@@ -18,6 +18,7 @@ import chunter, { type ChatMessage } from '@hcengineering/chunter'
 import contact from '@hcengineering/contact'
 import core, {
   DOMAIN_TX,
+  type MeasureContext,
   SortingOrder,
   TxOperations,
   TxProcessor,
@@ -43,6 +44,7 @@ import { MongoClient } from 'mongodb'
 export const DOMAIN_ACTIVITY = 'activity' as Domain
 
 export async function cleanWorkspace (
+  ctx: MeasureContext,
   mongoUrl: string,
   workspaceId: WorkspaceId,
   storageAdapter: StorageAdapter,
@@ -67,14 +69,14 @@ export async function cleanWorkspace (
       attachments.map((it) => it.file).concat(contacts.map((it) => it.avatar).filter((it) => it) as string[])
     )
 
-    const minioList = await storageAdapter.list(workspaceId)
+    const minioList = await storageAdapter.list(ctx, workspaceId)
     const toClean: string[] = []
     for (const mv of minioList) {
-      if (!files.has(mv.name)) {
-        toClean.push(mv.name)
+      if (!files.has(mv._id)) {
+        toClean.push(mv._id)
       }
     }
-    await storageAdapter.remove(workspaceId, toClean)
+    await storageAdapter.remove(ctx, workspaceId, toClean)
     // connection.loadChunk(DOMAIN_BLOB, idx = )
 
     if (opt.recruit) {
@@ -145,16 +147,20 @@ export async function cleanWorkspace (
   }
 }
 
-export async function fixMinioBW (workspaceId: WorkspaceId, storageService: StorageAdapter): Promise<void> {
+export async function fixMinioBW (
+  ctx: MeasureContext,
+  workspaceId: WorkspaceId,
+  storageService: StorageAdapter
+): Promise<void> {
   console.log('try clean bw miniature for ', workspaceId.name)
-  const from = new Date(new Date().setDate(new Date().getDate() - 7))
-  const list = await storageService.list(workspaceId)
+  const from = new Date(new Date().setDate(new Date().getDate() - 7)).getTime()
+  const list = await storageService.list(ctx, workspaceId)
   console.log('found', list.length)
   let removed = 0
   for (const obj of list) {
-    if (obj.lastModified < from) continue
-    if (obj.name.includes('%size%')) {
-      await storageService.remove(workspaceId, [obj.name])
+    if (obj.modifiedOn < from) continue
+    if ((obj._id as string).includes('%size%')) {
+      await storageService.remove(ctx, workspaceId, [obj._id])
       removed++
       if (removed % 100 === 0) {
         console.log('removed: ', removed)

dev/tool/src/cleanOrphan.ts

@@ -1,5 +1,5 @@
 import { dropWorkspace, setWorkspaceDisabled, type Workspace } from '@hcengineering/account'
-import core, { AccountRole, MeasureMetricsContext, SortingOrder } from '@hcengineering/core'
+import core, { AccountRole, type MeasureContext, MeasureMetricsContext, SortingOrder } from '@hcengineering/core'
 import contact from '@hcengineering/model-contact'
 import { getWorkspaceDB } from '@hcengineering/mongo'
 import { type StorageAdapter } from '@hcengineering/server-core'
@@ -7,6 +7,7 @@ import { connect } from '@hcengineering/server-tool'
 import { type Db, type MongoClient } from 'mongodb'
 
 export async function checkOrphanWorkspaces (
+  ctx: MeasureContext,
   workspaces: Workspace[],
   transactorUrl: string,
   productId: string,
@@ -40,7 +41,7 @@ export async function checkOrphanWorkspaces (
 
       // Find last transaction index:
       const wspace = { name: ws.workspace, productId }
-      const hasBucket = await storageAdapter.exists(wspace)
+      const hasBucket = await storageAdapter.exists(ctx, wspace)
       const [lastTx] = await connection.findAll(
         core.class.Tx,
         {
@@ -69,12 +70,13 @@ export async function checkOrphanWorkspaces (
           const workspaceDb = getWorkspaceDB(client, { name: ws.workspace, productId })
           await workspaceDb.dropDatabase()
           if (storageAdapter !== undefined && hasBucket) {
-            const docs = await storageAdapter.list(wspace)
+            const docs = await storageAdapter.list(ctx, wspace)
             await storageAdapter.remove(
+              ctx,
               wspace,
-              docs.map((it) => it.name)
+              docs.map((it) => it._id)
             )
-            await storageAdapter?.delete(wspace)
+            await storageAdapter.delete(ctx, wspace)
           }
         }
       }

dev/tool/src/index.ts

@@ -410,6 +410,7 @@ export function devTool (
 
         // We need to update workspaces with missing workspaceUrl
         await checkOrphanWorkspaces(
+          toolCtx,
           workspaces,
           transactorUrl,
           productId,
@@ -482,7 +483,6 @@ export function devTool (
   program
     .command('backup <dirName> <workspace>')
     .description('dump workspace transactions and minio resources')
-    .requiredOption('-i --index <index>', 'Index name for elastic')
     .option('-s, --skip <skip>', 'A list of ; separated domain names to skip during backup', '')
     .option('-f, --force', 'Force backup', false)
     .action(async (dirName: string, workspace: string, cmd: { skip: string, force: boolean }) => {
@@ -518,7 +518,12 @@ export function devTool (
     .description('dump workspace transactions and minio resources')
     .action(async (bucketName: string, dirName: string, workspace: string, cmd) => {
       const { storageAdapter } = prepareTools()
-      const storage = await createStorageBackupStorage(storageAdapter, getWorkspaceId(bucketName, productId), dirName)
+      const storage = await createStorageBackupStorage(
+        toolCtx,
+        storageAdapter,
+        getWorkspaceId(bucketName, productId),
+        dirName
+      )
       await backup(transactorUrl, getWorkspaceId(workspace, productId), storage)
     })
   program
@@ -526,7 +531,7 @@ export function devTool (
     .description('dump workspace transactions and minio resources')
     .action(async (bucketName: string, dirName: string, workspace: string, date, cmd) => {
       const { storageAdapter } = prepareTools()
-      const storage = await createStorageBackupStorage(storageAdapter, getWorkspaceId(bucketName), dirName)
+      const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName)
       await restore(transactorUrl, getWorkspaceId(workspace, productId), storage, parseInt(date ?? '-1'))
     })
   program
@@ -535,7 +540,12 @@ export function devTool (
     .action(async (bucketName: string, dirName: string, cmd) => {
       const { storageAdapter } = prepareTools()
 
-      const storage = await createStorageBackupStorage(storageAdapter, getWorkspaceId(bucketName, productId), dirName)
+      const storage = await createStorageBackupStorage(
+        toolCtx,
+        storageAdapter,
+        getWorkspaceId(bucketName, productId),
+        dirName
+      )
       await backupList(storage)
     })
 
@@ -576,7 +586,7 @@ export function devTool (
         }
 
         console.log(`clearing ${workspace} history:`)
-        await clearTelegramHistory(mongodbUri, getWorkspaceId(workspace, productId), telegramDB, minio)
+        await clearTelegramHistory(toolCtx, mongodbUri, getWorkspaceId(workspace, productId), telegramDB, minio)
       })
     })
 
@@ -596,7 +606,7 @@ export function devTool (
 
         for (const w of workspaces) {
           console.log(`clearing ${w.workspace} history:`)
-          await clearTelegramHistory(mongodbUri, getWorkspaceId(w.workspace, productId), telegramDB, minio)
+          await clearTelegramHistory(toolCtx, mongodbUri, getWorkspaceId(w.workspace, productId), telegramDB, minio)
         }
       })
     })
@@ -624,6 +634,7 @@ export function devTool (
       const { mongodbUri, storageAdapter: minio } = prepareTools()
       await withDatabase(mongodbUri, async (db) => {
         await cleanWorkspace(
+          toolCtx,
           mongodbUri,
           getWorkspaceId(workspace, productId),
           minio,
@@ -636,7 +647,7 @@ export function devTool (
 
   program.command('fix-bw-workspace <workspace>').action(async (workspace: string) => {
     const { storageAdapter: minio } = prepareTools()
-    await fixMinioBW(getWorkspaceId(workspace, productId), minio)
+    await fixMinioBW(toolCtx, getWorkspaceId(workspace, productId), minio)
   })
 
   program

dev/tool/src/telegram.ts

@@ -14,7 +14,7 @@
 // limitations under the License.
 //
 
-import { DOMAIN_TX, type Ref, type WorkspaceId } from '@hcengineering/core'
+import { DOMAIN_TX, type MeasureContext, type Ref, type WorkspaceId } from '@hcengineering/core'
 import { type StorageAdapter } from '@hcengineering/server-core'
 import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment'
 import contact, { DOMAIN_CHANNEL } from '@hcengineering/model-contact'
@@ -29,6 +29,7 @@ const LastMessages = 'last-msgs'
  * @public
  */
 export async function clearTelegramHistory (
+  ctx: MeasureContext,
   mongoUrl: string,
   workspaceId: WorkspaceId,
   tgDb: string,
@@ -90,7 +91,7 @@ export async function clearTelegramHistory (
       workspaceDB.collection(DOMAIN_ATTACHMENT).deleteMany({
         attachedToClass: telegram.class.Message
       }),
-      storageAdapter.remove(workspaceId, Array.from(attachments))
+      storageAdapter.remove(ctx, workspaceId, Array.from(attachments))
     ])
 
     console.log('clearing telegram service data...')