UBERF-8122: Fix backup service

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

server/core/src/__tests__/memAdapters.ts

@@ -56,7 +56,7 @@ export class MemStorageAdapter implements StorageAdapter {
     const files = Array.from(this.files.values()).filter((it) => it.workspace === workspaceId.name)
     return {
       next: async () => {
-        return files.shift()
+        return files.splice(0, 100)
       },
       close: async () => {}
     }
@@ -189,8 +189,7 @@ export class MemRawDBAdapter implements RawDBAdapter {
     }
     return {
       next: async () => {
-        const doc = result.shift()
-        return doc
+        return result.splice(0, 50)
       },
       close: async () => {}
     }

server/core/src/adapter.ts

@@ -57,7 +57,7 @@ export interface DomainHelper {
 }
 
 export interface RawDBAdapterStream<T extends Doc> {
-  next: () => Promise<T | undefined>
+  next: () => Promise<T[]>
   close: () => Promise<void>
 }
 

server/core/src/mem.ts

@@ -86,7 +86,7 @@ export class DummyDbAdapter implements DbAdapter {
 
   find (ctx: MeasureContext, domain: Domain): StorageIterator {
     return {
-      next: async () => undefined,
+      next: async () => [],
       close: async () => {}
     }
   }

server/core/src/server/aggregator.ts

@@ -99,31 +99,20 @@ export class AggregatorStorageAdapter implements StorageAdapter, StorageAdapterE
   find (ctx: MeasureContext, workspaceId: WorkspaceId): StorageIterator {
     const storageIterator = this.makeStorageIterator(ctx, workspaceId)
 
-    let buffer: ListBlobResult[] = []
-
     return {
-      next: async (ctx) => {
-        const docInfo = await storageIterator.next()
-        if (docInfo !== undefined) {
-          buffer.push(docInfo)
+      next: async () => {
+        const docInfos = await storageIterator.next()
+        if (docInfos.length > 0) {
+          await this.doSyncDocs(ctx, workspaceId, docInfos)
         }
-        if (buffer.length > 50) {
-          await this.doSyncDocs(ctx, workspaceId, buffer)
 
-          buffer = []
-        }
-        if (docInfo !== undefined) {
-          return {
-            hash: docInfo.etag,
-            id: docInfo._id,
-            size: docInfo.size
-          }
-        }
+        return docInfos.map((it) => ({
+          hash: it.etag,
+          id: it._id,
+          size: it.size
+        }))
       },
       close: async (ctx) => {
-        if (buffer.length > 0) {
-          await this.doSyncDocs(ctx, workspaceId, buffer)
-        }
         await storageIterator.close()
       }
     }
@@ -134,22 +123,21 @@ export class AggregatorStorageAdapter implements StorageAdapter, StorageAdapterE
     let iterator: BlobStorageIterator | undefined
     return {
       next: async () => {
-        while (true) {
-          if (iterator === undefined && adapters.length > 0) {
-            iterator = await (adapters.shift() as StorageAdapter).listStream(ctx, workspaceId)
-          }
-          if (iterator === undefined) {
-            return undefined
-          }
-          const docInfo = await iterator.next()
-          if (docInfo !== undefined) {
-            // We need to check if our stored version is fine
-            return docInfo
-          } else {
-            // We need to take next adapter
-            await iterator.close()
-            iterator = undefined
-          }
+        if (iterator === undefined && adapters.length > 0) {
+          iterator = await (adapters.shift() as StorageAdapter).listStream(ctx, workspaceId)
+        }
+        if (iterator === undefined) {
+          return []
+        }
+        const docInfos = await iterator.next()
+        if (docInfos.length > 0) {
+          // We need to check if our stored version is fine
+          return docInfos
+        } else {
+          // We need to take next adapter
+          await iterator.close()
+          iterator = undefined
+          return []
         }
       },
       close: async () => {
@@ -227,7 +215,7 @@ export class AggregatorStorageAdapter implements StorageAdapter, StorageAdapterE
   async listStream (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<BlobStorageIterator> {
     const data = await this.dbAdapter.findStream<Blob>(ctx, workspaceId, DOMAIN_BLOB, {})
     return {
-      next: async (): Promise<ListBlobResult | undefined> => {
+      next: async (): Promise<ListBlobResult[]> => {
         return await data.next()
       },
       close: async () => {

server/core/src/storage.ts

@@ -9,6 +9,7 @@ import {
   type StorageIterator,
   type WorkspaceId
 } from '@hcengineering/core'
+import { estimateDocSize } from './utils'
 
 export * from '@hcengineering/storage'
 
@@ -19,7 +20,7 @@ export function getBucketId (workspaceId: WorkspaceId): string {
   return toWorkspaceString(workspaceId)
 }
 
-const chunkSize = 2 * 1024 * 1024
+const chunkSize = 512 * 1024
 
 /**
  * @public
@@ -70,14 +71,15 @@ export class BackupClientOps {
       const docs: DocInfo[] = []
 
       while (size < chunkSize) {
-        const doc = await chunk.iterator.next(ctx)
-        if (doc === undefined) {
+        const _docs = await chunk.iterator.next(ctx)
+        if (_docs.length === 0) {
           chunk.finished = true
           break
         }
-
-        size += doc.size
-        docs.push(doc)
+        for (const doc of _docs) {
+          size += estimateDocSize(doc)
+          docs.push(doc)
+        }
       }
 
       return {