UBERF-8619: Rework backup %hash% usage (#7273)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

server/core/src/stats.ts

@@ -101,6 +101,15 @@ export function initStatisticsContext (
     let oldMetricsValue = ''
     const serviceId = encodeURIComponent(os.hostname() + '-' + serviceName)
 
+    const handleError = (err: any): void => {
+      errorToSend++
+      if (errorToSend % 2 === 0) {
+        if (err.code !== 'UND_ERR_SOCKET') {
+          console.error(err)
+        }
+      }
+    }
+
     const intTimer = setInterval(() => {
       try {
         if (metricsFile !== undefined || ops?.logConsole === true) {
@@ -138,18 +147,10 @@ export function initStatisticsContext (
               },
               body: statData
             }
-          ).catch((err) => {
-            errorToSend++
-            if (errorToSend % 2 === 0) {
-              console.error(err)
-            }
-          })
+          ).catch(handleError)
         }
       } catch (err: any) {
-        errorToSend++
-        if (errorToSend % 20 === 0) {
-          console.error(err)
-        }
+        handleError(err)
       }
     }, METRICS_UPDATE_INTERVAL)
 

server/core/src/storage.ts

@@ -9,7 +9,6 @@ import {
   type StorageIterator,
   type WorkspaceId
 } from '@hcengineering/core'
-import { estimateDocSize } from './utils'
 
 export * from '@hcengineering/storage'
 
@@ -20,7 +19,7 @@ export function getBucketId (workspaceId: WorkspaceId): string {
   return toWorkspaceString(workspaceId)
 }
 
-const chunkSize = 512 * 1024
+const chunkSize = 200
 
 /**
  * @public
@@ -44,8 +43,7 @@ export class BackupClientOps {
   loadChunk (
     ctx: MeasureContext,
     domain: Domain,
-    idx?: number,
-    recheck?: boolean
+    idx?: number
   ): Promise<{
       idx: number
       docs: DocInfo[]
@@ -64,22 +62,18 @@ export class BackupClientOps {
           }
         }
       } else {
-        chunk = { idx, iterator: this.storage.find(ctx, domain, recheck), finished: false, index: 0 }
+        chunk = { idx, iterator: this.storage.find(ctx, domain), finished: false, index: 0 }
         this.chunkInfo.set(idx, chunk)
       }
-      let size = 0
       const docs: DocInfo[] = []
 
-      while (size < chunkSize) {
+      while (docs.length < chunkSize) {
         const _docs = await chunk.iterator.next(ctx)
         if (_docs.length === 0) {
           chunk.finished = true
           break
         }
-        for (const doc of _docs) {
-          size += estimateDocSize(doc)
-          docs.push(doc)
-        }
+        docs.push(..._docs)
       }
 
       return {

server/core/src/utils.ts

@@ -12,7 +12,6 @@ import core, {
   type Class,
   type Client,
   type Doc,
-  type DocInfo,
   type MeasureContext,
   type ModelDb,
   type Ref,
@@ -21,7 +20,7 @@ import core, {
   type WorkspaceIdWithUrl
 } from '@hcengineering/core'
 import platform, { PlatformError, Severity, Status, unknownError } from '@hcengineering/platform'
-import { createHash, type Hash } from 'crypto'
+import { type Hash } from 'crypto'
 import fs from 'fs'
 import { BackupClientOps } from './storage'
 import type { Pipeline } from './types'
@@ -86,7 +85,7 @@ export function estimateDocSize (_obj: any): number {
   return result
 }
 /**
- * Return some estimation for object size
+ * Calculate hash for object
  */
 export function updateHashForDoc (hash: Hash, _obj: any): void {
   const toProcess = [_obj]
@@ -98,7 +97,9 @@ export function updateHashForDoc (hash: Hash, _obj: any): void {
     if (typeof obj === 'function') {
       continue
     }
-    for (const key in obj) {
+    const keys = Object.keys(obj).sort()
+    // We need sorted list of keys to make it consistent
+    for (const key of keys) {
       // include prototype properties
       const value = obj[key]
       const type = getTypeOf(value)
@@ -220,38 +221,6 @@ export function loadBrandingMap (brandingPath?: string): BrandingMap {
   return brandings
 }
 
-export function toDocInfo (d: Doc, bulkUpdate: Map<Ref<Doc>, string>, recheck?: boolean): DocInfo {
-  let digest: string | null = (d as any)['%hash%']
-  if ('%hash%' in d) {
-    delete d['%hash%']
-  }
-  const pos = (digest ?? '').indexOf('|')
-  const oldDigest = digest
-  if (digest == null || digest === '' || recheck === true) {
-    const size = estimateDocSize(d)
-
-    const hash = createHash('sha256')
-    updateHashForDoc(hash, d)
-    digest = hash.digest('base64')
-    const newDigest = `${digest}|${size.toString(16)}`
-
-    if (recheck !== true || oldDigest !== newDigest) {
-      bulkUpdate.set(d._id, `${digest}|${size.toString(16)}`)
-    }
-    return {
-      id: d._id,
-      hash: digest,
-      size
-    }
-  } else {
-    return {
-      id: d._id,
-      hash: pos >= 0 ? digest.slice(0, pos) : digest,
-      size: pos >= 0 ? parseInt(digest.slice(pos + 1), 16) : 0
-    }
-  }
-}
-
 export function wrapPipeline (
   ctx: MeasureContext,
   pipeline: Pipeline,
@@ -284,7 +253,7 @@ export function wrapPipeline (
     closeChunk: (idx) => backupOps.closeChunk(ctx, idx),
     getHierarchy: () => pipeline.context.hierarchy,
     getModel: () => pipeline.context.modelDb,
-    loadChunk: (domain, idx, recheck) => backupOps.loadChunk(ctx, domain, idx, recheck),
+    loadChunk: (domain, idx) => backupOps.loadChunk(ctx, domain, idx),
     loadDocs: (domain, docs) => backupOps.loadDocs(ctx, domain, docs),
     upload: (domain, docs) => backupOps.upload(ctx, domain, docs),
     searchFulltext: async (query, options) => ({ docs: [], total: 0 }),