UBERF-8122: Fix backup service

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
2026-09-13 21:27:46 +02:00 · 2024-09-17 02:25:31 +07:00
parent c3a41ea1bb
commit b06f433bab
20 changed files with 287 additions and 284 deletions
@@ -14,12 +14,13 @@
    "_phase:bundle": "rushx bundle",
    "_phase:docker-build": "rushx docker:build",
    "_phase:docker-staging": "rushx docker:staging",
-    "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --minify --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) > bundle/bundle.js",
+    "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --keep-names --sourcemap=external --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --log-level=error --outfile=bundle/bundle.js",
    "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/tool",
    "docker:tbuild": "docker build -t hardcoreeng/tool . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool",
    "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging",
    "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool",
    "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --max-old-space-size=18000 ./bundle/bundle.js",
+    "run-local-brk": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js",
    "run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js",
    "upgrade": "rushx run-local upgrade",
    "format": "format src",
@@ -104,12 +104,15 @@ export async function cleanWorkspace (
    const minioList = await storageAdapter.listStream(ctx, workspaceId)
    const toClean: string[] = []
    while (true) {
-      const mv = await minioList.next()
-      if (mv === undefined) {
+      const mvFiles = await minioList.next()
+      if (mvFiles.length === 0) {
        break
      }
-      if (!files.has(mv._id)) {
-        toClean.push(mv._id)
+
+      for (const mv of mvFiles) {
+        if (!files.has(mv._id)) {
+          toClean.push(mv._id)
+        }
      }
    }
    await storageAdapter.remove(ctx, workspaceId, toClean)
@@ -192,16 +195,18 @@ export async function fixMinioBW (
  const list = await storageService.listStream(ctx, workspaceId)
  let removed = 0
  while (true) {
-    const obj = await list.next()
-    if (obj === undefined) {
+    const objs = await list.next()
+    if (objs.length === 0) {
      break
    }
-    if (obj.modifiedOn < from) continue
-    if ((obj._id as string).includes('%preview%')) {
-      await storageService.remove(ctx, workspaceId, [obj._id])
-      removed++
-      if (removed % 100 === 0) {
-        console.log('removed: ', removed)
+    for (const obj of objs) {
+      if (obj.modifiedOn < from) continue
+      if ((obj._id as string).includes('%preview%')) {
+        await storageService.remove(ctx, workspaceId, [obj._id])
+        removed++
+        if (removed % 100 === 0) {
+          console.log('removed: ', removed)
+        }
      }
    }
  }
@@ -40,20 +40,22 @@ export async function syncFiles (
      const iterator = await adapter.listStream(ctx, workspaceId)
      try {
        while (true) {
-          const data = await iterator.next()
-          if (data === undefined) break
+          const dataBulk = await iterator.next()
+          if (dataBulk.length === 0) break

-          const blob = await exAdapter.stat(ctx, workspaceId, data._id)
-          if (blob !== undefined) continue
+          for (const data of dataBulk) {
+            const blob = await exAdapter.stat(ctx, workspaceId, data._id)
+            if (blob !== undefined) continue

-          await exAdapter.syncBlobFromStorage(ctx, workspaceId, data._id, name)
+            await exAdapter.syncBlobFromStorage(ctx, workspaceId, data._id, name)

-          count += 1
-          if (count % 100 === 0) {
-            const duration = Date.now() - time
-            time = Date.now()
+            count += 1
+            if (count % 100 === 0) {
+              const duration = Date.now() - time
+              time = Date.now()

-            console.log('...processed', count, Math.round(duration / 1000) + 's')
+              console.log('...processed', count, Math.round(duration / 1000) + 's')
+            }
          }
        }
        console.log('processed', count)
@@ -112,64 +114,67 @@ async function processAdapter (
  const iterator = await source.listStream(ctx, workspaceId)
  try {
    while (true) {
-      const data = await iterator.next()
-      if (data === undefined) break
+      const dataBulk = await iterator.next()
+      if (dataBulk.length === 0) break

-      const blob = (await exAdapter.stat(ctx, workspaceId, data._id)) ?? (await source.stat(ctx, workspaceId, data._id))
+      for (const data of dataBulk) {
+        const blob =
+          (await exAdapter.stat(ctx, workspaceId, data._id)) ?? (await source.stat(ctx, workspaceId, data._id))

-      if (blob === undefined) {
-        console.error('blob not found', data._id)
-        continue
-      }
-
-      if (blob.provider !== exAdapter.defaultAdapter) {
-        if (blob.size <= params.blobSizeLimitMb * 1024 * 1024) {
-          await rateLimiter.exec(async () => {
-            try {
-              await retryOnFailure(
-                ctx,
-                5,
-                async () => {
-                  await processFile(ctx, source, params.move ? exAdapter : target, workspaceId, blob)
-                },
-                50
-              )
-              movedCnt += 1
-              movedBytes += blob.size
-              batchBytes += blob.size
-            } catch (err) {
-              console.error('failed to process blob', data._id, err)
-            }
-          })
-        } else {
-          skippedCnt += 1
-          console.log('skipping large blob', data._id, Math.round(blob.size / 1024 / 1024))
+        if (blob === undefined) {
+          console.error('blob not found', data._id)
+          continue
        }
-      }

-      processedCnt += 1
-      processedBytes += blob.size
+        if (blob.provider !== exAdapter.defaultAdapter) {
+          if (blob.size <= params.blobSizeLimitMb * 1024 * 1024) {
+            await rateLimiter.exec(async () => {
+              try {
+                await retryOnFailure(
+                  ctx,
+                  5,
+                  async () => {
+                    await processFile(ctx, source, params.move ? exAdapter : target, workspaceId, blob)
+                  },
+                  50
+                )
+                movedCnt += 1
+                movedBytes += blob.size
+                batchBytes += blob.size
+              } catch (err) {
+                console.error('failed to process blob', data._id, err)
+              }
+            })
+          } else {
+            skippedCnt += 1
+            console.log('skipping large blob', data._id, Math.round(blob.size / 1024 / 1024))
+          }
+        }

-      if (processedCnt % 100 === 0) {
-        await rateLimiter.waitProcessing()
+        processedCnt += 1
+        processedBytes += blob.size

-        const duration = Date.now() - time
+        if (processedCnt % 100 === 0) {
+          await rateLimiter.waitProcessing()

-        console.log(
-          '...processed',
-          processedCnt,
-          Math.round(processedBytes / 1024 / 1024) + 'MB',
-          'moved',
-          movedCnt,
-          Math.round(movedBytes / 1024 / 1024) + 'MB',
-          '+' + Math.round(batchBytes / 1024 / 1024) + 'MB',
-          'skipped',
-          skippedCnt,
-          Math.round(duration / 1000) + 's'
-        )
+          const duration = Date.now() - time

-        batchBytes = 0
-        time = Date.now()
+          console.log(
+            '...processed',
+            processedCnt,
+            Math.round(processedBytes / 1024 / 1024) + 'MB',
+            'moved',
+            movedCnt,
+            Math.round(movedBytes / 1024 / 1024) + 'MB',
+            '+' + Math.round(batchBytes / 1024 / 1024) + 'MB',
+            'skipped',
+            skippedCnt,
+            Math.round(duration / 1000) + 's'
+          )
+
+          batchBytes = 0
+          time = Date.now()
+        }
      }
    }