From 46448e3ae822dc8514097bd24928f4843bb76f4a Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 7 Jul 2025 15:03:33 +0700 Subject: [PATCH] QFIX: Fix reducedCalls to stop in case of error (#9482) Signed-off-by: Andrey Sobolev --- packages/core/src/utils.ts | 8 +- server/backup/src/service.ts | 132 ++++++++++++----------- services/github/pod-github/src/worker.ts | 6 +- 3 files changed, 79 insertions(+), 67 deletions(-) diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index cbd6d31fef..3c30bf772e 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -749,12 +749,16 @@ export function reduceCalls) => Promise< currentCall = nextCall nextCall = undefined if (currentCall !== undefined) { - void currentCall.op() + void currentCall.op().catch() } } return async function (...args: ReduceParameters): Promise { const myOp = async (): Promise => { - await operation(...args) + try { + await operation(...args) + } catch (err: any) { + console.error('Error occurred in reduceCalls:', err) + } next() } diff --git a/server/backup/src/service.ts b/server/backup/src/service.ts index 481ae1a210..b8de4127a2 100644 --- a/server/backup/src/service.ts +++ b/server/backup/src/service.ts @@ -85,75 +85,79 @@ class BackupWorker { } recheckWorkspaces = reduceCalls(async (ctx: MeasureContext) => { - const workspacesIgnore = new Set(this.config.SkipWorkspaces.split(';')) - const now = Date.now() - const allWorkspaces = await this.getWorkspacesList() + try { + const workspacesIgnore = new Set(this.config.SkipWorkspaces.split(';')) + const now = Date.now() + const allWorkspaces = await this.getWorkspacesList() - let skipped = 0 - const workspaces = allWorkspaces.filter((it) => { - if (this.workspacesToBackup.has(it.uuid) || this.activeWorkspaces.has(it.uuid)) { - // We already had ws in set - return false - } - if (!isActiveMode(it.mode)) { - // We should backup only active workspaces - skipped++ - return false + let skipped = 0 + const workspaces = allWorkspaces.filter((it) => { + if (this.workspacesToBackup.has(it.uuid) || this.activeWorkspaces.has(it.uuid)) { + // We already had ws in set + return false + } + if (!isActiveMode(it.mode)) { + // We should backup only active workspaces + skipped++ + return false + } + + const createdOn = Math.floor((now - it.createdOn) / 1000) + if (createdOn <= 2) { + // Skip if we created is less 2 days + return false + } + + const lastBackup = it.backupInfo?.lastBackup ?? 0 + if ((now - lastBackup) / 1000 < this.config.Interval && this.config.Interval !== 0) { + // No backup required, interval not elapsed + skipped++ + return false + } + + if (it.lastVisit == null) { + skipped++ + return false + } + + const lastVisitSec = Math.floor((now - it.lastVisit) / 1000) + if (lastVisitSec > this.config.Interval) { + // No backup required, interval not elapsed + skipped++ + return false + } + return !workspacesIgnore.has(it.uuid) + }) + + workspaces.sort((a, b) => { + return (a.backupInfo?.lastBackup ?? 0) - (b.backupInfo?.lastBackup ?? 0) + }) + + // Shift new with existing ones. + const existingNew = groupByArray(workspaces, (it) => it.backupInfo != null) + + const existing = existingNew.get(true) ?? [] + const newOnes = existingNew.get(false) ?? [] + const mixedBackupSorting: WorkspaceInfoWithStatus[] = [] + + while (existing.length > 0 || newOnes.length > 0) { + const e = existing.shift() + const n = newOnes.shift() + if (e != null) { + mixedBackupSorting.push(e) + } + if (n != null) { + mixedBackupSorting.push(n) + } } - const createdOn = Math.floor((now - it.createdOn) / 1000) - if (createdOn <= 2) { - // Skip if we created is less 2 days - return false - } - - const lastBackup = it.backupInfo?.lastBackup ?? 0 - if ((now - lastBackup) / 1000 < this.config.Interval && this.config.Interval !== 0) { - // No backup required, interval not elapsed - skipped++ - return false - } - - if (it.lastVisit == null) { - skipped++ - return false - } - - const lastVisitSec = Math.floor((now - it.lastVisit) / 1000) - if (lastVisitSec > this.config.Interval) { - // No backup required, interval not elapsed - skipped++ - return false - } - return !workspacesIgnore.has(it.uuid) - }) - - workspaces.sort((a, b) => { - return (a.backupInfo?.lastBackup ?? 0) - (b.backupInfo?.lastBackup ?? 0) - }) - - // Shift new with existing ones. - const existingNew = groupByArray(workspaces, (it) => it.backupInfo != null) - - const existing = existingNew.get(true) ?? [] - const newOnes = existingNew.get(false) ?? [] - const mixedBackupSorting: WorkspaceInfoWithStatus[] = [] - - while (existing.length > 0 || newOnes.length > 0) { - const e = existing.shift() - const n = newOnes.shift() - if (e != null) { - mixedBackupSorting.push(e) - } - if (n != null) { - mixedBackupSorting.push(n) + for (const ws of mixedBackupSorting) { + this.workspacesToBackup.set(ws.uuid, ws) } + ctx.info('skipped workspaces', { skipped, workspaces: this.workspacesToBackup.size, workspacesIgnore }) + } catch (err: any) { + ctx.error('Error in recheckWorkspaces', { error: err }) } - - for (const ws of mixedBackupSorting) { - this.workspacesToBackup.set(ws.uuid, ws) - } - ctx.info('skipped workspaces', { skipped, workspaces: this.workspacesToBackup.size, workspacesIgnore }) }) async schedule (ctx: MeasureContext): Promise { diff --git a/services/github/pod-github/src/worker.ts b/services/github/pod-github/src/worker.ts index cf9356be24..d38d28ac3b 100644 --- a/services/github/pod-github/src/worker.ts +++ b/services/github/pod-github/src/worker.ts @@ -1551,7 +1551,11 @@ export class GithubWorker implements IntegrationManager { } performFullSync = reduceCalls(async () => { - await this._performFullSync() + try { + await this._performFullSync() + } catch (err: any) { + this.ctx.error('Failed to perform full sync', { error: err }) + } }) async _performFullSync (): Promise {