From c36dd74fef9801090082dceb738755b47a63be8a Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Tue, 21 Jul 2026 20:51:02 +0500 Subject: [PATCH 1/7] Add AllMatchValue function and related updates across multiple files (#10990) Signed-off-by: Denis Bykhov --- models/process/src/functions.ts | 13 ++++++++++++ models/server-process/src/index.ts | 4 ++++ plugins/process-assets/lang/cs.json | 3 ++- plugins/process-assets/lang/de.json | 1 + plugins/process-assets/lang/en.json | 1 + plugins/process-assets/lang/es.json | 1 + plugins/process-assets/lang/fr.json | 1 + plugins/process-assets/lang/it.json | 1 + plugins/process-assets/lang/ja.json | 1 + plugins/process-assets/lang/ko.json | 1 + plugins/process-assets/lang/pl.json | 1 + plugins/process-assets/lang/pt-br.json | 1 + plugins/process-assets/lang/pt.json | 1 + plugins/process-assets/lang/ru.json | 1 + plugins/process-assets/lang/tr.json | 1 + plugins/process-assets/lang/zh.json | 1 + plugins/process-resources/src/plugin.ts | 1 + plugins/process/src/index.ts | 1 + server-plugins/process-resources/src/index.ts | 2 ++ .../process-resources/src/transform.ts | 20 +++++++++++++++++++ server-plugins/process/src/index.ts | 1 + 21 files changed, 57 insertions(+), 1 deletion(-) diff --git a/models/process/src/functions.ts b/models/process/src/functions.ts index 11d982c12c..2ed2985176 100644 --- a/models/process/src/functions.ts +++ b/models/process/src/functions.ts @@ -185,6 +185,19 @@ export function defineFunctions (builder: Builder): void { process.function.All ) + builder.createDoc( + process.class.ProcessFunction, + core.space.Model, + { + of: core.class.ArrOf, + category: 'array', + label: process.string.AllMatchValue, + type: 'reduce', + editor: process.transformEditor.FilterEditor + }, + process.function.AllMatchValue + ) + builder.createDoc( process.class.ProcessFunction, core.space.Model, diff --git a/models/server-process/src/index.ts b/models/server-process/src/index.ts index a1179e8160..eb040fa7c1 100644 --- a/models/server-process/src/index.ts +++ b/models/server-process/src/index.ts @@ -388,6 +388,10 @@ export function createModel (builder: Builder): void { func: serverProcess.transform.FirstMatchValue }) + builder.mixin(process.function.AllMatchValue, process.class.ProcessFunction, serverProcess.mixin.FuncImpl, { + func: serverProcess.transform.AllMatchValue + }) + builder.mixin(process.function.Filter, process.class.ProcessFunction, serverProcess.mixin.FuncImpl, { func: serverProcess.transform.Filter }) diff --git a/plugins/process-assets/lang/cs.json b/plugins/process-assets/lang/cs.json index 33c159f0d3..715f5be61d 100644 --- a/plugins/process-assets/lang/cs.json +++ b/plugins/process-assets/lang/cs.json @@ -117,6 +117,7 @@ "ExecutionInitiator": "Iniciátor provedení", "ExecutionStarted": "Provedení spuštěno", "Filter": "Filtr", + "AllMatchValue": "Všechny odpovídající hodnoty", "FirstMatchValue": "První odpovídající hodnota", "ConfigLabel": "Konfigurace procesů", "ConfigDescription": "Definujte procesy pro automatizaci pracovních postupů a obchodních procesů.", @@ -197,4 +198,4 @@ "TooDeepTransitionRecursion": "Příliš hluboká rekurze přechodů", "ToDoAlreadyCompleted": "Akční položka již byla dokončena" } -} \ No newline at end of file +} diff --git a/plugins/process-assets/lang/de.json b/plugins/process-assets/lang/de.json index 33f6d9512c..d7c4ba82e3 100644 --- a/plugins/process-assets/lang/de.json +++ b/plugins/process-assets/lang/de.json @@ -117,6 +117,7 @@ "ExecutionInitiator": "Ausführungsinitiator", "ExecutionStarted": "Ausführung gestartet", "Filter": "Filter", + "AllMatchValue": "Alle passenden Werte", "FirstMatchValue": "Erster passender Wert", "ConfigLabel": "Prozesskonfiguration", "ConfigDescription": "Definieren Sie Prozesse, um Workflows und Geschäftsprozesse zu automatisieren.", diff --git a/plugins/process-assets/lang/en.json b/plugins/process-assets/lang/en.json index bccf7c8eeb..4c22b97054 100644 --- a/plugins/process-assets/lang/en.json +++ b/plugins/process-assets/lang/en.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Execution initiator", "ExecutionStarted": "Execution started", "Filter": "Filter", + "AllMatchValue": "All matching values", "FirstMatchValue": "First match value", "ConfigLabel": "Process configuration", "ConfigDescription": "Define processes to automate workflows and business processes.", diff --git a/plugins/process-assets/lang/es.json b/plugins/process-assets/lang/es.json index 1e378ef541..efc1eb50b0 100644 --- a/plugins/process-assets/lang/es.json +++ b/plugins/process-assets/lang/es.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Iniciador de Ejecución", "ExecutionStarted": "Ejecución Iniciada", "Filter": "Filtro", + "AllMatchValue": "Todos los valores coincidentes", "FirstMatchValue": "Primer valor coincidente", "ConfigLabel": "Configuración de procesos", "ConfigDescription": "Defina procesos para automatizar flujos de trabajo y procesos comerciales.", diff --git a/plugins/process-assets/lang/fr.json b/plugins/process-assets/lang/fr.json index 23ca3bacd7..54c238bdfb 100644 --- a/plugins/process-assets/lang/fr.json +++ b/plugins/process-assets/lang/fr.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Initiateur d'exécution", "ExecutionStarted": "Exécution démarrée", "Filter": "Filtrer", + "AllMatchValue": "Toutes les valeurs correspondantes", "FirstMatchValue": "Première valeur correspondante", "ConfigLabel": "Configuration des processus", "ConfigDescription": "Définissez des processus pour automatiser les flux de travail et les processus métier.", diff --git a/plugins/process-assets/lang/it.json b/plugins/process-assets/lang/it.json index ba760e1993..065536fa15 100644 --- a/plugins/process-assets/lang/it.json +++ b/plugins/process-assets/lang/it.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Iniziatore dell'esecuzione", "ExecutionStarted": "Esecuzione avviata", "Filter": "Filtro", + "AllMatchValue": "Tutti i valori corrispondenti", "FirstMatchValue": "Primo valore corrispondente", "ConfigLabel": "Configurazione del processo", "ConfigDescription": "Definire i processi per automatizzare i flussi di lavoro e i processi aziendali.", diff --git a/plugins/process-assets/lang/ja.json b/plugins/process-assets/lang/ja.json index 5764530c01..f73657ee90 100644 --- a/plugins/process-assets/lang/ja.json +++ b/plugins/process-assets/lang/ja.json @@ -123,6 +123,7 @@ "ExecutionInitiator": "実行の発起人", "ExecutionStarted": "実行が開始されました", "Filter": "フィルター", + "AllMatchValue": "一致するすべての値", "FirstMatchValue": "最初の一致する値", "ConfigLabel": "プロセス構成", "ConfigDescription": "ワークフローやビジネスプロセスを自動化するためのプロセスを定義します。", diff --git a/plugins/process-assets/lang/ko.json b/plugins/process-assets/lang/ko.json index b85b181871..c5143204b3 100644 --- a/plugins/process-assets/lang/ko.json +++ b/plugins/process-assets/lang/ko.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "실행 시작자", "ExecutionStarted": "실행 시작됨", "Filter": "필터", + "AllMatchValue": "모든 일치 값", "FirstMatchValue": "첫 일치 값", "ConfigLabel": "프로세스 구성", "ConfigDescription": "워크플로와 비즈니스 프로세스를 자동화할 프로세스를 정의합니다.", diff --git a/plugins/process-assets/lang/pl.json b/plugins/process-assets/lang/pl.json index 1c2916fd17..50a71e204d 100644 --- a/plugins/process-assets/lang/pl.json +++ b/plugins/process-assets/lang/pl.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Inicjator wykonania", "ExecutionStarted": "Wykonanie rozpoczęte", "Filter": "Filtr", + "AllMatchValue": "Wszystkie pasujące wartości", "FirstMatchValue": "Pierwsza pasująca wartość", "ConfigLabel": "Konfiguracja procesu", "ConfigDescription": "Definiuj procesy do automatyzacji organizacji zadań i procesów biznesowych.", diff --git a/plugins/process-assets/lang/pt-br.json b/plugins/process-assets/lang/pt-br.json index 9f50306818..470219d7c2 100644 --- a/plugins/process-assets/lang/pt-br.json +++ b/plugins/process-assets/lang/pt-br.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Iniciador de Execução", "ExecutionStarted": "Execução Iniciada", "Filter": "Filtrar", + "AllMatchValue": "Todos os valores correspondentes", "FirstMatchValue": "Primeiro Valor Correspondente", "ConfigLabel": "Configuração de processos", "ConfigDescription": "Defina processos para automatizar fluxos de trabalho e processos de negócios.", diff --git a/plugins/process-assets/lang/pt.json b/plugins/process-assets/lang/pt.json index de5c2dd6c2..f660e43e63 100644 --- a/plugins/process-assets/lang/pt.json +++ b/plugins/process-assets/lang/pt.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Iniciador de Execução", "ExecutionStarted": "Execução Iniciada", "Filter": "Filtrar", + "AllMatchValue": "Todos os valores correspondentes", "FirstMatchValue": "Primeiro Valor Correspondente", "ConfigLabel": "Configuração de processos", "ConfigDescription": "Defina processos para automatizar fluxos de trabalho e processos de negócios.", diff --git a/plugins/process-assets/lang/ru.json b/plugins/process-assets/lang/ru.json index 446fca4f13..1a80cf32ce 100644 --- a/plugins/process-assets/lang/ru.json +++ b/plugins/process-assets/lang/ru.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "Инициатор выполнения", "ExecutionStarted": "Выполнение начато", "Filter": "Фильтр", + "AllMatchValue": "Все подходящие значения", "FirstMatchValue": "Первое подходящее значение", "ConfigLabel": "Конфигурация процессов", "ConfigDescription": "Определяйте процессы для автоматизации рабочих процессов и бизнес-процессов.", diff --git a/plugins/process-assets/lang/tr.json b/plugins/process-assets/lang/tr.json index 437e325a76..bc2181f246 100644 --- a/plugins/process-assets/lang/tr.json +++ b/plugins/process-assets/lang/tr.json @@ -121,6 +121,7 @@ "ProcessFinished": "\"{process}\" süreci \"{state}\" durumunda tamamlandı", "NewProcessToDo": "Yeni süreç Eylem öğesi", "Filter": "Filtre", + "AllMatchValue": "Tüm eşleşen değerler", "FirstMatchValue": "İlk eşleşen değer", "ConfigLabel": "Süreç yapılandırması", "ConfigDescription": "İş akışlarını ve iş süreçlerini otomatikleştirmek için süreçler tanımlayın.", diff --git a/plugins/process-assets/lang/zh.json b/plugins/process-assets/lang/zh.json index 9c245cbbc8..3d6faf0406 100644 --- a/plugins/process-assets/lang/zh.json +++ b/plugins/process-assets/lang/zh.json @@ -124,6 +124,7 @@ "ExecutionInitiator": "执行发起者", "ExecutionStarted": "执行已启动", "Filter": "过滤", + "AllMatchValue": "所有匹配值", "FirstMatchValue": "第一个匹配值", "ConfigLabel": "流程配置", "ConfigDescription": "定义流程以自动化工作流和业务流程。", diff --git a/plugins/process-resources/src/plugin.ts b/plugins/process-resources/src/plugin.ts index 70d4c32262..d512dcc172 100644 --- a/plugins/process-resources/src/plugin.ts +++ b/plugins/process-resources/src/plugin.ts @@ -157,6 +157,7 @@ export default mergeIds(processId, process, { Split: '' as IntlString, Cut: '' as IntlString, Filter: '' as IntlString, + AllMatchValue: '' as IntlString, FirstMatchValue: '' as IntlString, FirstValue: '' as IntlString, LastValue: '' as IntlString, diff --git a/plugins/process/src/index.ts b/plugins/process/src/index.ts index 52015fef0a..9568ea8602 100644 --- a/plugins/process/src/index.ts +++ b/plugins/process/src/index.ts @@ -353,6 +353,7 @@ export default plugin(processId, { OnEvent: '' as Asset }, function: { + AllMatchValue: '' as Ref, FirstMatchValue: '' as Ref, Filter: '' as Ref, FirstValue: '' as Ref, diff --git a/server-plugins/process-resources/src/index.ts b/server-plugins/process-resources/src/index.ts index 2f3ca49fbd..4791dec4d0 100644 --- a/server-plugins/process-resources/src/index.ts +++ b/server-plugins/process-resources/src/index.ts @@ -85,6 +85,7 @@ import { FieldChangedRollback, ToDoCancellRollback, ToDoCloseRollback } from './ import { Absolute, Add, + AllMatchValue, All, Append, Ceil, @@ -819,6 +820,7 @@ export default async () => ({ EmptyValue, ExecutionInitiator, ExecutionStarted, + AllMatchValue, FirstMatchValue, Filter, StringFromNumber, diff --git a/server-plugins/process-resources/src/transform.ts b/server-plugins/process-resources/src/transform.ts index 66e412efce..d863c7b53c 100644 --- a/server-plugins/process-resources/src/transform.ts +++ b/server-plugins/process-resources/src/transform.ts @@ -63,6 +63,26 @@ export async function FirstMatchValue ( } } +export async function AllMatchValue ( + value: any[], + props: Record, + control: ProcessControl +): Promise { + if (value == null) { + return + } + if (!Array.isArray(value)) return value + const { _class, ...otherProps } = props + if (_class == null) return + if (value.length === 0) return + if (typeof value[0] === 'string') { + const docs = await control.client.findAll(_class, { _id: { $in: value } }) + return matchQuery(docs, otherProps, core.class.Doc, control.client.getHierarchy(), true).map((p) => p._id) + } else if (typeof value[0] === 'object') { + return matchQuery(value, otherProps, core.class.Doc, control.client.getHierarchy(), true) + } +} + // #endregion // #region Array diff --git a/server-plugins/process/src/index.ts b/server-plugins/process/src/index.ts index b3198c791c..d5b4f8c25b 100644 --- a/server-plugins/process/src/index.ts +++ b/server-plugins/process/src/index.ts @@ -115,6 +115,7 @@ export default plugin(serverProcessId, { EmptyValue: '' as Resource, EmptyArray: '' as Resource, Filter: '' as Resource, + AllMatchValue: '' as Resource, FirstMatchValue: '' as Resource, ExecutionInitiator: '' as Resource, ExecutionStarted: '' as Resource, From e34f546b8c9fde7ae0bc6cc10f0504fa7ac9d5f9 Mon Sep 17 00:00:00 2001 From: Why_So_Serious <91526214+kimnamwook1@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:11:38 +0900 Subject: [PATCH 2/7] fix(github): restore the recovery path for revoked user OAuth tokens (#10995) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(github): honour the force flag in checkRefreshToken `checkRefreshToken` accepts a `force` parameter but never reads it, so the only caller that passes `force = true` (worker.ts, when re-syncing a user) behaves exactly like the non-forced call and skips the refresh entirely. Gate the refresh on `force || expired` instead of on expiry alone. As a side effect a record with a `refreshToken` but a null `expiresIn` is now validated when forced, instead of being reported as valid unchecked. Signed-off-by: koreanjoker * fix(github): return undefined from getOctokit when the token is dead When `checkRefreshToken` reports failure, `getOctokit` cleared `record.octokit` and then fell straight through to constructing a new Octokit from the very token that was just rejected. The method therefore never returned `undefined` for a revoked user, so the `(await getOctokit(...)) ?? container.container.octokit` installation-token fallback that the sync code already writes at 13 call sites was unreachable. Return `undefined` after clearing the client so the existing fallback can take effect. Signed-off-by: koreanjoker * fix(github): preserve accounts when deserialising a user secret `updateUser` serialises the whole `GithubUserRecord` — `accounts` included — into the integration secret, but `secretToUserRecord` placed a literal `accounts: {}` after the spread of the parsed payload, discarding whatever was stored. Every consumer of a record loaded through `getAccount` therefore saw an empty map. `revokeUserAuth` iterates `Object.entries(record.accounts)`, so its body never ran and the re-authorisation notice was never written to any workspace. Read `accounts` back from the parsed payload, keeping `{}` as the fallback for records written before the field existed. Signed-off-by: koreanjoker --------- Signed-off-by: koreanjoker --- services/github/pod-github/src/platform.ts | 3 ++- services/github/pod-github/src/users.ts | 5 +++-- services/github/pod-github/src/worker.ts | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/services/github/pod-github/src/platform.ts b/services/github/pod-github/src/platform.ts index edf1929209..97455c5eb0 100644 --- a/services/github/pod-github/src/platform.ts +++ b/services/github/pod-github/src/platform.ts @@ -738,7 +738,8 @@ export class PlatformWorker { } async checkRefreshToken (ctx: MeasureContext, auth: GithubUserRecord, force: boolean = false): Promise { - if (auth.refreshToken != null && auth.expiresIn != null && auth.expiresIn < Date.now() / 1000) { + const expired = auth.expiresIn != null && auth.expiresIn < Date.now() / 1000 + if (auth.refreshToken != null && (force || expired)) { const uri = 'https://github.com/login/oauth/access_token?' + makeQuery({ diff --git a/services/github/pod-github/src/users.ts b/services/github/pod-github/src/users.ts index 3771cd52be..ccb12372cd 100644 --- a/services/github/pod-github/src/users.ts +++ b/services/github/pod-github/src/users.ts @@ -36,11 +36,12 @@ export class UserManager { } private secretToUserRecord (secret: IntegrationSecret, login: string): GithubUserRecord | undefined { + const parsed = JSON.parse(secret.secret) ?? {} // TODO: Add security return { - ...(JSON.parse(secret.secret) ?? {}), // TODO: Add security + ...parsed, account: secret.socialId, _id: login, - accounts: {} + accounts: parsed.accounts ?? {} } } diff --git a/services/github/pod-github/src/worker.ts b/services/github/pod-github/src/worker.ts index 244de40828..530f0d35b0 100644 --- a/services/github/pod-github/src/worker.ts +++ b/services/github/pod-github/src/worker.ts @@ -643,6 +643,7 @@ export class GithubWorker implements IntegrationManager { ctx.info('get octokit', { account, recordId: record._id, workspace: this.workspace.uuid }) if (!(await this.platform.checkRefreshToken(ctx, record))) { record.octokit = undefined + return undefined } if (record.octokit !== undefined) { return record.octokit From afe11faec1b56f278c0882355a69b08cff1eebd9 Mon Sep 17 00:00:00 2001 From: Don Kendall Date: Mon, 27 Jul 2026 00:46:18 -0400 Subject: [PATCH 3/7] fix(account): authorize mergeSpecifiedPersons / canMergeSpecifiedPersons (#10997) * fix(account): authorize mergeSpecifiedPersons / canMergeSpecifiedPersons Both operations decoded the caller token and discarded the result, so any authenticated caller could merge any two persons by uuid. Merging re-points the secondary person's social ids onto the primary one, and neither login nor password recovery require a social id to be verified, so this reached as far as taking over an arbitrary account. Authorize both behind verifyMergePersonsAuthority: - global admin tokens and the tool/workspace services pass, matching the account level mergeSpecifiedAccounts; - everybody else must maintain the workspace their token carries, and both persons must be within its reach: a person holding an account elsewhere, and the platform wide system and guest accounts, are refused; - a login capable social id may not move onto an account the caller does not own. doMergePersons only refuses verified secondary social ids, which leaves the unverified ones a maintainer could mint for themselves. canMergeSpecifiedPersons answers false instead of throwing: it is the predicate the merge dialog polls, and it awaits it without a catch. Claude-Session: https://claude.ai/code/session_01ANdoXbdn5k2hZy734EwKe7 Signed-off-by: Don Kendall * test(account): cover merge persons authorization Nineteen cases over both operations: the reported attack shape, the maintainer to owner escalation through an unverified email, foreign accounts on either side of the merge, the platform guest account, and the paths that must keep working (workspace contacts, members, tool service and admin tokens). Removing the authorization check fails eight of them. Claude-Session: https://claude.ai/code/session_01ANdoXbdn5k2hZy734EwKe7 Signed-off-by: Don Kendall --------- Signed-off-by: Don Kendall --- .../account/src/__tests__/operations.test.ts | 288 +++++++++++++++++- server/account/src/operations.ts | 93 +++++- 2 files changed, 377 insertions(+), 4 deletions(-) diff --git a/server/account/src/__tests__/operations.test.ts b/server/account/src/__tests__/operations.test.ts index 686f9c6e33..55ad76337e 100644 --- a/server/account/src/__tests__/operations.test.ts +++ b/server/account/src/__tests__/operations.test.ts @@ -53,7 +53,9 @@ import { createAccessLink, getSubscriptions, leaveWorkspace, - checkJoin + checkJoin, + mergeSpecifiedPersons, + canMergeSpecifiedPersons } from '../operations' import { accountPlugin } from '../plugin' @@ -3183,3 +3185,287 @@ describe('getSubscriptions', () => { await expect(getSubscriptions(mockCtx, mockDb, mockBranding, 'test-token', {})).rejects.toThrow(PlatformError) }) }) + +describe('merge specified persons', () => { + const mockCtx = { + error: jest.fn(), + info: jest.fn(), + warn: jest.fn() + } as unknown as MeasureContext + + const mockBranding = null + const workspaceUuid = 'caller-workspace-uuid' as WorkspaceUuid + const callerUuid = 'caller-account-uuid' as AccountUuid + const primaryPerson = 'primary-person-uuid' as PersonUuid + const secondaryPerson = 'secondary-person-uuid' as PersonUuid + const params = { primaryPerson, secondaryPerson } + + let mockDb: any + + beforeEach(() => { + jest.clearAllMocks() + jest.restoreAllMocks() + + mockDb = { + account: { + findOne: jest.fn().mockResolvedValue(null) + }, + person: { + findOne: jest.fn().mockImplementation(async ({ uuid }: { uuid: PersonUuid }) => ({ uuid })) + }, + socialId: { + find: jest.fn().mockResolvedValue([]) + }, + getWorkspaceRole: jest.fn().mockResolvedValue(null) + } + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: callerUuid, + workspace: workspaceUuid, + extra: {} + }) + }) + + // The caller maintains the workspace, and neither merged person belongs to another one. + const asWorkspaceMaintainer = (): void => { + mockDb.getWorkspaceRole.mockImplementation(async (account: AccountUuid) => + account === callerUuid ? AccountRole.Maintainer : null + ) + } + + describe('mergeSpecifiedPersons', () => { + test('should throw BadRequest for empty params', async () => { + await expect( + mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', { + primaryPerson: '' as PersonUuid, + secondaryPerson + }) + ).rejects.toThrow(PlatformError) + + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + + test('should throw Forbidden for a token without workspace', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ account: callerUuid, extra: {} }) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect(mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).rejects.toThrow( + PlatformError + ) + + expect(spy).not.toHaveBeenCalled() + // Pins the workspace guard itself rather than the role lookup that follows it. + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + + test('should throw Forbidden when caller is below Maintainer', async () => { + mockDb.getWorkspaceRole.mockResolvedValue(AccountRole.User) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect(mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).rejects.toThrow( + PlatformError + ) + + expect(spy).not.toHaveBeenCalled() + }) + + test('should throw Forbidden when the secondary person is an account of another workspace', async () => { + asWorkspaceMaintainer() + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === secondaryPerson ? { uuid } : null + ) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect(mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).rejects.toThrow( + PlatformError + ) + + expect(spy).not.toHaveBeenCalled() + }) + + test('should throw Forbidden when the primary person is an account of another workspace', async () => { + asWorkspaceMaintainer() + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === primaryPerson ? { uuid } : null + ) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect(mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).rejects.toThrow( + PlatformError + ) + + expect(spy).not.toHaveBeenCalled() + }) + + test('should merge workspace contacts without accounts for a Maintainer', async () => { + asWorkspaceMaintainer() + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(spy).toHaveBeenCalledWith(mockDb, primaryPerson, secondaryPerson) + }) + + test('should merge a contact into a member of the caller workspace', async () => { + mockDb.getWorkspaceRole.mockImplementation(async (account: AccountUuid) => + account === secondaryPerson ? null : AccountRole.Maintainer + ) + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === primaryPerson ? { uuid } : null + ) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(spy).toHaveBeenCalledWith(mockDb, primaryPerson, secondaryPerson) + }) + + test('should throw Forbidden when a login capable social id would move onto a foreign account', async () => { + // A maintainer minting a person that carries their own email and merging it into a + // co-member would hand them that member's account through password recovery. + mockDb.getWorkspaceRole.mockImplementation(async (account: AccountUuid) => + account === secondaryPerson ? null : AccountRole.Maintainer + ) + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === primaryPerson ? { uuid } : null + ) + mockDb.socialId.find.mockResolvedValue([{ _id: 'attacker-email', type: SocialIdType.EMAIL }]) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect(mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).rejects.toThrow( + PlatformError + ) + + expect(spy).not.toHaveBeenCalled() + }) + + test('should allow a login capable social id to move onto the caller own account', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: primaryPerson as unknown as AccountUuid, + workspace: workspaceUuid, + extra: {} + }) + mockDb.getWorkspaceRole.mockImplementation(async (account: AccountUuid) => + account === secondaryPerson ? null : AccountRole.Maintainer + ) + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === primaryPerson ? { uuid } : null + ) + mockDb.socialId.find.mockResolvedValue([{ _id: 'own-email', type: SocialIdType.EMAIL }]) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(spy).toHaveBeenCalledWith(mockDb, primaryPerson, secondaryPerson) + }) + + test('should throw Forbidden when merging the platform guest account', async () => { + asWorkspaceMaintainer() + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await expect( + mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', { + primaryPerson: readOnlyGuestAccountUuid as PersonUuid, + secondaryPerson + }) + ).rejects.toThrow(PlatformError) + + expect(spy).not.toHaveBeenCalled() + }) + + test('should merge for an allowed service token', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: systemAccountUuid, + extra: { service: 'tool' } + }) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(spy).toHaveBeenCalledWith(mockDb, primaryPerson, secondaryPerson) + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + + test('should merge for a global admin token', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: callerUuid, + extra: { admin: 'true' } + }) + const spy = jest.spyOn(utils, 'doMergePersons').mockResolvedValue() + + await mergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(spy).toHaveBeenCalledWith(mockDb, primaryPerson, secondaryPerson) + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + }) + + describe('canMergeSpecifiedPersons', () => { + beforeEach(() => { + asWorkspaceMaintainer() + }) + + // The merge dialog awaits this predicate without a catch, so refusals must be answered, + // not thrown: a rejection leaves it spinning on a disabled Save button forever. + test('should return false without looking persons up when caller does not maintain a workspace', async () => { + mockDb.getWorkspaceRole.mockResolvedValue(null) + + expect(await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).toBe(false) + expect(mockDb.person.findOne).not.toHaveBeenCalled() + }) + + test('should return false when a person is an account of another workspace', async () => { + mockDb.account.findOne.mockImplementation(async ({ uuid }: { uuid: AccountUuid }) => + uuid === secondaryPerson ? { uuid } : null + ) + + expect(await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).toBe(false) + expect(mockDb.person.findOne).not.toHaveBeenCalled() + }) + + test('should return false for equal persons without authorizing or looking them up', async () => { + const result = await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', { + primaryPerson, + secondaryPerson: primaryPerson + }) + + expect(result).toBe(false) + expect(mockDb.person.findOne).not.toHaveBeenCalled() + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + + test('should return true for a Maintainer when secondary has no verified social ids', async () => { + const result = await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(result).toBe(true) + expect(mockDb.socialId.find).toHaveBeenCalledWith({ personUuid: secondaryPerson, verifiedOn: { $ne: null } }) + }) + + test('should return false when secondary person has verified social ids', async () => { + mockDb.socialId.find.mockResolvedValue([{ _id: 'verified-social-id' }]) + + const result = await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params) + + expect(result).toBe(false) + }) + + test('should allow an allowed service token', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: systemAccountUuid, + extra: { service: 'tool' } + }) + + expect(await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).toBe(true) + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + + test('should allow a global admin token', async () => { + ;(decodeTokenVerbose as jest.Mock).mockReturnValue({ + account: callerUuid, + extra: { admin: 'true' } + }) + + expect(await canMergeSpecifiedPersons(mockCtx, mockDb, mockBranding, 'test-token', params)).toBe(true) + expect(mockDb.getWorkspaceRole).not.toHaveBeenCalled() + }) + }) +}) diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index 3f2f3d5921..6d88d46fe7 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -37,7 +37,13 @@ import { type IntegrationKind } from '@hcengineering/core' import platform, { getMetadata, PlatformError, Severity, Status, translate } from '@hcengineering/platform' -import { decodeToken, decodeTokenVerbose, generateToken, type PermissionsGrant } from '@hcengineering/server-token' +import { + decodeToken, + decodeTokenVerbose, + generateToken, + type PermissionsGrant, + type Token +} from '@hcengineering/server-token' import { isAdminEmail } from './admin' import { accountPlugin } from './plugin' @@ -2877,6 +2883,79 @@ export async function deleteAccount ( }) } +// Social ids that resolve to an account on their own, and therefore hand over the ability to +// authenticate as its owner once they are re-pointed. Password recovery and OTP login look an +// account up by social id value alone (see requestPasswordReset, loginOtp). +const loginCapableSocialTypes = [SocialIdType.EMAIL, SocialIdType.HULY] + +/** + * Merging re-points the secondary person's social ids onto the primary person, so an unrestricted + * caller could both absorb the identifiers of a person they do not own and inject their own + * identifiers into somebody else's person. Restrict it to callers with authority over both persons. + */ +async function verifyMergePersonsAuthority ( + db: AccountDB, + { account, workspace, extra }: Token, + primaryPerson: PersonUuid, + secondaryPerson: PersonUuid, + shouldThrow = true +): Promise { + // Global admins and the tool/workspace services act on behalf of the whole installation, + // the same way the account level merge (mergeSpecifiedAccounts) allows them to. + // Note this must precede the workspace check below: such tokens carry no workspace. + if (extra?.admin === 'true' || verifyAllowedServices(['tool', 'workspace'], extra, false)) { + return true + } + + const forbidden = (): boolean => { + if (shouldThrow) { + throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {})) + } + + return false + } + + // Everybody else acts within a single workspace they maintain. + if (workspace == null) { + return forbidden() + } + + if (!verifyAllowedRole(await db.getWorkspaceRole(account, workspace), AccountRole.Maintainer, extra, false)) { + return forbidden() + } + + // The platform wide accounts are not anybody's to merge. + for (const person of [primaryPerson, secondaryPerson]) { + if (person === systemAccountUuid || person === readOnlyGuestAccountUuid) { + return forbidden() + } + + if ((await db.getWorkspaceRole(person as AccountUuid, workspace)) != null) { + // A member of the caller's workspace. + continue + } + + if ((await db.account.findOne({ uuid: person as AccountUuid })) != null) { + // An account outside of the caller's workspace: no workspace maintainer may take it over. + return forbidden() + } + } + + // Both persons are in reach of the caller by now, but the primary keeps receiving the secondary's + // social ids. When the primary is somebody else's account, a login capable social id would grant + // whoever controls it access to that account, so leave those merges to the verification flows. + // Note doMergePersons only refuses *verified* secondary social ids, which does not cover this. + if (primaryPerson !== account && (await db.account.findOne({ uuid: primaryPerson as AccountUuid })) != null) { + const secondarySocialIds = await db.socialId.find({ personUuid: secondaryPerson }) + + if (secondarySocialIds.some((si) => loginCapableSocialTypes.includes(si.type))) { + return forbidden() + } + } + + return true +} + export async function canMergeSpecifiedPersons ( ctx: MeasureContext, db: AccountDB, @@ -2887,7 +2966,7 @@ export async function canMergeSpecifiedPersons ( secondaryPerson: PersonUuid } ): Promise { - decodeTokenVerbose(ctx, token) + const decodedToken = decodeTokenVerbose(ctx, token) const { primaryPerson, secondaryPerson } = params if (primaryPerson == null || primaryPerson === '' || secondaryPerson == null || secondaryPerson === '') { @@ -2899,6 +2978,12 @@ export async function canMergeSpecifiedPersons ( return false } + // This is a predicate the merge dialog polls, so an unauthorized caller is answered + // rather than thrown at. mergeSpecifiedPersons below enforces the same rules. + if (!(await verifyMergePersonsAuthority(db, decodedToken, primaryPerson, secondaryPerson, false))) { + return false + } + const primaryPersonObj = await db.person.findOne({ uuid: primaryPerson }) if (primaryPersonObj == null) { throw new PlatformError(new Status(Severity.ERROR, platform.status.PersonNotFound, { person: primaryPerson })) @@ -2928,13 +3013,15 @@ export async function mergeSpecifiedPersons ( secondaryPerson: PersonUuid } ): Promise { - decodeTokenVerbose(ctx, token) + const decodedToken = decodeTokenVerbose(ctx, token) const { primaryPerson, secondaryPerson } = params if (primaryPerson == null || primaryPerson === '' || secondaryPerson == null || secondaryPerson === '') { throw new PlatformError(new Status(Severity.ERROR, platform.status.BadRequest, {})) } + await verifyMergePersonsAuthority(db, decodedToken, primaryPerson, secondaryPerson) + await doMergePersons(db, primaryPerson, secondaryPerson) } From de784bf2ef917b4946b1127cb36a135a096c25b2 Mon Sep 17 00:00:00 2001 From: Artyom Savchenko Date: Mon, 27 Jul 2026 11:54:07 +0700 Subject: [PATCH 4/7] Clean up commented tools (#10993) * Clean up commented tools Signed-off-by: Artyom Savchenko * Clean up Signed-off-by: Artyom Savchenko --------- Signed-off-by: Artyom Savchenko --- dev/tool/src/index.ts | 1583 ----------------------------------------- 1 file changed, 1583 deletions(-) diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 1d8720c9c0..986f9c2d33 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -114,7 +114,6 @@ import { type StorageAdapter } from '@hcengineering/server-core' import { getAccountDBUrl, getKvsUrl, getMongoDBUrl } from './__start' -// import { fillGithubUsers, fixAccountEmails, renameAccount } from './account' import { changeConfiguration } from './configuration' import { performCalendarAccountMigrations } from './calendar' @@ -264,77 +263,6 @@ export function devTool ( }) }) - // program - // .command('reset-account ') - // .description('create user and corresponding account in master database') - // .option('-p, --password ', 'new user password') - // .action(async (email: string, cmd) => { - // await withAccountDatabase(async (db) => { - // console.log(`update account ${email} ${cmd.first as string} ${cmd.last as string}...`) - // await replacePassword(db, email, cmd.password) - // }) - // }) - - // program - // .command('reset-email ') - // .description('rename account in accounts and all workspaces') - // .action(async (email: string, newEmail: string, cmd) => { - // await withAccountDatabase(async (db) => { - // console.log(`update account ${email} to ${newEmail}`) - // await renameAccount(toolCtx, db, accountsUrl, email, newEmail) - // }) - // }) - - // program - // .command('fix-email ') - // .description('fix email in all workspaces to be proper one') - // .action(async (email: string, newEmail: string, cmd) => { - // await withAccountDatabase(async (db) => { - // console.log(`update account ${email} to ${newEmail}`) - // await fixAccountEmails(toolCtx, db, accountsUrl, email, newEmail) - // }) - // }) - - // program - // .command('compact-db-mongo') - // .description('compact all db collections') - // .option('-w, --workspace ', 'A selected "workspace" only', '') - // .action(async (cmd: { workspace: string }) => { - // const dbUrl = getMongoDBUrl() - // await withAccountDatabase(async (db) => { - // console.log('compacting db ...') - // let gtotal: number = 0 - // const client = getMongoClient(dbUrl) - // const _client = await client.getClient() - // try { - // const workspaces = await listWorkspacesPure(db) - // for (const workspace of workspaces) { - // if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - // continue - // } - // let total: number = 0 - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - // const collections = wsDb.listCollections() - // while (true) { - // const collInfo = await collections.next() - // if (collInfo === null) { - // break - // } - // const result = await wsDb.command({ compact: collInfo.name }) - // total += result.bytesFreed - // } - // gtotal += total - // console.log('total feed for db', workspace.workspaceName, Math.round(total / (1024 * 1024))) - // } - // console.log('global total feed', Math.round(gtotal / (1024 * 1024))) - // } catch (err: any) { - // console.error(err) - // } finally { - // client.close() - // } - // }) - // }) - program .command('assign-workspace ') .description('assign workspace') @@ -358,16 +286,6 @@ export function devTool ( }) }) - // program - // .command('show-user ') - // .description('show user') - // .action(async (email) => { - // await withAccountDatabase(async (db) => { - // const info = await getAccount(db, email) - // console.log(info) - // }) - // }) - program .command('create-workspace ') .description('create workspace') @@ -458,16 +376,6 @@ export function devTool ( }) }) - // program - // .command('set-user-admin ') - // .description('set user role') - // .action(async (email: string, role: string) => { - // console.log(`set user ${email} admin...`) - // await withAccountDatabase(async (db) => { - // await setAccountAdmin(db, email, role === 'true') - // }) - // }) - async function doUpgrade ( toolCtx: MeasureMetricsContext, workspace: WorkspaceUuid, @@ -531,405 +439,6 @@ export function devTool ( await doUpgrade(toolCtx, workspace, cmd.force, cmd.indexes) }) - // program - // .command('upgrade') - // .description('upgrade') - // .option('-l|--logs ', 'Default logs folder', './logs') - // .option('-i|--ignore [ignore]', 'Ignore workspaces', '') - // .option('-r|--region [region]', 'Region of workspaces', '') - // .option( - // '-c|--console', - // 'Display all information into console(default will create logs folder with {workspace}.log files', - // false - // ) - // .option('-f|--force [force]', 'Force update', false) - // .action(async (cmd: { logs: string, force: boolean, console: boolean, ignore: string, region: string }) => { - // const { version, txes, migrateOperations } = prepareTools() - // await withAccountDatabase(async (db) => { - // const workspaces = (await listWorkspacesRaw(db, cmd.region)).filter((ws) => !cmd.ignore.includes(ws.workspace)) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - // const measureCtx = new MeasureMetricsContext('upgrade', {}) - - // for (const ws of workspaces) { - // console.warn('UPGRADING', ws.workspaceName) - // const logger = cmd.console - // ? consoleModelLogger - // : new FileModelLogger(path.join(cmd.logs, `${ws.workspace}.log`)) - - // try { - // await upgradeWorkspace( - // measureCtx, - // version, - // txes, - // migrateOperations, - // ws, - // logger, - // async () => {}, - // cmd.force, - // false, - // true - // ) - - // await updateWorkspace(db, ws, { - // mode: 'active', - // progress: 100, - // version, - // attempts: 0 - // }) - // } catch (err: any) { - // toolCtx.error('failed to upgrade', { err, workspace: ws.workspace, workspaceName: ws.workspaceName }) - // continue - // } - // } - // console.log('upgrade done') - // }) - // }) - - // program - // .command('list-unused-workspaces') - // .description('list unused workspaces. Without it will only mark them disabled') - // .option('-t|--timeout [timeout]', 'Timeout in days', '60') - // .action(async (cmd: { disable: boolean, exclude: string, timeout: string }) => { - // await withAccountDatabase(async (db) => { - // const workspaces = new Map((await listWorkspacesPure(db)).map((p) => [p._id.toString(), p])) - - // const accounts = await listAccounts(db) - - // const _timeout = parseInt(cmd.timeout) ?? 7 - - // let used = 0 - // let unused = 0 - - // for (const a of accounts) { - // const authored = a.workspaces - // .map((it) => workspaces.get(it.toString())) - // .filter((it) => it !== undefined && it.createdBy?.trim() === a.email?.trim()) as Workspace[] - // authored.sort((a, b) => b.lastVisit - a.lastVisit) - // if (authored.length > 0) { - // const lastLoginDays = Math.floor((Date.now() - a.lastVisit) / 1000 / 3600 / 24) - // toolCtx.info(a.email, { - // workspaces: a.workspaces.length, - // firstName: a.first, - // lastName: a.last, - // lastLoginDays - // }) - // for (const ws of authored) { - // const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - - // if (lastVisitDays > _timeout) { - // unused++ - // toolCtx.warn(' --- unused', { - // url: ws.workspaceUrl, - // id: ws.workspace, - // lastVisitDays - // }) - // } else { - // used++ - // toolCtx.warn(' +++ used', { - // url: ws.workspaceUrl, - // id: ws.workspace, - // createdBy: ws.createdBy, - // lastVisitDays - // }) - // } - // } - // } - // } - - // console.log('Used: ', used, 'Unused: ', unused) - // }) - // }) - - // program - // .command('archive-workspaces') - // .description('Archive and delete non visited workspaces...') - // .option('-r|--remove [remove]', 'Pass to remove all data', false) - // .option('--region [region]', 'Pass to remove all data', '') - // .option('-t|--timeout [timeout]', 'Timeout in days', '60') - // .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '') - // .action( - // async (cmd: { - // disable: boolean - // exclude: string - // timeout: string - // remove: boolean - // workspace: string - // region: string - // }) => { - // const { dbUrl, txes } = prepareTools() - // await withAccountDatabase(async (db) => { - // const workspaces = (await listWorkspacesPure(db)) - // .sort((a, b) => a.lastVisit - b.lastVisit) - // .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) - - // const _timeout = parseInt(cmd.timeout) ?? 7 - - // let unused = 0 - // for (const ws of workspaces) { - // const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - - // if (lastVisitDays > _timeout && isActiveMode(ws.mode)) { - // unused++ - // toolCtx.warn('--- unused', { - // url: ws.workspaceUrl, - // id: ws.workspace, - // lastVisitDays, - // mode: ws.mode - // }) - // try { - // await backupWorkspace( - // toolCtx, - // ws, - // (dbUrl, storageAdapter) => { - // const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { - // externalStorage: storageAdapter, - // usePassedCtx: true - // }) - // return factory - // }, - // (ctx, dbUrls, workspace, branding, externalStorage) => { - // return getConfig(ctx, dbUrls, ctx, { - // externalStorage, - // disableTriggers: true - // }) - // }, - // cmd.region, - // 5000, // 5 gigabytes per blob - // async (storage, workspaceStorage) => { - // if (cmd.remove) { - // await updateArchiveInfo(toolCtx, db, ws.workspace, true) - // const files = await workspaceStorage.listStream(toolCtx, { name: ws.workspace }) - - // while (true) { - // const docs = await files.next() - // if (docs.length === 0) { - // break - // } - // await workspaceStorage.remove( - // toolCtx, - // { name: ws.workspace }, - // docs.map((it) => it._id) - // ) - // } - - // program - // .command('backup-all') - // .description('Backup all workspaces...') - // .option('--region [region]', 'Force backup of selected workspace', '') - // .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '') - // .action(async (cmd: { workspace: string, region: string }) => { - // const { txes } = prepareTools() - // await withAccountDatabase(async (db) => { - // const workspaces = (await listWorkspacesPure(db)) - // .sort((a, b) => a.lastVisit - b.lastVisit) - // .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) - - // let processed = 0 - - // // We need to update workspaces with missing workspaceUrl - // for (const ws of workspaces) { - // try { - // if ( - // await backupWorkspace( - // toolCtx, - // ws, - // (dbUrl, storageAdapter) => { - // const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { - // externalStorage: storageAdapter, - // usePassedCtx: true - // }) - // return factory - // }, - // (ctx, dbUrls, workspace, branding, externalStorage) => { - // return getConfig(ctx, dbUrls, ctx, { - // externalStorage, - // disableTriggers: true - // }) - // }, - // cmd.region, - // 100, - // ) - // ) { - // processed++ - // } - // } catch (err: any) { - // toolCtx.error('Failed to backup workspace', { workspace: ws.workspace }) - // } - // } - // console.log('Processed workspaces', processed) - // }) - // }) - - // program - // .command('drop-workspace ') - // .description('drop workspace') - // .option('--full [full]', 'Force remove all data', false) - // .action(async (workspace, cmd: { full: boolean }) => { - // const { dbUrl } = prepareTools() - - // await withStorage(async (storageAdapter) => { - // await withAccountDatabase(async (db) => { - // const ws = await getWorkspaceById(db, workspace) - // if (ws === null) { - // console.log('no workspace exists') - // return - // } - // if (cmd.full) { - // await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace, storageAdapter) - // } else { - // await dropWorkspace(toolCtx, db, null, workspace) - // } - // }) - // }) - // }) - - // program - // .command('drop-workspace-by-email ') - // .description('drop workspace') - // .option('--full [full]', 'Force remove all data', false) - // .action(async (email, cmd: { full: boolean }) => { - // const { dbUrl } = prepareTools() - // await withStorage(async (storageAdapter) => { - // await withAccountDatabase(async (db) => { - // for (const workspace of await listWorkspacesByAccount(db, email)) { - // if (cmd.full) { - // await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace.workspace, storageAdapter) - // } else { - // await dropWorkspace(toolCtx, db, null, workspace.workspace) - // } - // } - // }) - // }) - // }) - // program - // .command('list-workspace-by-email ') - // .description('drop workspace') - // .option('--full [full]', 'Force remove all data', false) - // .action(async (email, cmd: { full: boolean }) => { - // await withAccountDatabase(async (db) => { - // for (const workspace of await listWorkspacesByAccount(db, email)) { - // console.log(workspace.workspace, workspace.workspaceUrl, workspace.workspaceName) - // } - // }) - // }) - - // program - // .command('drop-workspace-last-visit') - // .description('drop old workspaces') - // .action(async (cmd: any) => { - // const { dbUrl } = prepareTools() - - // await withStorage(async (storageAdapter) => { - // await withAccountDatabase(async (db) => { - // const workspacesJSON = await listWorkspacesPure(db) - // for (const ws of workspacesJSON) { - // const lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - // if (lastVisit > 60) { - // await dropWorkspaceFull(toolCtx, db, dbUrl, null, ws.workspace, storageAdapter) - // } - // } - // }) - // }) - // }) - - // program - // .command('list-workspaces') - // .description('List workspaces') - // .option('-e|--expired [expired]', 'Show only expired', false) - // .action(async (cmd: { expired: boolean }) => { - // const { version } = prepareTools() - // await withAccountDatabase(async (db) => { - // const workspacesJSON = await listWorkspacesPure(db) - // for (const ws of workspacesJSON) { - // let lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - // if (cmd.expired && lastVisit <= 7) { - // continue - // } - // console.log( - // colorConstants.colorBlue + - // '####################################################################################################' + - // colorConstants.reset - // ) - // console.log('id:', colorConstants.colorWhiteCyan + ws.workspace + colorConstants.reset) - // console.log('url:', ws.workspaceUrl, 'name:', ws.workspaceName) - // console.log( - // 'version:', - // ws.version !== undefined ? versionToString(ws.version) : 'not-set', - // !deepEqual(ws.version, version) ? `upgrade to ${versionToString(version)} is required` : '' - // ) - // console.log('disabled:', ws.disabled) - // console.log('mode:', ws.mode) - // console.log('created by:', ws.createdBy) - // console.log('members:', (ws.accounts ?? []).length) - // if (Number.isNaN(lastVisit)) { - // lastVisit = 365 - // } - // if (lastVisit > 30) { - // console.log(colorConstants.colorRed + `last visit: ${lastVisit} days ago` + colorConstants.reset) - // } else if (lastVisit > 7) { - // console.log(colorConstants.colorRedYellow + `last visit: ${lastVisit} days ago` + colorConstants.reset) - // } else { - // console.log('last visit:', lastVisit, 'days ago') - // } - // } - - // console.log('latest model version:', JSON.stringify(version)) - // }) - // }) - - // program.command('fix-person-accounts-mongo').action(async () => { - // const { version } = prepareTools() - // const mongodbUri = getMongoDBUrl() - // await withAccountDatabase(async (db) => { - // const ws = await listWorkspacesPure(db) - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - // try { - // for (const w of ws) { - // const wsDb = getWorkspaceMongoDB(_client, { name: w.workspace }) - // await wsDb.collection('tx').updateMany( - // { - // objectClass: contact.class.PersonAccount, - // objectSpace: null - // }, - // { $set: { objectSpace: core.space.Model } } - // ) - // } - // } finally { - // client.close() - // } - - // console.log('latest model version:', JSON.stringify(version)) - // }) - // }) - - // program - // .command('show-accounts') - // .description('Show accounts') - // .action(async () => { - // await withAccountDatabase(async (db) => { - // const workspaces = await listWorkspacesPure(db) - // const accounts = await listAccounts(db) - // for (const a of accounts) { - // const wss = a.workspaces.map((it) => it.toString()) - // console.info( - // a.email, - // a.confirmed, - // workspaces.filter((it) => wss.includes(it._id.toString())).map((it) => it.workspaceUrl ?? it.workspace) - // ) - // } - // }) - // }) - - // program - // .command('drop-account ') - // .description('drop account') - // .action(async (email: string, cmd) => { - // await withAccountDatabase(async (db) => { - // await dropAccount(toolCtx, db, null, email) - // }) - // }) - program .command('backup ') .description('dump workspace transactions, blobs and accounts') @@ -1416,85 +925,6 @@ export function devTool ( } ) - // program - // .command('backup-list ') - // .description('list snaphost ids for backup') - // .action(async (dirName: string, cmd) => { - // const storage = await createFileBackupStorage(dirName) - // await backupList(storage) - // }) - - // program - // .command('backup-s3 ') - // .description('dump workspace transactions and minio resources') - // .action(async (bucketName: string, dirName: string, workspace: string, cmd) => { - // await withStorage(async (adapter) => { - // const storage = await createStorageBackupStorage(toolCtx, adapter, getWorkspaceId(bucketName), dirName) - // const wsid = getWorkspaceId(workspace) - // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - // await backup(toolCtx, endpoint, wsIds, storage) - // }) - // }) - - // program - // .command('backup-s3-clean ') - // .description('dump workspace transactions and minio resources') - // .action(async (bucketName: string, days: string, cmd) => { - // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - - // const daysInterval = Date.now() - parseInt(days) * 24 * 60 * 60 * 1000 - // try { - // const token = generateToken(systemAccountUuid, undefined, { service: 'tool' }) - // const accountClient = getAccountClient(token) - // const workspaces = (await accountClient.listWorkspaces(null, 'active')).filter((it) => { - // const lastBackup = it.backupInfo?.lastBackup ?? 0 - // if (lastBackup > daysInterval) { - // // No backup required, interval not elapsed - // return true - // } - - // if (it.lastVisit == null) { - // return false - // } - - // return false - // }) - // workspaces.sort((a, b) => { - // return (b.backupInfo?.backupSize ?? 0) - (a.backupInfo?.backupSize ?? 0) - // }) - - // for (const ws of workspaces) { - // const storage = await createStorageBackupStorage( - // toolCtx, - // storageAdapter, - // getWorkspaceId(bucketName), - // ws.workspace - // ) - // await backupRemoveLast(storage, daysInterval) - // const accountClient = getAccountClient(token) - // await accountClient.updateBackupInfo({ - // backups: ws.backupInfo?.backups ?? 0, - // backupSize: ws.backupInfo?.backupSize ?? 0, - // blobsSize: ws.backupInfo?.blobsSize ?? 0, - // dataSize: ws.backupInfo?.dataSize ?? 0, - // lastBackup: daysInterval - // }) - // } - // } finally { - // await storageAdapter.close() - // } - // }) - - // program - // .command('backup-clean ') - // .description('dump workspace transactions and minio resources') - // .action(async (dirName: string, days: string, cmd) => { - // const daysInterval = Date.now() - parseInt(days) * 24 * 60 * 60 * 1000 - // const storage = await createFileBackupStorage(dirName) - // await backupRemoveLast(storage, daysInterval) - // }) - program .command('backup-s3-compact ') .description('Compact a given backup to just one snapshot') @@ -1525,68 +955,6 @@ export function devTool ( } await storageAdapter.close() }) - // program - // .command('backup-s3-check ') - // .description('Compact a given backup to just one snapshot') - // .action(async (bucketName: string, dirName: string, cmd: any) => { - // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - // try { - // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - // await checkBackupIntegrity(toolCtx, storage) - // } catch (err: any) { - // toolCtx.error('failed to size backup', { err }) - // } - // await storageAdapter.close() - // }) - - // program - // .command('backup-s3-restore [date]') - // .description('dump workspace transactions and minio resources') - // .action(async (bucketName: string, dirName: string, workspace: string, date, cmd) => { - // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - // try { - // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - // const wsid = getWorkspaceId(workspace) - // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - // await restore(toolCtx, endpoint, wsid, storage, { - // date: parseInt(date ?? '-1') - // }) - // } catch (err: any) { - // toolCtx.error('failed to size backup', { err }) - // } - // await storageAdapter.close() - // }) - // program - // .command('backup-s3-list ') - // .description('list snaphost ids for backup') - // .action(async (bucketName: string, dirName: string, cmd) => { - // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - // try { - // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - // await backupList(storage) - // } catch (err: any) { - // toolCtx.error('failed to size backup', { err }) - // } - // await storageAdapter.close() - // }) - - // program - // .command('backup-s3-size ') - // .description('list snaphost ids for backup') - // .action(async (bucketName: string, dirName: string, cmd) => { - // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - // try { - // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - // await backupSize(storage) - // } catch (err: any) { - // toolCtx.error('failed to size backup', { err }) - // } - // await storageAdapter.close() - // }) program .command('backup-s3-download ') @@ -1606,309 +974,6 @@ export function devTool ( await storageAdapter.close() }) - // program - // .command('copy-s3-datalake') - // .description('copy files from s3 to datalake') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('-c, --concurrency ', 'Number of files being processed concurrently', '10') - // .option('-s, --skip ', 'Number of workspaces to skip', '0') - // .option('-e, --existing', 'Copy existing blobs', false) - // .action(async (cmd: { workspace: string, concurrency: string, existing: boolean, skip: string }) => { - // const params = { - // concurrency: parseInt(cmd.concurrency), - // existing: cmd.existing - // } - // const skip = parseInt(cmd.skip) - - // const storageConfig = storageConfigFromEnv(process.env.STORAGE) - - // const storages = storageConfig.storages.filter((p) => p.kind === S3_CONFIG_KIND) as S3Config[] - // if (storages.length === 0) { - // throw new Error('S3 storage config is required') - // } - - // const datalakeConfig = storageConfig.storages.find((p) => p.kind === DATALAKE_CONFIG_KIND) - // if (datalakeConfig === undefined) { - // throw new Error('Datalake storage config is required') - // } - - // toolCtx.info('using datalake', { datalake: datalakeConfig }) - - // let workspaces: Workspace[] = [] - // await withAccountDatabase(async (db) => { - // workspaces = await listWorkspacesPure(db) - // workspaces = workspaces - // .filter((p) => isActiveMode(p.mode) || isArchivingMode(p.mode)) - // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // // .sort((a, b) => b.lastVisit - a.lastVisit) - // .sort((a, b) => { - // if (a.backupInfo !== undefined && b.backupInfo !== undefined) { - // return b.backupInfo.blobsSize - a.backupInfo.blobsSize - // } else if (b.backupInfo !== undefined) { - // return 1 - // } else if (a.backupInfo !== undefined) { - // return -1 - // } else { - // return b.lastVisit - a.lastVisit - // } - // }) - // }) - - // const count = workspaces.length - // console.log('found workspaces', count) - - // let index = 0 - // for (const workspace of workspaces) { - // index++ - // if (index <= skip) { - // toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) - // continue - // } - - // toolCtx.info('processing workspace', { - // workspace: workspace.workspace, - // index, - // count, - // blobsSize: workspace.backupInfo?.blobsSize ?? 0 - // }) - // const workspaceId = getWorkspaceId(workspace.workspace) - // const token = generateToken(systemAccountEmail, workspaceId) - // const datalake = createDatalakeClient(datalakeConfig as DatalakeConfig, token) - - // for (const config of storages) { - // const storage = new S3Service(config) - // await copyToDatalake(toolCtx, workspaceId, config, storage, datalake, params) - // } - // } - // }) - - // program - // .command('restore-wiki-content-mongo') - // .description('restore wiki document contents') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('-d, --dryrun', 'Dry run', false) - // .action(async (cmd: { workspace: string, dryrun: boolean }) => { - // const params = { - // dryRun: cmd.dryrun - // } - - // const { version } = prepareTools() - - // let workspaces: Workspace[] = [] - // await withAccountDatabase(async (db) => { - // workspaces = await listWorkspacesPure(db) - // workspaces = workspaces - // .filter((p) => isActiveMode(p.mode)) - // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // .sort((a, b) => b.lastVisit - a.lastVisit) - // }) - - // console.log('found workspaces', workspaces.length) - - // await withStorage(async (storageAdapter) => { - // const mongodbUri = getMongoDBUrl() - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - - // try { - // const count = workspaces.length - // let index = 0 - // for (const workspace of workspaces) { - // index++ - - // toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) - // if (workspace.version === undefined || !deepEqual(workspace.version, version)) { - // console.log(`upgrade to ${versionToString(version)} is required`) - // continue - // } - - // const workspaceId = getWorkspaceId(workspace.workspace) - // const workspaceDataId = workspace.dataId ?? workspace.uuid - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - // await restoreWikiContentMongo(toolCtx, wsDb, workspaceDataId, storageAdapter, params) - // } - // } finally { - // client.close() - // } - // }) - // }) - - // program - // .command('restore-controlled-content-mongo') - // .description('restore controlled document contents') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('-d, --dryrun', 'Dry run', false) - // .option('-f, --force', 'Force update', false) - // .action(async (cmd: { workspace: string, dryrun: boolean, force: boolean }) => { - // const params = { - // dryRun: cmd.dryrun - // } - - // const { version } = prepareTools() - - // let workspaces: Workspace[] = [] - // await withAccountDatabase(async (db) => { - // workspaces = await listWorkspacesPure(db) - // workspaces = workspaces - // .filter((p) => p.mode !== 'archived') - // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // .sort((a, b) => b.lastVisit - a.lastVisit) - // }) - - // console.log('found workspaces', workspaces.length) - - // await withStorage(async (storageAdapter) => { - // await withAccountDatabase(async (db) => { - // const mongodbUri = getMongoDBUrl() - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - - // try { - // const count = workspaces.length - // let index = 0 - // for (const workspace of workspaces) { - // index++ - - // toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) - - // if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) { - // console.log(`upgrade to ${versionToString(version)} is required`) - // continue - // } - - // const workspaceId = getWorkspaceId(workspace.workspace) - // const workspaceDataId = workspace.dataId ?? workspace.uuid - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - // await restoreControlledDocContentMongo(toolCtx, wsDb, workspaceDataId, storageAdapter, params) - // } - // } finally { - // client.close() - // } - // }) - // }) - // }) - - // program - // .command('restore-markup-ref-mongo') - // .description('restore markup document content refs') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('-f, --force', 'Force update', false) - // .action(async (cmd: { workspace: string, force: boolean }) => { - // const { txes, version } = prepareTools() - - // const { hierarchy } = await buildModel(toolCtx, txes) - - // let workspaces: Workspace[] = [] - // await withAccountDatabase(async (db) => { - // workspaces = await listWorkspacesPure(db) - // workspaces = workspaces - // .filter((p) => isActiveMode(p.mode)) - // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // .sort((a, b) => b.lastVisit - a.lastVisit) - // }) - - // console.log('found workspaces', workspaces.length) - - // await withStorage(async (storageAdapter) => { - // const mongodbUri = getMongoDBUrl() - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - - // try { - // const count = workspaces.length - // let index = 0 - // for (const workspace of workspaces) { - // index++ - - // toolCtx.info('processing workspace', { - // workspace: workspace.workspace, - // version: workspace.version, - // index, - // count - // }) - - // if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) { - // console.log(`upgrade to ${versionToString(version)} is required`) - // continue - // } - - // const workspaceId = getWorkspaceId(workspace.workspace) - // const workspaceDataId = workspace.dataId ?? workspace.uuid - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - // await restoreMarkupRefsMongo(toolCtx, wsDb, workspaceDataId, hierarchy, storageAdapter) - // } - // } finally { - // client.close() - // } - // }) - // }) - - // program - // .command('confirm-email ') - // .description('confirm user email') - // .action(async (email: string, cmd) => { - // await withAccountDatabase(async (db) => { - // const account = await getAccount(db, email) - // if (account?.confirmed === true) { - // console.log(`Already confirmed:${email}`) - // } else { - // await confirmEmail(db, email) - // } - // }) - // }) - - // program - // .command('diff-workspace ') - // .description('restore workspace transactions and minio resources from previous dump.') - // .action(async (workspace: string, cmd) => { - // const { dbUrl, txes } = prepareTools() - // await diffWorkspace(dbUrl, workspace, txes) - // }) - - // program - // .command('clear-telegram-history ') - // .description('clear telegram history') - // .option('-w, --workspace ', 'target workspace') - // .action(async (workspace: string, cmd) => { - // const { dbUrl } = prepareTools() - // await withStorage(async (adapter) => { - // const telegramDB = process.env.TELEGRAM_DATABASE - // if (telegramDB === undefined) { - // console.error('please provide TELEGRAM_DATABASE.') - // process.exit(1) - // } - - // console.log(`clearing ${workspace} history:`) - // await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(workspace), telegramDB, adapter) - // }) - // }) - - // program - // .command('clear-telegram-all-history') - // .description('clear telegram history') - // .action(async (cmd) => { - // const { dbUrl } = prepareTools() - // await withStorage(async (adapter) => { - // await withAccountDatabase(async (db) => { - // const telegramDB = process.env.TELEGRAM_DATABASE - // if (telegramDB === undefined) { - // console.error('please provide TELEGRAM_DATABASE.') - // process.exit(1) - // } - - // const workspaces = await listWorkspacesPure(db) - - // for (const w of workspaces) { - // console.log(`clearing ${w.workspace} history:`) - // await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(w.workspace), telegramDB, adapter) - // } - // }) - // }) - // }) - program .command('generate-token ') .description('generate token') @@ -1980,248 +1045,6 @@ export function devTool ( console.log(decodeToken(token)) }) - // program - // .command('clean-workspace ') - // .description('clean workspace') - // .option('--recruit', 'Clean recruit', false) - // .option('--tracker', 'Clean tracker', false) - // .option('--removedTx', 'Clean removed transactions', false) - // .action(async (workspace: string, cmd: { recruit: boolean, tracker: boolean, removedTx: boolean }) => { - // const { dbUrl } = prepareTools() - // await withStorage(async (adapter) => { - // const wsid = getWorkspaceId(workspace) - // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - // await cleanWorkspace(toolCtx, dbUrl, wsid, adapter, endpoint, cmd) - // }) - // }) - // program - // .command('clean-empty-buckets') - // .option('--prefix [prefix]', 'Prefix', '') - // .action(async (cmd: { prefix: string }) => { - // await withStorage(async (adapter) => { - // const buckets = await adapter.listBuckets(toolCtx) - // for (const ws of buckets) { - // if (ws.name.startsWith(cmd.prefix)) { - // console.log('Checking', ws.name) - // const l = await ws.list() - // const docs = await l.next() - // if (docs.length === 0) { - // await l.close() - // // No data, we could delete it. - // console.log('Clean bucket', ws.name) - // await ws.delete() - // } else { - // await l.close() - // } - // } - // } - // }) - // }) - // program - // .command('upload-file ') - // .action(async (workspace: string, local: string, remote: string, contentType: string, cmd: any) => { - // const wsId: WorkspaceId = { - // name: workspace - // } - // const token = generateToken(systemAccountEmail, wsId) - // const endpoint = await getTransactorEndpoint(token) - // const blobClient = new BlobClient(endpoint, token, wsId) - // const buffer = readFileSync(local) - // await blobClient.upload(toolCtx, remote, buffer.length, contentType, buffer) - // }) - - // program - // .command('download-file ') - // .action(async (workspace: string, remote: string, local: string, cmd: any) => { - // const wsId: WorkspaceId = { - // name: workspace - // } - // const token = generateToken(systemAccountEmail, wsId) - // const endpoint = await getTransactorEndpoint(token) - // const blobClient = new BlobClient(endpoint, token, wsId) - // const wrstream = createWriteStream(local) - // await blobClient.writeTo(toolCtx, remote, -1, { - // write: (buffer, cb) => { - // wrstream.write(buffer, cb) - // }, - // end: (cb) => { - // wrstream.end(cb) - // } - // }) - // }) - - // program - // .command('move-files') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('-m, --move ', 'When set to true, the files will be moved, otherwise copied', 'false') - // .option('-bl, --blobLimit ', 'A blob size limit in megabytes (default 50mb)', '999999') - // .option('-c, --concurrency ', 'Number of files being processed concurrently', '10') - // .option('--disabled', 'Include disabled workspaces', false) - // .action( - // async (cmd: { workspace: string, move: string, blobLimit: string, concurrency: string, disabled: boolean }) => { - // const params = { - // concurrency: parseInt(cmd.concurrency), - // move: cmd.move === 'true' - // } - - // await withAccountDatabase(async (db) => { - // await withStorage(async (adapter) => { - // try { - // const exAdapter = adapter as StorageAdapterEx - // if (exAdapter.adapters === undefined || exAdapter.adapters.length < 2) { - // throw new Error('bad storage config, at least two storage providers are required') - // } - - // console.log('moving files to storage provider', exAdapter.adapters[0].name) - - // let index = 1 - // const workspaces = await listWorkspacesPure(db) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - - // const rateLimit = new RateLimiter(10) - // for (const workspace of workspaces) { - // if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - // continue - // } - // if (!isActiveMode(workspace.mode)) { - // console.log('ignore non active workspace', workspace.workspace, workspace.mode) - // continue - // } - // if (workspace.disabled === true && !cmd.disabled) { - // console.log('ignore disabled workspace', workspace.workspace) - // continue - // } - - // await rateLimit.exec(async () => { - // console.log('start', workspace.workspace, index, '/', workspaces.length) - // await moveFiles(toolCtx, getWorkspaceId(workspace.workspace), exAdapter, params) - // console.log('done', workspace.workspace) - // index += 1 - // }) - // } - // await rateLimit.waitProcessing() - // } catch (err: any) { - // console.error(err) - // } - // }) - // }) - // } - // ) - - // program - // .command('show-lost-files-mongo') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .option('--disabled', 'Include disabled workspaces', false) - // .option('--all', 'Show all files', false) - // .action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => { - // await withAccountDatabase(async (db) => { - // await withStorage(async (adapter) => { - // const mongodbUri = getMongoDBUrl() - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - // try { - // let index = 1 - // const workspaces = await listWorkspacesPure(db) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - - // for (const workspace of workspaces) { - // if (!isActiveMode(workspace.mode)) { - // console.log('ignore non active workspace', workspace.workspace, workspace.mode) - // continue - // } - // if (workspace.disabled === true && !cmd.disabled) { - // console.log('ignore disabled workspace', workspace.workspace) - // continue - // } - - // if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - // continue - // } - - // try { - // console.log('start', workspace.workspace, index, '/', workspaces.length) - // const workspaceId = getWorkspaceId(workspace.workspace) - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - // await showLostFiles(toolCtx, workspaceId, wsDb, adapter, { showAll: cmd.all }) - // console.log('done', workspace.workspace) - // } catch (err) { - // console.error(err) - // } - - // index += 1 - // } - // } catch (err: any) { - // console.error(err) - // } finally { - // client.close() - // } - // }) - // }) - // }) - - // program.command('fix-bw-workspace ').action(async (workspace: string) => { - // await withStorage(async (adapter) => { - // await fixMinioBW(toolCtx, getWorkspaceId(workspace), adapter) - // }) - // }) - - // program - // .command('clean-removed-transactions ') - // .description('clean removed transactions') - // .action(async (workspace: string, cmd: any) => { - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await cleanRemovedTransactions(wsid, endpoint) - // }) - - // program - // .command('clean-archived-spaces ') - // .description('clean archived spaces') - // .action(async (workspace: string, cmd: any) => { - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await cleanArchivedSpaces(wsid, endpoint) - // }) - - // program - // .command('chunter-fix-comments ') - // .description('chunter-fix-comments') - // .action(async (workspace: string, cmd: any) => { - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await fixCommentDoubleIdCreate(wsid, endpoint) - // }) - - // program - // .command('mixin-show-foreign-attributes ') - // .description('mixin-show-foreign-attributes') - // .option('--mixin ', 'Mixin class', '') - // .option('--property ', 'Property name', '') - // .option('--detail ', 'Show details', false) - // .action(async (workspace: string, cmd: { detail: boolean, mixin: string, property: string }) => { - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await showMixinForeignAttributes(wsid, endpoint, cmd) - // }) - - // program - // .command('mixin-fix-foreign-attributes-mongo ') - // .description('mixin-fix-foreign-attributes') - // .option('--mixin ', 'Mixin class', '') - // .option('--property ', 'Property name', '') - // .action(async (workspace: string, cmd: { mixin: string, property: string }) => { - // const mongodbUri = getMongoDBUrl() - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // FIXME: add dataId - // await fixMixinForeignAttributes(mongodbUri, wsid, endpoint, cmd) - // }) - program .command('configure ') .description('clean archived spaces') @@ -2240,152 +1063,6 @@ export function devTool ( }) }) - // program - // .command('configure-all') - // .description('configure all spaces') - // .option('--enable ', 'Enable plugin configuration', '') - // .option('--disable ', 'Disable plugin configuration', '') - // .option('--list', 'List plugin states', false) - // .action(async (cmd: { enable: string, disable: string, list: boolean }) => { - // await withAccountDatabase(async (db) => { - // console.log('configure all workspaces') - // console.log(JSON.stringify(cmd)) - // const workspaces = await listWorkspacesRaw(db) - // for (const ws of workspaces) { - // console.log('configure', ws.workspaceName ?? ws.workspace) - // const wsid = getWorkspaceId(ws.workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await changeConfiguration(wsid, endpoint, cmd) - // } - // }) - // }) - - // program - // .command('optimize-model ') - // .description('optimize model') - // .action(async (workspace: string, cmd: { enable: string, disable: string, list: boolean }) => { - // console.log(JSON.stringify(cmd)) - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await optimizeModel(wsid, endpoint) - // }) - - // program - // .command('benchmark') - // .description('benchmark') - // .option('--from ', 'Min client count', '10') - // .option('--steps ', 'Step with client count', '10') - // .option('--sleep ', 'Random Delay max between operations', '0') - // .option('--binary ', 'Use binary data transfer', false) - // .option('--compression ', 'Use protocol compression', false) - // .option('--write ', 'Perform write operations', false) - // .option('--workspaces ', 'Workspaces to test on, comma separated', '') - // .option('--mode ', 'A benchmark mode. Supported values: `find-all`, `connect-only` ', 'find-all') - // .action( - // async (cmd: { - // from: string - // steps: string - // sleep: string - // workspaces: string - // binary: string - // compression: string - // write: string - // mode: 'find-all' | 'connect-only' - // }) => { - // await withAccountDatabase(async (db) => { - // console.log(JSON.stringify(cmd)) - // if (!['find-all', 'connect-only'].includes(cmd.mode)) { - // console.log('wrong mode') - // return - // } - - // const allWorkspacesPure = Array.from(await listWorkspacesPure(db)) - // const allWorkspaces = new Map(allWorkspacesPure.map((it) => [it.workspace, it])) - - // let workspaces = cmd.workspaces - // .split(',') - // .map((it) => it.trim()) - // .filter((it) => it.length > 0) - // .map((it) => getWorkspaceId(it)) - - // if (cmd.workspaces.length === 0) { - // workspaces = allWorkspacesPure.map((it) => getWorkspaceId(it.workspace)) - // } - // const accounts = new Map(Array.from(await listAccounts(db)).map((it) => [it._id.toString(), it.email])) - - // const accountWorkspaces = new Map() - // for (const ws of workspaces) { - // const wsInfo = allWorkspaces.get(ws.name) - // if (wsInfo !== undefined) { - // accountWorkspaces.set( - // ws.name, - // wsInfo.accounts.map((it) => accounts.get(it.toString()) as string) - // ) - // } - // } - // await benchmark(workspaces, accountWorkspaces, accountsUrl, { - // steps: parseInt(cmd.steps), - // from: parseInt(cmd.from), - // sleep: parseInt(cmd.sleep), - // binary: cmd.binary === 'true', - // compression: cmd.compression === 'true', - // write: cmd.write === 'true', - // mode: cmd.mode - // }) - // }) - // } - // ) - // program - // .command('benchmarkWorker') - // .description('benchmarkWorker') - // .action(async (cmd: any) => { - // console.log(JSON.stringify(cmd)) - // benchmarkWorker() - // }) - - // program - // .command('stress ') - // .description('stress benchmark') - // .option('--mode ', 'A benchmark mode. Supported values: `wrong`, `connect-disconnect` ', 'wrong') - // .action(async (transactor: string, cmd: { mode: StressBenchmarkMode }) => { - // await stressBenchmark(transactor, cmd.mode) - // }) - - // program - // .command('fix-skills-mongo ') - // .description('fix skills for workspace') - // .action(async (workspace: string, step: string) => { - // const mongodbUri = getMongoDBUrl() - // const wsid = getWorkspaceId(workspace) - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token) - // await fixSkills(mongodbUri, wsid, endpoint, step) - // }) - - // program - // .command('restore-ats-types-mongo ') - // .description('Restore recruiting task types for workspace') - // .action(async (workspace: string) => { - // const mongodbUri = getMongoDBUrl() - // console.log('Restoring recruiting task types in workspace ', workspace, '...') - // const wsid = getWorkspaceId(workspace) - // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - // await restoreRecruitingTaskTypes(mongodbUri, wsid, endpoint) - // }) - - // program - // .command('restore-ats-types-2-mongo ') - // .description('Restore recruiting task types for workspace 2') - // .action(async (workspace: string) => { - // const mongodbUri = getMongoDBUrl() - // console.log('Restoring recruiting task types in workspace ', workspace, '...') - // const wsid = getWorkspaceId(workspace) - // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - // await restoreHrTaskTypesFromUpdates(mongodbUri, wsid, endpoint) - // }) - program .command('change-field ') .description('change field value for the object') @@ -2471,149 +1148,6 @@ export function devTool ( console.log('done') }) - // program - // .command('remove-duplicates-ids-mongo ') - // .description('remove duplicates ids for futue migration') - // .action(async (workspaces: string) => { - // const mongodbUri = getMongoDBUrl() - // await withStorage(async (adapter) => { - // await removeDuplicateIds(toolCtx, mongodbUri, adapter, accountsUrl, workspaces) - // }) - // }) - - // program.command('move-to-pg ').action(async (region: string) => { - // const { dbUrl } = prepareTools() - // const mongodbUri = getMongoDBUrl() - - // await withAccountDatabase(async (db) => { - // const workspaces = await listWorkspacesRaw(db) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - // await moveFromMongoToPG( - // db, - // mongodbUri, - // dbUrl, - // workspaces.filter((p) => p.region !== region), - // region - // ) - // }) - // }) - - // program - // .command('move-workspace-to-pg ') - // .option('-i, --include ', 'A list of ; separated domain names to include during backup', '*') - // .option('-f|--force [force]', 'Force update', false) - // .action( - // async ( - // workspace: string, - // region: string, - // cmd: { - // include: string - // force: boolean - // } - // ) => { - // const { dbUrl } = prepareTools() - // const mongodbUri = getMongoDBUrl() - - // await withAccountDatabase(async (db) => { - // const ws = await getWorkspace(db, workspace) - // if (ws === null) { - // throw new Error(`workspace ${workspace} not found`) - // } - // await updateField(ws.uuid, await getWorkspaceTransactorEndpoint(ws.uuid), cmd) - // }) - // } - // ) - - // program - // .command('recreate-elastic-indexes-mongo ') - // .description('reindex workspace to elastic') - // .action(async (workspace: string) => { - // const mongodbUri = getMongoDBUrl() - // const wsid = getWorkspaceId(workspace) - // await recreateElastic(mongodbUri, wsid) - // }) - - // program - // .command('recreate-all-elastic-indexes-mongo') - // .description('reindex elastic') - // .action(async () => { - // const { dbUrl } = prepareTools() - // const mongodbUri = getMongoDBUrl() - - // await withAccountDatabase(async (db) => { - // const workspaces = await listWorkspacesRaw(db) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - // for (const workspace of workspaces) { - // const wsid = getWorkspaceId(workspace.workspace) - // await recreateElastic(mongodbUri ?? dbUrl, wsid) - // } - // }) - // }) - - // program - // .command('remove-duplicates-ids-mongo ') - // .description('remove duplicates ids for futue migration') - // .action(async (workspaces: string) => { - // const mongodbUri = getMongoDBUrl() - // await withStorage(async (adapter) => { - // await removeDuplicateIds(toolCtx, mongodbUri, adapter, accountsUrl, workspaces) - // }) - // }) - - // program.command('move-to-pg ').action(async (region: string) => { - // const { dbUrl } = prepareTools() - // const mongodbUri = getMongoDBUrl() - - // await withAccountDatabase(async (db) => { - // const workspaces = await listWorkspacesRaw(db) - // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - // await moveFromMongoToPG( - // db, - // mongodbUri, - // dbUrl, - // workspaces.filter((p) => p.region !== region), - // region - // ) - // }) - // }) - - // program - // .command('move-workspace-to-pg ') - // .option('-i, --include ', 'A list of ; separated domain names to include during backup', '*') - // .option('-f|--force [force]', 'Force update', false) - // .action( - // async ( - // workspace: string, - // region: string, - // cmd: { - // include: string - // force: boolean - // } - // ) => { - // const { dbUrl } = prepareTools() - // const mongodbUri = getMongoDBUrl() - - // await withAccountDatabase(async (db) => { - // const workspaceInfo = await getWorkspaceById(db, workspace) - // if (workspaceInfo === null) { - // throw new Error(`workspace ${workspace} not found`) - // } - // if (workspaceInfo.region === region && !cmd.force) { - // throw new Error(`workspace ${workspace} is already migrated`) - // } - // await moveWorkspaceFromMongoToPG( - // db, - // mongodbUri, - // dbUrl, - // workspaceInfo, - // region, - // cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())), - // cmd.force - // ) - // }) - // } - // ) - program.command('move-account-db-to-pg').action(async () => { const { dbUrl } = prepareTools() const mongodbUri = getMongoDBUrl() @@ -2718,123 +1252,6 @@ export function devTool ( }, dbUrl) }) - // program - // .command('perfomance') - // .option('-p, --parallel', '', false) - // .action(async (cmd: { parallel: boolean }) => { - // const { txes, version, migrateOperations } = prepareTools() - // await withAccountDatabase(async (db) => { - // const email = generateId() - // const ws = generateId() - // const wsid = getWorkspaceId(ws) - // const start = new Date() - // const measureCtx = new MeasureMetricsContext('create-workspace', {}) - // const wsInfo = await createWorkspaceRecord(measureCtx, db, null, email, ws, ws) - - // // update the record so it's not taken by one of the workers for the next 60 seconds - // await updateWorkspace(db, wsInfo, { - // mode: 'creating', - // progress: 0, - // lastProcessingTime: Date.now() + 1000 * 60 - // }) - - // await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations, undefined, true) - - // await updateWorkspace(db, wsInfo, { - // mode: 'active', - // progress: 100, - // disabled: false, - // version - // }) - // await createAcc(toolCtx, db, null, email, '1234', '', '', true) - // await assignAccountToWs(toolCtx, db, null, email, ws, AccountRole.User) - // console.log('Workspace created in', new Date().getTime() - start.getTime(), 'ms') - // const token = generateToken(systemAccountEmail, wsid) - // const endpoint = await getTransactorEndpoint(token, 'external') - // await generateWorkspaceData(endpoint, ws, cmd.parallel, email) - // await testFindAll(endpoint, ws, email) - // await dropWorkspace(toolCtx, db, null, ws) - // }) - // }) - - // program - // .command('reset-ws-attempts ') - // .description('Reset workspace creation/upgrade attempts counter') - // .action(async (workspace) => { - // await withAccountDatabase(async (db) => { - // const info = await getWorkspaceById(db, workspace) - // if (info === null) { - // throw new Error(`workspace ${workspace} not found`) - // } - - // await updateWorkspace(db, info, { - // attempts: 0 - // }) - - // console.log('Attempts counter for workspace', workspace, 'has been reset') - // }) - // }) - - // program - // .command('add-controlled-doc-rank-mongo') - // .description('add rank to controlled documents') - // .option('-w, --workspace ', 'Selected workspace only', '') - // .action(async (cmd: { workspace: string }) => { - // const { version } = prepareTools() - - // let workspaces: Workspace[] = [] - // await withAccountDatabase(async (db) => { - // workspaces = await listWorkspacesPure(db) - // workspaces = workspaces - // .filter((p) => isActiveMode(p.mode)) - // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // .sort((a, b) => b.lastVisit - a.lastVisit) - // }) - - // console.log('found workspaces', workspaces.length) - - // const mongodbUri = getMongoDBUrl() - // const client = getMongoClient(mongodbUri) - // const _client = await client.getClient() - - // try { - // const count = workspaces.length - // let index = 0 - // for (const workspace of workspaces) { - // index++ - - // toolCtx.info('processing workspace', { - // workspace: workspace.workspace, - // version: workspace.version, - // index, - // count - // }) - - // if (workspace.version === undefined || !deepEqual(workspace.version, version)) { - // console.log(`upgrade to ${versionToString(version)} is required`) - // continue - // } - // const workspaceId = getWorkspaceId(workspace.workspace) - // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - // await addControlledDocumentRank(toolCtx, wsDb, workspaceId) - // } - // } finally { - // client.close() - // } - // }) - - // Not needed anymore? - // program - // .command('fill-github-users') - // .option('-t, --token ', 'Github token to increase the limit of requests to GitHub') - // .description('adds github username info to all accounts') - // .action(async (cmd: { token?: string }) => { - // await withAccountDatabase(async (db) => { - // await fillGithubUsers(toolCtx, db, cmd.token) - // }) - // }) - program .command('migrate-github-account') .option('--db ', 'Github DB', '%github') From 9b607d36ea7aeee1003578bea5e6b4c51d459abb Mon Sep 17 00:00:00 2001 From: Artyom Savchenko Date: Tue, 28 Jul 2026 15:56:18 +0700 Subject: [PATCH 5/7] Fix cross-workspace blob/thumbnail and metadata disclosure (#10972) * Fix cross-workspace blob/thumbnail and metadata disclosure The preview pod's thumbnail and metadata routes did not validate the caller's token against the workspace taken from the URL. Reported-by: Ugur Ozer, Aeon AI Risk Management Signed-off-by: Artyom Savchenko * Update copyright year Signed-off-by: Artyom Savchenko * Use secure mode Signed-off-by: Artyom Savchenko * Fix formatting Signed-off-by: Artyom Savchenko * Remove uuid check Signed-off-by: Artyom Savchenko --------- Signed-off-by: Artyom Savchenko --- pods/preview/src/__tests__/middleware.test.ts | 140 ++++++++++++++++++ pods/preview/src/middleware.ts | 34 +++++ pods/preview/src/server.ts | 6 +- 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 pods/preview/src/__tests__/middleware.test.ts diff --git a/pods/preview/src/__tests__/middleware.test.ts b/pods/preview/src/__tests__/middleware.test.ts new file mode 100644 index 0000000000..e8687da2f1 --- /dev/null +++ b/pods/preview/src/__tests__/middleware.test.ts @@ -0,0 +1,140 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type AccountUuid, systemAccountUuid, type WorkspaceUuid } from '@hcengineering/core' +import { extractToken } from '@hcengineering/server-client' +import { type Token } from '@hcengineering/server-token' +import { type NextFunction, type Response } from 'express' + +import { HttpError } from '../error' +import { type RequestWithAuth, withAuthorization, withBlob } from '../middleware' + +jest.mock('@hcengineering/server-client', () => ({ + extractToken: jest.fn() +})) + +const extractTokenMock = extractToken as jest.MockedFunction + +const workspaceA = '00000000-0000-4000-8000-00000000000a' as WorkspaceUuid +const workspaceB = '00000000-0000-4000-8000-00000000000b' as WorkspaceUuid +const account = '00000000-0000-4000-8000-0000000000ac' as AccountUuid + +function makeToken (token: Partial): Token { + const result: Token = { account, workspace: workspaceA, extra: {} } + return { ...result, ...token } +} + +function makeRequest (workspace: string, name: string, token?: Token): RequestWithAuth { + return { headers: {}, params: { workspace, name }, token } as unknown as RequestWithAuth +} + +const res = {} as unknown as Response + +describe('withAuthorization', () => { + beforeEach(() => { + extractTokenMock.mockReset() + }) + + it('rejects requests without a token', () => { + extractTokenMock.mockReturnValue(undefined) + const next = jest.fn() as unknown as NextFunction + + withAuthorization(makeRequest(workspaceA, 'blob'), res, next) + + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 401 })) + }) + + it('rejects guest and readonly tokens', () => { + for (const extra of [{ guest: 'true' }, { readonly: 'true' }]) { + extractTokenMock.mockReturnValue(makeToken({ extra })) + const next = jest.fn() as unknown as NextFunction + + withAuthorization(makeRequest(workspaceA, 'blob'), res, next) + + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 401 })) + } + }) + + it('attaches a valid token to the request', () => { + const token = makeToken({}) + extractTokenMock.mockReturnValue(token) + const req = makeRequest(workspaceA, 'blob') + const next = jest.fn() as unknown as NextFunction + + withAuthorization(req, res, next) + + expect(req.token).toBe(token) + expect(next).toHaveBeenCalledWith() + }) +}) + +describe('withBlob', () => { + it('rejects a missing workspace', () => { + const next = jest.fn() as unknown as NextFunction + withBlob(makeRequest('', 'blob'), res, next) + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 400 })) + }) + + it('accepts a non-uuid workspace id when the token matches it', () => { + const workspace = 'not-a-uuid' + const next = jest.fn() as unknown as NextFunction + + withBlob(makeRequest(workspace, 'blob', makeToken({ workspace: workspace as WorkspaceUuid })), res, next) + + expect(next).toHaveBeenCalledWith() + }) + + it('rejects a missing blob name', () => { + const next = jest.fn() as unknown as NextFunction + withBlob(makeRequest(workspaceA, ''), res, next) + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 400 })) + }) + + it('rejects a token scoped to another workspace', () => { + const next = jest.fn() as unknown as NextFunction + + withBlob(makeRequest(workspaceB, 'blob', makeToken({ workspace: workspaceA })), res, next) + + expect(next).toHaveBeenCalledWith(expect.any(HttpError)) + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 401 })) + }) + + it('allows a token scoped to the requested workspace', () => { + const next = jest.fn() as unknown as NextFunction + + withBlob(makeRequest(workspaceA, 'blob', makeToken({ workspace: workspaceA })), res, next) + + expect(next).toHaveBeenCalledWith() + }) + + it('allows the system account and admins to access any workspace', () => { + for (const token of [ + makeToken({ account: systemAccountUuid, workspace: workspaceA }), + makeToken({ workspace: workspaceA, extra: { admin: 'true' } }) + ]) { + const next = jest.fn() as unknown as NextFunction + withBlob(makeRequest(workspaceB, 'blob', token), res, next) + expect(next).toHaveBeenCalledWith() + } + }) + + it('rejects requests with no token attached', () => { + const next = jest.fn() as unknown as NextFunction + + withBlob(makeRequest(workspaceA, 'blob'), res, next) + + expect(next).toHaveBeenCalledWith(expect.objectContaining({ code: 401 })) + }) +}) diff --git a/pods/preview/src/middleware.ts b/pods/preview/src/middleware.ts index 0d8b0e0187..017a053a1c 100644 --- a/pods/preview/src/middleware.ts +++ b/pods/preview/src/middleware.ts @@ -66,6 +66,40 @@ export const withAuthorization = (req: RequestWithAuth, res: Response, next: Nex } } +/** + * Validates blob route params and ensures the caller's token grants access to + * the workspace taken from the URL. Must run after `withAuthorization`, which + * guarantees a token is present. + */ +export const withBlob = (req: RequestWithAuth, res: Response, next: NextFunction): void => { + try { + const workspace = req.params.workspace + const name = req.params.name + + if (workspace === undefined || workspace === '') { + throw new HttpError(400, 'Missing workspace') + } + if (name === undefined || name === '') { + throw new HttpError(400, 'Missing blob name') + } + + const token = req.token + if (token == null) { + throw new HttpError(401, 'Unauthorized') + } + + const hasWorkspaceAccess = + (token.workspace as string) === workspace || token.account === systemAccountUuid || token.extra?.admin === 'true' + if (!hasWorkspaceAccess) { + throw new HttpError(401, 'Unauthorized') + } + + next() + } catch (err: any) { + next(err) + } +} + export interface ErrorHandlerOptions { ctx: MeasureContext } diff --git a/pods/preview/src/server.ts b/pods/preview/src/server.ts index 18026a42f1..32fa7d67f0 100644 --- a/pods/preview/src/server.ts +++ b/pods/preview/src/server.ts @@ -29,7 +29,7 @@ import { pipeline } from 'stream/promises' import { createCache } from './cache' import { type Config } from './config' -import { type RequestWithAuth, errorHandler, keepAlive } from './middleware' +import { type RequestWithAuth, errorHandler, keepAlive, withAuthorization, withBlob } from './middleware' import { createPreviewService, ThumbnailParams } from './service' import { TemporaryDir } from './tempdir' @@ -174,6 +174,8 @@ export async function createServer (ctx: MeasureContext, config: Config): Promis app.get( '/metadata/:workspace/:name', + withAuthorization, + withBlob, wrapRequest(ctx, 'getMetadata', async (ctx, req, res) => { const workspace = req.params.workspace as WorkspaceUuid const name = req.params.name @@ -185,6 +187,8 @@ export async function createServer (ctx: MeasureContext, config: Config): Promis app.get( '/image/:transform/:workspace/:name', + withAuthorization, + withBlob, wrapRequest(ctx, 'getThumbnail', async (ctx, req, res) => { const workspace = req.params.workspace as WorkspaceUuid const name = req.params.name From b0566673720bdc23758ef1f1fd1df0bdabc8eda2 Mon Sep 17 00:00:00 2001 From: Artyom Savchenko Date: Sat, 1 Aug 2026 07:35:24 +0700 Subject: [PATCH 6/7] Fix rekoni temp files clean up (#11000) * Fix rekoni files clean up Signed-off-by: Artyom Savchenko * Fix formatting Signed-off-by: Artyom Savchenko * Fix copyright Signed-off-by: Artyom Savchenko --------- Signed-off-by: Artyom Savchenko --- .../rekoni/src/__tests__/tempfile.test.ts | 53 ++++++++++++++++++ services/rekoni/src/extractors/doc.ts | 54 +++++++++---------- services/rekoni/src/extractors/pdf.ts | 27 ++++------ services/rekoni/src/extractors/rtf.ts | 32 ++++++----- services/rekoni/src/tempfile.ts | 42 +++++++++++++++ 5 files changed, 144 insertions(+), 64 deletions(-) create mode 100644 services/rekoni/src/__tests__/tempfile.test.ts create mode 100644 services/rekoni/src/tempfile.ts diff --git a/services/rekoni/src/__tests__/tempfile.test.ts b/services/rekoni/src/__tests__/tempfile.test.ts new file mode 100644 index 0000000000..158843d9ad --- /dev/null +++ b/services/rekoni/src/__tests__/tempfile.test.ts @@ -0,0 +1,53 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { stat } from 'fs/promises' +import { withTempFile } from '../tempfile' + +async function exists (path: string): Promise { + try { + await stat(path) + return true + } catch { + return false + } +} + +describe('withTempFile', () => { + it('removes the temp dir after a successful run', async () => { + let capturedDir = '' + const result = await withTempFile('content.txt', Buffer.from('hello'), async (filePath, tempDir) => { + capturedDir = tempDir + expect(await exists(filePath)).toBe(true) + return 'ok' + }) + + expect(result).toBe('ok') + expect(await exists(capturedDir)).toBe(false) + }) + + it('removes the temp dir even when the run callback throws', async () => { + let capturedDir = '' + + await expect( + withTempFile('content.txt', Buffer.from('hello'), async (_filePath, tempDir) => { + capturedDir = tempDir + throw new Error('boom') + }) + ).rejects.toThrow('boom') + + expect(capturedDir).not.toBe('') + expect(await exists(capturedDir)).toBe(false) + }) +}) diff --git a/services/rekoni/src/extractors/doc.ts b/services/rekoni/src/extractors/doc.ts index 1df888886d..510d08adb5 100644 --- a/services/rekoni/src/extractors/doc.ts +++ b/services/rekoni/src/extractors/doc.ts @@ -1,8 +1,6 @@ import { exec } from 'child_process' -import { mkdtemp, rm, rmdir, writeFile } from 'fs/promises' import { contentType } from 'mime-types' -import { tmpdir } from 'os' -import { join } from 'path' +import { withTempFile } from '../tempfile' import { rtfExtractor } from './rtf' import { DocumentExtractor } from './types' @@ -18,36 +16,32 @@ export const docExtractor: DocumentExtractor = { }, async extract (fileName: string, type: string, data): Promise { - const tempDir = await mkdtemp(join(tmpdir(), 'rekoni-')) - const distFileName = join(tempDir, 'content.doc') - await writeFile(distFileName, data) - const text = await new Promise((resolve, reject) => { - exec( - `antiword -i 1 -f -m UTF-8 "${distFileName}"`, - { encoding: 'utf-8', cwd: tempDir }, - (error, stdout, stderr) => { - if (error != null) { - if (stderr.includes('is not a Word Document. It is probably a Rich Text Format file')) { - rtfExtractor - .extract(fileName, type, data) - .then((value) => { - resolve(value) - }) - .catch((err) => { - reject(err) - }) - return + return await withTempFile('content.doc', data, async (distFileName, tempDir) => { + return await new Promise((resolve, reject) => { + exec( + `antiword -i 1 -f -m UTF-8 "${distFileName}"`, + { encoding: 'utf-8', cwd: tempDir }, + (error, stdout, stderr) => { + if (error != null) { + if (stderr.includes('is not a Word Document. It is probably a Rich Text Format file')) { + rtfExtractor + .extract(fileName, type, data) + .then((value) => { + resolve(value) + }) + .catch((err) => { + reject(err) + }) + return + } + reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) + } else { + resolve(stdout) } - reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) - } else { - resolve(stdout) } - } - ) + ) + }) }) - await rm(distFileName) - await rmdir(tempDir) - return text } } function isType (type: string): boolean { diff --git a/services/rekoni/src/extractors/pdf.ts b/services/rekoni/src/extractors/pdf.ts index ffd55c6736..cdff4b84b5 100644 --- a/services/rekoni/src/extractors/pdf.ts +++ b/services/rekoni/src/extractors/pdf.ts @@ -1,8 +1,6 @@ import { exec } from 'child_process' -import { mkdtemp, rm, rmdir, writeFile } from 'fs/promises' import { contentType } from 'mime-types' -import { tmpdir } from 'os' -import { join } from 'path' +import { withTempFile } from '../tempfile' import { DocumentExtractor } from './types' export const pdfExtractor: DocumentExtractor = { @@ -22,21 +20,16 @@ export const pdfExtractor: DocumentExtractor = { }, async extract (fileName: string, type: string, data): Promise { - const tempDir = await mkdtemp(join(tmpdir(), 'rekoni-')) - const distFileName = join(tempDir, 'content.pdf') - await writeFile(distFileName, data) - - const text = await new Promise((resolve, reject) => { - exec(`pdftotext -layout "${distFileName}" -`, { encoding: 'utf-8', cwd: tempDir }, (error, stdout, stderr) => { - if (error != null) { - reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) - } else { - resolve(stdout) - } + return await withTempFile('content.pdf', data, async (distFileName, tempDir) => { + return await new Promise((resolve, reject) => { + exec(`pdftotext -layout "${distFileName}" -`, { encoding: 'utf-8', cwd: tempDir }, (error, stdout, stderr) => { + if (error != null) { + reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) + } else { + resolve(stdout) + } + }) }) }) - await rm(distFileName) - await rmdir(tempDir) - return text } } diff --git a/services/rekoni/src/extractors/rtf.ts b/services/rekoni/src/extractors/rtf.ts index 896d52fb77..7825a4ca4a 100644 --- a/services/rekoni/src/extractors/rtf.ts +++ b/services/rekoni/src/extractors/rtf.ts @@ -1,8 +1,6 @@ import { exec } from 'child_process' -import { mkdtemp, rm, rmdir, writeFile } from 'fs/promises' import { contentType } from 'mime-types' -import { tmpdir } from 'os' -import { join } from 'path' +import { withTempFile } from '../tempfile' import { convertString } from './html' import { DocumentExtractor } from './types' @@ -18,23 +16,23 @@ export const rtfExtractor: DocumentExtractor = { }, async extract (fileName: string, type: string, data): Promise { - const tempDir = await mkdtemp(join(tmpdir(), 'rekoni-')) - const distFileName = join(tempDir, 'content.rtf') - await writeFile(distFileName, data) - const htmlText = await new Promise((resolve, reject) => { - exec(`unrtf --nopict --html "${distFileName}"`, { encoding: 'utf-8', cwd: tempDir }, (error, stdout, stderr) => { - if (error != null) { - reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) - } else { - resolve(stdout) - } + const htmlText = await withTempFile('content.rtf', data, async (distFileName, tempDir) => { + return await new Promise((resolve, reject) => { + exec( + `unrtf --nopict --html "${distFileName}"`, + { encoding: 'utf-8', cwd: tempDir }, + (error, stdout, stderr) => { + if (error != null) { + reject(new Error(`Error ${JSON.stringify(error)} ${stderr}`)) + } else { + resolve(stdout) + } + } + ) }) }) - const text = convertString(htmlText) - await rm(distFileName) - await rmdir(tempDir) - return text + return convertString(htmlText) } } function isType (type: string): boolean { diff --git a/services/rekoni/src/tempfile.ts b/services/rekoni/src/tempfile.ts new file mode 100644 index 0000000000..c7ede4840b --- /dev/null +++ b/services/rekoni/src/tempfile.ts @@ -0,0 +1,42 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { mkdtemp, rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' + +/** + * Writes `data` into a fresh temp directory under a file named `fileName`, runs `run` against it, + * and guarantees the whole temp directory is removed afterwards — whether `run` succeeds or throws. + * + * Extractors shell out to external binaries (pdftotext, antiword, unrtf) that can fail on malformed + * or unsupported input. Without a `finally`-guaranteed cleanup, a failed extraction leaks its temp + * directory (including the uploaded file content) for the lifetime of the host. + * + * @public + */ +export async function withTempFile ( + fileName: string, + data: Buffer, + run: (filePath: string, tempDir: string) => Promise +): Promise { + const tempDir = await mkdtemp(join(tmpdir(), 'rekoni-')) + try { + const filePath = join(tempDir, fileName) + await writeFile(filePath, data) + return await run(filePath, tempDir) + } finally { + await rm(tempDir, { recursive: true, force: true }) + } +} From 8c9df54eb04114b1d394502959498050dde37afd Mon Sep 17 00:00:00 2001 From: Artyom Savchenko Date: Sat, 1 Aug 2026 07:35:39 +0700 Subject: [PATCH 7/7] Add script to verify all backup data is present in a workspace (#10999) * Add script to verify all backup data is present in a workspace Signed-off-by: Artyom Savchenko * Verify blobs exist Signed-off-by: Artyom Savchenko * Fix formatting Signed-off-by: Artyom Savchenko --------- Signed-off-by: Artyom Savchenko --- dev/tool/src/index.ts | 68 +++++ .../src/__tests__/checkWorkspace.spec.ts | 83 ++++++ server/backup/src/check.ts | 254 ++++++++++++++++++ server/backup/src/index.ts | 1 + server/backup/src/utils.ts | 45 ++++ 5 files changed, 451 insertions(+) create mode 100644 server/backup/src/__tests__/checkWorkspace.spec.ts create mode 100644 server/backup/src/check.ts diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 986f9c2d33..b347af1b33 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -46,6 +46,7 @@ import { backupDownload, backupFind, checkBackupIntegrity, + checkWorkspaceBackup, compactBackup, createFileBackupStorage, createStorageBackupStorage, @@ -639,6 +640,73 @@ export function devTool ( await storageAdapter.close() }) + program + .command('backup-check-workspace [date]') + .description( + 'Check whether all data from a backup is present in the workspace. Read-only: no data is added, removed or changed.' + ) + .action(async (dirName: string, workspaceId: string, date: string | undefined) => { + await withAccountDatabase(async (db) => { + const { txes, dbUrl } = prepareTools() + const ws = await getWorkspace(db, workspaceId) + if (ws === null) { + throw new Error(`workspace ${workspaceId} not found`) + } + + const wsIds = { uuid: ws.uuid, dataId: ws.dataId, url: ws.url ?? '' } + const storage = await createFileBackupStorage(dirName) + const workspaceStorage: StorageAdapter = buildStorageFromConfig(storageConfigFromEnv()) + + let pipeline: Pipeline | undefined + try { + pipeline = await createBackupPipeline(toolCtx, dbUrl, txes, { + externalStorage: workspaceStorage, + usePassedCtx: true + })( + toolCtx, + { + uuid: ws.uuid, + url: ws.url ?? '', + dataId: ws.dataId + }, + createEmptyBroadcastOps(), + null + ) + if (pipeline === undefined) { + toolCtx.error('failed to check, pipeline is undefined', { workspaceId }) + process.exitCode = 1 + return + } + + const result = await checkWorkspaceBackup(toolCtx, pipeline, wsIds, storage, parseInt(date ?? '-1')) + + console.log('') + for (const d of result.domains) { + const ok = d.missing.length === 0 && d.modified.length === 0 + console.log( + `${ok ? 'OK ' : 'FAIL'} ${d.domain}: backup=${d.backupCount} workspace=${d.workspaceCount} missing=${d.missing.length} modified=${d.modified.length}` + ) + } + console.log( + `${result.blobs.ok ? 'OK ' : 'FAIL'} blobs (storage): total=${result.blobs.total} missing=${result.blobs.missing.length}` + ) + console.log('') + if (result.ok) { + console.log('OK: workspace contains all data from backup') + } else { + console.log('FAILED: workspace is missing data present in the backup') + process.exitCode = 1 + } + } catch (err: any) { + toolCtx.error('failed to check workspace against backup', { err, workspaceId }) + process.exitCode = 1 + } finally { + await pipeline?.close() + await workspaceStorage?.close() + } + }) + }) + program .command('validate-workspace ') .description('Validate a (restored) workspace: connect as system, check model, data counts and blob download') diff --git a/server/backup/src/__tests__/checkWorkspace.spec.ts b/server/backup/src/__tests__/checkWorkspace.spec.ts new file mode 100644 index 0000000000..be98044b19 --- /dev/null +++ b/server/backup/src/__tests__/checkWorkspace.spec.ts @@ -0,0 +1,83 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { compareDomainDigest, findMissingBlobs } from '../utils' + +describe('compareDomainDigest', () => { + it('reports nothing when workspace fully matches backup', () => { + const backup = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + const workspace = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] }) + }) + + it('reports documents present in backup but absent from workspace as missing', () => { + const backup = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + const workspace = new Map([['doc1', 'hash1']]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: ['doc2'], modified: [] }) + }) + + it('reports documents with a different hash as modified, not missing', () => { + const backup = new Map([['doc1', 'hash1']]) + const workspace = new Map([['doc1', 'hash1-changed']]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: ['doc1'] }) + }) + + it('ignores documents present in workspace but not in backup', () => { + const backup = new Map([['doc1', 'hash1']]) + const workspace = new Map([ + ['doc1', 'hash1'], + ['doc2', 'hash2'] + ]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] }) + }) + + it('treats quoted and unquoted equal hashes as the same (matches restore hash trimming)', () => { + const backup = new Map([['doc1', '"hash1"']]) + const workspace = new Map([['doc1', 'hash1']]) + expect(compareDomainDigest(backup, workspace)).toEqual({ missing: [], modified: [] }) + }) + + it('returns an empty result for an empty backup digest', () => { + const workspace = new Map([['doc1', 'hash1']]) + expect(compareDomainDigest(new Map(), workspace)).toEqual({ missing: [], modified: [] }) + }) +}) + +describe('findMissingBlobs', () => { + it('returns nothing when every backup blob exists in storage', () => { + expect(findMissingBlobs(['blob1', 'blob2'], new Set(['blob1', 'blob2', 'blob3']))).toEqual([]) + }) + + it('reports backup blobs absent from storage', () => { + expect(findMissingBlobs(['blob1', 'blob2'], new Set(['blob1']))).toEqual(['blob2']) + }) + + it('reports all backup blobs when storage is empty', () => { + expect(findMissingBlobs(['blob1', 'blob2'], new Set())).toEqual(['blob1', 'blob2']) + }) + + it('returns nothing for an empty list of backup blobs', () => { + expect(findMissingBlobs([], new Set(['blob1']))).toEqual([]) + }) +}) diff --git a/server/backup/src/check.ts b/server/backup/src/check.ts new file mode 100644 index 0000000000..9bb7edccb9 --- /dev/null +++ b/server/backup/src/check.ts @@ -0,0 +1,254 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { + Doc, + Domain, + DOMAIN_BLOB, + MeasureContext, + Ref, + type Blob, + type LowLevelStorage, + type WorkspaceIds +} from '@hcengineering/core' +import { BackupClientOps, createDummyStorageAdapter, type Pipeline } from '@hcengineering/server-core' +import { gunzipSync } from 'zlib' +import { BackupStorage } from './storage' +import type { BackupDocId, BackupInfo, BackupSnapshot } from './types' +import { compareDomainDigest, findMissingBlobs, isAccountDomain, loadDigest } from './utils' +export * from './storage' + +/** + * @public + */ +export interface DomainCheckResult { + domain: Domain + backupCount: number + workspaceCount: number + missing: BackupDocId[] + modified: BackupDocId[] +} + +/** + * @public + */ +export interface BlobCheckResult { + total: number + missing: Ref[] + ok: boolean +} + +/** + * @public + */ +export interface WorkspaceCheckResult { + date: number + domains: DomainCheckResult[] + blobs: BlobCheckResult + ok: boolean +} + +async function resolveSnapshots ( + storage: BackupStorage, + date: number +): Promise<{ backupInfo: BackupInfo, snapshots: BackupSnapshot[], date: number }> { + const infoFile = 'backup.json.gz' + if (!(await storage.exists(infoFile))) { + throw new Error(`${infoFile} should present to check`) + } + const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) + + let snapshots = backupInfo.snapshots + if (date !== -1) { + const bk = backupInfo.snapshots.findIndex((it) => it.date === date) + if (bk === -1) { + throw new Error(`${infoFile} has no snapshot at ${date}`) + } + snapshots = backupInfo.snapshots.slice(0, bk + 1) + } else { + date = snapshots[snapshots.length - 1]?.date ?? -1 + } + return { backupInfo, snapshots, date } +} + +/** + * Checks whether all documents recorded in a backup are present, and unchanged, in the given + * workspace's document domains, and whether every backed-up blob's content exists in blob + * storage (see {@link checkWorkspaceBlobs}). + * + * This is read-only: nothing is uploaded, removed, or otherwise modified in either the workspace + * or the backup. It is meant as a diagnostic to run before trusting a backup (or after a restore) + * — to find out if the workspace is missing data the backup has, without acting on it. + * + * Account domains (person/socialId) are skipped, since they live in the account database rather + * than in the workspace's own domains and can't be checked against `pipeline.context.lowLevelStorage`. + * + * @param date optional snapshot date to check against, defaults to the latest snapshot (-1). + * @public + */ +export async function checkWorkspaceBackup ( + ctx: MeasureContext, + pipeline: Pipeline, + wsIds: WorkspaceIds, + storage: BackupStorage, + date: number = -1 +): Promise { + const resolved = await resolveSnapshots(storage, date) + const snapshots = resolved.snapshots + date = resolved.date + + ctx.info('checking workspace against backup', { workspace: wsIds.uuid, date }) + + const domains = new Set() + for (const s of snapshots) { + Object.keys(s.domains).forEach((it) => domains.add(it as Domain)) + } + + const connection = pipeline.context.lowLevelStorage as LowLevelStorage + const ops = new BackupClientOps(connection) + + const results: DomainCheckResult[] = [] + + for (const domain of domains) { + if (isAccountDomain(domain)) { + continue + } + + ctx.info('checking domain', { domain }) + const backupDigest = (await loadDigest(ctx, storage, snapshots, domain, date)) as Map, string> + + const workspaceDigest = new Map, string>() + let idx: number | undefined + try { + while (true) { + const it = await ops.loadChunk(ctx, domain, idx) + idx = it.idx + for (const { id, hash } of it.docs) { + workspaceDigest.set(id as Ref, hash) + } + if (it.finished) { + break + } + } + } finally { + if (idx !== undefined) { + await ops.closeChunk(ctx, idx) + } + } + + const { missing, modified } = compareDomainDigest(backupDigest, workspaceDigest) + + const result: DomainCheckResult = { + domain, + backupCount: backupDigest.size, + workspaceCount: workspaceDigest.size, + missing, + modified + } + results.push(result) + + if (missing.length > 0 || modified.length > 0) { + ctx.warn('backup data not fully present in workspace', { + domain, + backupCount: result.backupCount, + workspaceCount: result.workspaceCount, + missing: missing.length, + modified: modified.length, + sampleMissing: missing.slice(0, 10), + sampleModified: modified.slice(0, 10) + }) + } else { + ctx.info('domain ok', { domain, count: result.backupCount }) + } + } + + const blobs = await checkWorkspaceBlobs(ctx, pipeline, wsIds, storage, date, snapshots) + + const ok = results.every((it) => it.missing.length === 0 && it.modified.length === 0) && blobs.ok + + ctx.info('check complete', { + workspace: wsIds.uuid, + ok, + domains: results.length, + missing: results.reduce((sum, it) => sum + it.missing.length, 0), + modified: results.reduce((sum, it) => sum + it.modified.length, 0), + missingBlobs: blobs.missing.length + }) + + return { date, domains: results, blobs, ok } +} + +/** + * Checks whether every blob recorded in a backup actually has its content present in the + * workspace's blob storage (S3/minio/datalake), as opposed to just a metadata record in + * `DOMAIN_BLOB`. + * + * Read-only: only lists and stats existing blobs, never uploads or removes anything. + * + * @param date optional snapshot date to check against, defaults to the latest snapshot (-1). + * @param snapshots pre-resolved snapshots, to avoid re-reading `backup.json.gz` when called from + * {@link checkWorkspaceBackup}. If omitted, it's resolved from `storage`/`date`. + * @public + */ +export async function checkWorkspaceBlobs ( + ctx: MeasureContext, + pipeline: Pipeline, + wsIds: WorkspaceIds, + storage: BackupStorage, + date: number = -1, + snapshots?: BackupSnapshot[] +): Promise { + if (snapshots === undefined) { + const resolved = await resolveSnapshots(storage, date) + snapshots = resolved.snapshots + date = resolved.date + } + + ctx.info('checking blobs against backup', { workspace: wsIds.uuid, date }) + + const backupDigest = await loadDigest(ctx, storage, snapshots, DOMAIN_BLOB, date) + + const storageAdapter = pipeline.context.storageAdapter ?? createDummyStorageAdapter() + const existingBlobIds = new Set() + const iterator = await storageAdapter.listStream(ctx, wsIds) + try { + while (true) { + const batch = await iterator.next() + if (batch.length === 0) { + break + } + for (const b of batch) { + existingBlobIds.add(b._id) + } + } + } finally { + await iterator.close() + } + + const missing = findMissingBlobs(backupDigest.keys(), existingBlobIds) as Ref[] + const ok = missing.length === 0 + + if (ok) { + ctx.info('blobs ok', { total: backupDigest.size }) + } else { + ctx.warn('backup blobs missing from storage', { + total: backupDigest.size, + missing: missing.length, + sampleMissing: missing.slice(0, 10) + }) + } + + return { total: backupDigest.size, missing, ok } +} diff --git a/server/backup/src/index.ts b/server/backup/src/index.ts index b2cd2bf630..43ea32b91a 100644 --- a/server/backup/src/index.ts +++ b/server/backup/src/index.ts @@ -13,6 +13,7 @@ // limitations under the License. // export * from './backup' +export * from './check' export * from './restore' export * from './service' export * from './types' diff --git a/server/backup/src/utils.ts b/server/backup/src/utils.ts index 5316d82521..149c5136cc 100644 --- a/server/backup/src/utils.ts +++ b/server/backup/src/utils.ts @@ -941,6 +941,51 @@ export function doTrimHash (s: string | undefined): string | undefined { return s } +/** + * Compares a per-domain digest reconstructed from a backup with a digest read from a live + * workspace and reports the difference from the backup's point of view. + * + * - `missing` — documents present in the backup but absent from the workspace. + * - `modified` — documents present in both, but with a different content hash (the workspace + * version diverged from the backed-up one). + * + * Documents present in the workspace but not in the backup are intentionally not reported here: + * this check only answers "is everything from the backup present in the workspace", not the + * reverse. + * @public + */ +export function compareDomainDigest ( + backupDigest: Map, + workspaceDigest: Map +): { missing: BackupDocId[], modified: BackupDocId[] } { + const missing: BackupDocId[] = [] + const modified: BackupDocId[] = [] + for (const [id, hash] of backupDigest) { + const workspaceHash = workspaceDigest.get(id) + if (workspaceHash === undefined) { + missing.push(id) + } else if (doTrimHash(workspaceHash) !== doTrimHash(hash)) { + modified.push(id) + } + } + return { missing, modified } +} + +/** + * Finds blob ids that are recorded in a backup but do not exist in the workspace's blob storage + * (e.g. S3/minio/datalake), as opposed to just the blob metadata record in a domain. + * @public + */ +export function findMissingBlobs (backupBlobIds: Iterable, existingBlobIds: Set): BackupDocId[] { + const missing: BackupDocId[] = [] + for (const id of backupBlobIds) { + if (!existingBlobIds.has(id as string)) { + missing.push(id) + } + } + return missing +} + export async function loadDigest ( ctx: MeasureContext, storage: BackupStorage,