From 7b497c93c1ead341df91ed62e793e4a25bf6377c Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Fri, 13 Dec 2024 00:20:35 +0700 Subject: [PATCH] UBERF-7670: Per region moves (#7444) Signed-off-by: Andrey Sobolev --- .gitignore | 3 +- .vscode/launch.json | 48 ++- desktop/src/main/backup.ts | 2 +- dev/.env | 2 + dev/docker-compose.yaml | 110 ++--- dev/tool/src/index.ts | 382 ++++++++---------- packages/core/src/classes.ts | 70 +++- .../src/components/AdminWorkspaces.svelte | 264 ++++++++++++ .../src/components/LoginApp.svelte | 111 ++--- .../src/components/SelectWorkspace.svelte | 14 +- plugins/login-resources/src/index.ts | 1 + plugins/login-resources/src/utils.ts | 111 ++++- .../src/components/SelectWorkspaceMenu.svelte | 3 +- server/account/src/collections/mongo.ts | 34 +- server/account/src/collections/postgres.ts | 28 +- server/account/src/operations.ts | 236 ++++++++++- server/account/src/types.ts | 8 +- server/account/src/utils.ts | 28 +- server/backup-service/src/index.ts | 3 +- server/backup/src/backup.ts | 115 ++++-- server/backup/src/service.ts | 85 +++- server/client/src/account.ts | 15 +- server/core/src/adapter.ts | 8 + server/mongo/src/index.ts | 20 + server/postgres/src/index.ts | 34 +- server/server-pipeline/src/pipeline.ts | 15 +- server/server/src/sessionManager.ts | 13 +- server/workspace-service/package.json | 3 +- server/workspace-service/src/index.ts | 34 +- server/workspace-service/src/service.ts | 245 ++++++++++- server/ws/src/server_http.ts | 4 +- services/github/pod-github/src/platform.ts | 5 +- tests/.env | 3 +- tests/docker-compose.yaml | 1 - tests/shutdown.sh | 2 + 35 files changed, 1597 insertions(+), 463 deletions(-) create mode 100644 plugins/login-resources/src/components/AdminWorkspaces.svelte create mode 100755 tests/shutdown.sh diff --git a/.gitignore b/.gitignore index d1a56c1f95..3c21ad7f29 100644 --- a/.gitignore +++ b/.gitignore @@ -103,4 +103,5 @@ bundle.js.map tests/profiles **/bundle/model.json .wrangler -dump \ No newline at end of file +dump +**/logs/** \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 6823fd564f..548cf05872 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -107,9 +107,9 @@ "DB_URL": "mongodb://localhost:27017", // "DB_URL": "postgresql://postgres:example@localhost:5432", "SERVER_SECRET": "secret", - "TRANSACTOR_URL": "ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg", + "REGION_INFO":"|Mongo;pg|Postgres;cockroach|CockroachDB", + "TRANSACTOR_URL": "ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg,ws://host.docker.internal:3332;;cockroach", "ACCOUNTS_URL": "http://localhost:3000", - "REGION_INFO": "|Mongo;pg|Postgree", "ACCOUNT_PORT": "3000", "FRONT_URL": "http://localhost:8080", "STATS_URL": "http://host.docker.internal:4900", @@ -147,15 +147,16 @@ "protocol": "inspector" }, { - "name": "Debug Workspace", + "name": "Debug Workspace(mongo)", "type": "node", "request": "launch", "args": ["src/__start.ts"], "env": { - "MONGO_URL": "mongodb://localhost:27017", "DB_URL": "mongodb://localhost:27017", - "REGION": "", // "DB_URL": "postgresql://postgres:example@localhost:5432", + // "DB_URL": "postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable", + + "REGION": "", "SERVER_SECRET": "secret", "TRANSACTOR_URL": "ws://localhost:3333", "ACCOUNTS_URL": "http://localhost:3000", @@ -164,7 +165,42 @@ "MINIO_ACCESS_KEY": "minioadmin", "MINIO_SECRET_KEY": "minioadmin", "MINIO_ENDPOINT": "localhost", - "MODEL_VERSION": "v0.6.287" + "MODEL_VERSION": "v0.6.287", + "WS_OPERATION": "all+backup", + "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin", + "BACKUP_BUCKET": "dev-backups", + // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml", + // "INIT_WORKSPACE": "onboarding", + }, + "runtimeVersion": "20", + "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], + "sourceMaps": true, + "outputCapture": "std", + "cwd": "${workspaceRoot}/pods/workspace", + "protocol": "inspector" + }, + { + "name": "Debug Workspace(cockroach)", + "type": "node", + "request": "launch", + "args": ["src/__start.ts"], + "env": { + // "DB_URL": "mongodb://localhost:27017", + // "DB_URL": "postgresql://postgres:example@localhost:5432", + "DB_URL": "postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable", + "REGION": "cockroach", + "SERVER_SECRET": "secret", + "TRANSACTOR_URL": "ws://localhost:3332", + "ACCOUNTS_URL": "http://localhost:3000", + "FRONT_URL": "http://localhost:8080", + "SES_URL": "", + "MINIO_ACCESS_KEY": "minioadmin", + "MINIO_SECRET_KEY": "minioadmin", + "MINIO_ENDPOINT": "localhost", + "MODEL_VERSION": "v0.6.287", + "WS_OPERATION": "all+backup", + "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin", + "BACKUP_BUCKET": "dev-backups", // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml", // "INIT_WORKSPACE": "onboarding", }, diff --git a/desktop/src/main/backup.ts b/desktop/src/main/backup.ts index 25e0ba7e18..73bad40eaf 100644 --- a/desktop/src/main/backup.ts +++ b/desktop/src/main/backup.ts @@ -53,7 +53,7 @@ async function doBackup (dirName: string, token: string, endpoint: string, works isCanceled: (): boolean => { return runningBackup == null }, - progress: (value: number): void => { + progress: async (value: number): Promise => { notify('backup', value) }, getConnection: async () => client diff --git a/dev/.env b/dev/.env index 7f2e5141e7..7828283edc 100644 --- a/dev/.env +++ b/dev/.env @@ -1,3 +1,5 @@ STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" MONGO_URL=mongodb://mongodb:27017?compressors=snappy DB_URL_PG=postgresql://postgres:example@postgres:5432 +BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" +BACKUP_BUCKET_NAME=dev-backups diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index bed9eceb30..46a6de2915 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -3,7 +3,7 @@ services: image: 'mongo:7-jammy' container_name: mongodb extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' healthcheck: test: echo "try { db.currentOp().ok } catch (err) { }" | mongosh --port 27017 --quiet interval: 5s @@ -30,14 +30,14 @@ services: - 5432:5432 restart: unless-stopped cockroach: - image: cockroachdb/cockroach:latest-v24.2 - ports: - - "26257:26257" - - "8089:8080" - command: start-single-node --insecure - volumes: - - cockroach_db:/cockroach/cockroach-data - restart: unless-stopped + image: cockroachdb/cockroach:latest-v24.2 + ports: + - '26257:26257' + - '8089:8080' + command: start-single-node --insecure + volumes: + - cockroach_db:/cockroach/cockroach-data + restart: unless-stopped minio: image: 'minio/minio' command: server /data --address ":9000" --console-address ":9001" @@ -62,7 +62,7 @@ services: - discovery.type=single-node - ES_JAVA_OPTS=-Xms1024m -Xmx1024m - http.cors.enabled=true - - http.cors.allow-origin=http://localhost:8082 + - http.cors.allow-origin=http://localhost:8082 healthcheck: interval: 20s retries: 10 @@ -71,7 +71,7 @@ services: account: image: hardcoreeng/account extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' links: - mongodb - minio @@ -87,7 +87,9 @@ services: # - DB_URL=postgresql://postgres:example@postgres:5432 - DB_URL=${MONGO_URL} # - DB_NS=account-2 + # Pass only one region to disallow selection for new workspaces.Ø - REGION_INFO=|Mongo;pg|Postgres;cockroach|CockroachDB + # - REGION_INFO=cockroach|CockroachDB - TRANSACTOR_URL=ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg,ws://host.docker.internal:3332;;cockroach, - SES_URL= - STORAGE_CONFIG=${STORAGE_CONFIG} @@ -100,12 +102,12 @@ services: - BRANDING_PATH=/var/cfg/branding.json # - DISABLE_SIGNUP=true # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml - # - INIT_WORKSPACE=onboarding + # - INIT_WORKSPACE=onboarding restart: unless-stopped stats: image: hardcoreeng/stats extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' ports: - 4900:4900 environment: @@ -115,10 +117,7 @@ services: workspace: image: hardcoreeng/workspace extra_hosts: - - "host.docker.internal:host-gateway" - # deploy: - # mode: replicated - # replicas: 3 + - 'host.docker.internal:host-gateway' links: - mongodb - minio @@ -126,11 +125,10 @@ services: volumes: - ./branding.json:/var/cfg/branding.json environment: - # - WS_OPERATION=create + - WS_OPERATION=all+backup - SERVER_SECRET=secret - DB_URL=${MONGO_URL} - STATS_URL=http://host.docker.internal:4900 - # - DB_URL=postgresql://postgres:example@postgres:5432 - SES_URL= - STORAGE_CONFIG=${STORAGE_CONFIG} - RESERVED_DB_NAMES=telegram,gmail,github @@ -139,11 +137,13 @@ services: - BRANDING_PATH=/var/cfg/branding.json # - PARALLEL=2 - INIT_WORKSPACE=test + - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} + - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} restart: unless-stopped workspacepg: image: hardcoreeng/workspace extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' links: - postgres - minio @@ -151,7 +151,7 @@ services: volumes: - ./branding.json:/var/cfg/branding.json environment: - # - WS_OPERATION=create + - WS_OPERATION=all+backup - SERVER_SECRET=secret - DB_URL=postgresql://postgres:example@postgres:5432 - STATS_URL=http://host.docker.internal:4900 @@ -165,11 +165,13 @@ services: - BRANDING_PATH=/var/cfg/branding.json # - PARALLEL=2 # - INIT_WORKSPACE=onboarding - restart: unless-stopped + - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} + - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} + restart: unless-stopped workspace_cockroach: image: hardcoreeng/workspace extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' links: - cockroach - minio @@ -177,7 +179,7 @@ services: volumes: - ./branding.json:/var/cfg/branding.json environment: - # - WS_OPERATION=create + - WS_OPERATION=all+backup - SERVER_SECRET=secret - DB_URL=postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable - STATS_URL=http://host.docker.internal:4900 @@ -190,11 +192,13 @@ services: - BRANDING_PATH=/var/cfg/branding.json # - PARALLEL=2 # - INIT_WORKSPACE=onboarding - restart: unless-stopped + - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} + - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} + restart: unless-stopped collaborator: image: hardcoreeng/collaborator extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' links: - mongodb - minio @@ -212,7 +216,7 @@ services: front: image: hardcoreeng/front extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' links: - mongodb - minio @@ -247,7 +251,7 @@ services: transactor: image: hardcoreeng/transactor extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' links: - mongodb - minio @@ -257,11 +261,11 @@ services: ports: - 3333:3333 volumes: - - ./branding.json:/var/cfg/branding.json + - ./branding.json:/var/cfg/branding.json environment: # - SERVER_PROVIDER=uweb # - UWS_HTTP_MAX_HEADERS_SIZE="32768" - - UV_THREADPOOL_SIZE=10 + - UV_THREADPOOL_SIZE=10 - SERVER_PORT=3333 - SERVER_SECRET=secret - ENABLE_COMPRESSION=true @@ -273,12 +277,12 @@ services: - 'MONGO_OPTIONS={"appName": "transactor", "maxPoolSize": 10}' - METRICS_CONSOLE=false - METRICS_FILE=metrics.txt - - STORAGE_CONFIG=${STORAGE_CONFIG} + - STORAGE_CONFIG=${STORAGE_CONFIG} - FRONT_URL=http://host.docker.internal:8087 # - APM_SERVER_URL=http://apm-server:8200 - SES_URL='' - ACCOUNTS_URL=http://host.docker.internal:3000 - - LAST_NAME_FIRST=true + - LAST_NAME_FIRST=true - BRANDING_PATH=/var/cfg/branding.json - SUPPORT_WORKSPACE=support - AI_BOT_URL=http://host.docker.internal:4010 @@ -286,7 +290,7 @@ services: transactor_pg: image: hardcoreeng/transactor extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' links: - postgres - minio @@ -296,11 +300,11 @@ services: ports: - 3331:3331 volumes: - - ./branding.json:/var/cfg/branding.json + - ./branding.json:/var/cfg/branding.json environment: # - SERVER_PROVIDER=uweb # - UWS_HTTP_MAX_HEADERS_SIZE="32768" - # - UV_THREADPOOL_SIZE=10 + # - UV_THREADPOOL_SIZE=10 - SERVER_PORT=3331 - SERVER_SECRET=secret - ENABLE_COMPRESSION=true @@ -314,13 +318,13 @@ services: # - APM_SERVER_URL=http://apm-server:8200 - SES_URL='' - ACCOUNTS_URL=http://host.docker.internal:3000 - - LAST_NAME_FIRST=true - - BRANDING_PATH=/var/cfg/branding.json + - LAST_NAME_FIRST=true + - BRANDING_PATH=/var/cfg/branding.json restart: unless-stopped transactor_cockroach: image: hardcoreeng/transactor extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' links: - cockroach - minio @@ -330,7 +334,7 @@ services: ports: - 3332:3332 volumes: - - ./branding.json:/var/cfg/branding.json + - ./branding.json:/var/cfg/branding.json environment: # - SERVER_PROVIDER=uweb # - UWS_HTTP_MAX_HEADERS_SIZE="32768" @@ -348,9 +352,9 @@ services: # - APM_SERVER_URL=http://apm-server:8200 - SES_URL='' - ACCOUNTS_URL=http://host.docker.internal:3000 - - LAST_NAME_FIRST=true - - BRANDING_PATH=/var/cfg/branding.json - restart: unless-stopped + - LAST_NAME_FIRST=true + - BRANDING_PATH=/var/cfg/branding.json + restart: unless-stopped rekoni: image: hardcoreeng/rekoni-service restart: unless-stopped @@ -359,7 +363,7 @@ services: fulltext: image: hardcoreeng/fulltext extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' restart: unless-stopped links: - elastic @@ -378,11 +382,11 @@ services: fulltext_pg: image: hardcoreeng/fulltext extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' restart: unless-stopped links: - elastic - - postgres + - postgres ports: - 4701:4701 environment: @@ -398,11 +402,11 @@ services: fulltext_cockroach: image: hardcoreeng/fulltext extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' restart: unless-stopped links: - elastic - - cockroach + - cockroach ports: - 4702:4702 environment: @@ -414,11 +418,11 @@ services: - STORAGE_CONFIG=${STORAGE_CONFIG} - STATS_URL=http://host.docker.internal:4900 - REKONI_URL=http://host.docker.internal:4004 - - ACCOUNTS_URL=http://host.docker.internal:3000 + - ACCOUNTS_URL=http://host.docker.internal:3000 print: image: hardcoreeng/print extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' restart: unless-stopped ports: - 4005:4005 @@ -429,7 +433,7 @@ services: sign: image: hardcoreeng/sign extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' restart: unless-stopped ports: - 4006:4006 @@ -443,13 +447,13 @@ services: - ACCOUNTS_URL=http://host.docker.internal:3000 - MINIO_SECRET_KEY=minioadmin - CERTIFICATE_PATH=/var/cfg/certificate.p12 - - SERVICE_ID=sign-service + - SERVICE_ID=sign-service - BRANDING_PATH=/var/cfg/branding.json - STATS_URL=http://host.docker.internal:4900 analytics: image: hardcoreeng/analytics-collector extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' restart: unless-stopped ports: - 4017:4017 @@ -467,7 +471,7 @@ services: ports: - 4010:4010 extra_hosts: - - "host.docker.internal:host-gateway" + - 'host.docker.internal:host-gateway' restart: unless-stopped environment: - SERVER_SECRET=secret diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 66594fabab..bfd9cf5d72 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -58,7 +58,7 @@ import serverClientPlugin, { listAccountWorkspaces, updateBackupInfo } from '@hcengineering/server-client' -import { createBackupPipeline, getConfig } from '@hcengineering/server-pipeline' +import { createBackupPipeline, getConfig, getWorkspaceDestroyAdapter } from '@hcengineering/server-pipeline' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' import { FileModelLogger } from '@hcengineering/server-tool' import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' @@ -73,6 +73,8 @@ import core, { AccountRole, generateId, getWorkspaceId, + isActiveMode, + isArchivingMode, MeasureMetricsContext, metricsToString, RateLimiter, @@ -90,6 +92,9 @@ import contact from '@hcengineering/model-contact' import { getMongoClient, getWorkspaceMongoDB, shutdown } from '@hcengineering/mongo' import { backupDownload } from '@hcengineering/server-backup/src/backup' +import { createDatalakeClient, DatalakeService, type DatalakeConfig } from '@hcengineering/datalake' +import { getModelVersion } from '@hcengineering/model-all' +import { S3Service, type S3Config } from '@hcengineering/s3' import type { PipelineFactory, StorageAdapter, StorageAdapterEx } from '@hcengineering/server-core' import { deepEqual } from 'fast-equals' import { createWriteStream, readFileSync } from 'fs' @@ -116,13 +121,10 @@ import { } from './clean' import { changeConfiguration } from './configuration' import { moveAccountDbFromMongoToPG, moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db' +import { restoreControlledDocContentMongo, restoreWikiContentMongo } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { fixAccountEmails, renameAccount } from './renameAccount' import { copyToDatalake, moveFiles, showLostFiles } from './storage' -import { getModelVersion } from '@hcengineering/model-all' -import { type DatalakeConfig, DatalakeService, createDatalakeClient } from '@hcengineering/datalake' -import { S3Service, type S3Config } from '@hcengineering/s3' -import { restoreControlledDocContentMongo, restoreWikiContentMongo } from './markup' const colorConstants = { colorRed: '\u001b[31m', @@ -171,7 +173,8 @@ export function devTool ( setMetadata(serverClientPlugin.metadata.Endpoint, accountsUrl) setMetadata(serverToken.metadata.Secret, serverSecret) - async function withDatabase (uri: string, f: (db: AccountDB) => Promise): Promise { + async function withAccountDatabase (f: (db: AccountDB) => Promise, dbOverride?: string): Promise { + const uri = dbOverride ?? getAccountDBUrl() console.log(`connecting to database '${uri}'...`) const [accountDb, closeAccountsDb] = await getAccountDB(uri) @@ -211,8 +214,7 @@ export function devTool ( .requiredOption('-f, --first ', 'first name') .requiredOption('-l, --last ', 'last name') .action(async (email: string, cmd) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { console.log(`creating account ${cmd.first as string} ${cmd.last as string} (${email})...`) await createAcc(toolCtx, db, null, email, cmd.password, cmd.first, cmd.last, true) }) @@ -223,8 +225,7 @@ export function devTool ( .description('create user and corresponding account in master database') .option('-p, --password ', 'new user password') .action(async (email: string, cmd) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { console.log(`update account ${email} ${cmd.first as string} ${cmd.last as string}...`) await replacePassword(db, email, cmd.password) }) @@ -234,8 +235,7 @@ export function devTool ( .command('reset-email ') .description('rename account in accounts and all workspaces') .action(async (email: string, newEmail: string, cmd) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { console.log(`update account ${email} to ${newEmail}`) await renameAccount(toolCtx, db, accountsUrl, email, newEmail) }) @@ -245,8 +245,7 @@ export function devTool ( .command('fix-email ') .description('fix email in all workspaces to be proper one') .action(async (email: string, newEmail: string, cmd) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { console.log(`update account ${email} to ${newEmail}`) await fixAccountEmails(toolCtx, db, accountsUrl, email, newEmail) }) @@ -257,12 +256,11 @@ export function devTool ( .description('compact all db collections') .option('-w, --workspace ', 'A selected "workspace" only', '') .action(async (cmd: { workspace: string }) => { - const { dbUrl } = prepareTools() - const mongodbUri = getMongoDBUrl() - await withDatabase(dbUrl, async (db) => { + const dbUrl = getMongoDBUrl() + await withAccountDatabase(async (db) => { console.log('compacting db ...') let gtotal: number = 0 - const client = getMongoClient(mongodbUri ?? dbUrl) + const client = getMongoClient(dbUrl) const _client = await client.getClient() try { const workspaces = await listWorkspacesPure(db) @@ -297,8 +295,7 @@ export function devTool ( .command('assign-workspace ') .description('assign workspace') .action(async (email: string, workspace: string, cmd) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { console.log(`assigning user ${email} to ${workspace}...`) try { const workspaceInfo = await getWorkspaceById(db, workspace) @@ -332,8 +329,7 @@ export function devTool ( .command('show-user ') .description('show user') .action(async (email) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const info = await getAccount(db, email) console.log(info) }) @@ -352,8 +348,8 @@ export function devTool ( workspace, cmd: { email: string, workspaceName: string, init?: string, branding?: string, region?: string } ) => { - const { dbUrl, txes, version, migrateOperations } = prepareTools() - await withDatabase(dbUrl, async (db) => { + const { txes, version, migrateOperations } = prepareTools() + await withAccountDatabase(async (db) => { const measureCtx = new MeasureMetricsContext('create-workspace', {}) const brandingObj = cmd.branding !== undefined || cmd.init !== undefined ? { key: cmd.branding, initWorkspace: cmd.init } : null @@ -386,9 +382,8 @@ export function devTool ( .command('set-user-role ') .description('set user role') .action(async (email: string, workspace: string, role: AccountRole, cmd) => { - const { dbUrl } = prepareTools() console.log(`set user ${email} role for ${workspace}...`) - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const workspaceInfo = await getWorkspaceById(db, workspace) if (workspaceInfo === null) { throw new Error(`workspace ${workspace} not found`) @@ -406,9 +401,8 @@ export function devTool ( .command('set-user-admin ') .description('set user role') .action(async (email: string, role: string) => { - const { dbUrl } = prepareTools() console.log(`set user ${email} admin...`) - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { await setAccountAdmin(db, email, role === 'true') }) }) @@ -421,8 +415,7 @@ export function devTool ( .action(async (workspace, cmd: { force: boolean, indexes: boolean }) => { const { version, txes, migrateOperations } = prepareTools() - const accountUrl = getAccountDBUrl() - await withDatabase(accountUrl, async (db) => { + await withAccountDatabase(async (db) => { const info = await getWorkspaceById(db, workspace) if (info === null) { throw new Error(`workspace ${workspace} not found`) @@ -469,8 +462,7 @@ export function devTool ( .option('-f|--force [force]', 'Force update', false) .action(async (cmd: { logs: string, force: boolean, console: boolean, ignore: string, region: string }) => { const { version, txes, migrateOperations } = prepareTools() - const accountUrl = getAccountDBUrl() - await withDatabase(accountUrl, async (db) => { + await withAccountDatabase(async (db) => { const workspaces = (await listWorkspacesRaw(db, cmd.region)).filter((ws) => !cmd.ignore.includes(ws.workspace)) workspaces.sort((a, b) => b.lastVisit - a.lastVisit) const measureCtx = new MeasureMetricsContext('upgrade', {}) @@ -512,11 +504,10 @@ export function devTool ( program .command('list-unused-workspaces') - .description('remove unused workspaces. Without it will only mark them disabled') + .description('list unused workspaces. Without it will only mark them disabled') .option('-t|--timeout [timeout]', 'Timeout in days', '60') .action(async (cmd: { disable: boolean, exclude: string, timeout: string }) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const workspaces = new Map((await listWorkspacesPure(db)).map((p) => [p._id.toString(), p])) const accounts = await listAccounts(db) @@ -566,7 +557,7 @@ export function devTool ( }) }) program - .command('archive-workspaces-mongo') + .command('archive-workspaces') .description('Archive and delete non visited workspaces...') .option('-r|--remove [remove]', 'Pass to remove all data', false) .option('--region [region]', 'Pass to remove all data', '') @@ -582,8 +573,7 @@ export function devTool ( region: string }) => { const { dbUrl, txes } = prepareTools() - const mongodbUri = getMongoDBUrl() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const workspaces = (await listWorkspacesPure(db)) .sort((a, b) => a.lastVisit - b.lastVisit) .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) @@ -591,72 +581,65 @@ export function devTool ( const _timeout = parseInt(cmd.timeout) ?? 7 let unused = 0 + for (const ws of workspaces) { + const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - // We need to update workspaces with missing workspaceUrl - const client = getMongoClient(mongodbUri ?? dbUrl) - const mongoClient = await client.getClient() - try { - for (const ws of workspaces) { - const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) + if (lastVisitDays > _timeout && isActiveMode(ws.mode)) { + unused++ + toolCtx.warn('--- unused', { + url: ws.workspaceUrl, + id: ws.workspace, + lastVisitDays, + mode: ws.mode + }) + try { + await backupWorkspace( + toolCtx, + ws, + (dbUrl, storageAdapter) => { + const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { + externalStorage: storageAdapter, + usePassedCtx: true + }) + return factory + }, + (ctx, dbUrls, workspace, branding, externalStorage) => { + return getConfig(ctx, dbUrls, ctx, { + externalStorage, + disableTriggers: true + }) + }, + cmd.region, + true, + true, + 5000, // 5 gigabytes per blob + async (storage, workspaceStorage) => { + if (cmd.remove) { + await updateArchiveInfo(toolCtx, db, ws.workspace, true) + const files = await workspaceStorage.listStream(toolCtx, { name: ws.workspace }) - if (lastVisitDays > _timeout && ws.mode !== 'archived') { - unused++ - toolCtx.warn('--- unused', { - url: ws.workspaceUrl, - id: ws.workspace, - lastVisitDays, - mode: ws.mode - }) - try { - await backupWorkspace( - toolCtx, - ws, - (dbUrl, storageAdapter) => { - const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { - externalStorage: storageAdapter, - usePassedCtx: true - }) - return factory - }, - (ctx, dbUrls, workspace, branding, externalStorage) => { - return getConfig(ctx, dbUrls, ctx, { - externalStorage, - disableTriggers: true - }) - }, - cmd.region, - true, - true, - 5000, // 5 gigabytes per blob - async (storage, workspaceStorage) => { - if (cmd.remove) { - await updateArchiveInfo(toolCtx, db, ws.workspace, true) - const files = await workspaceStorage.listStream(toolCtx, { name: ws.workspace }) - - while (true) { - const docs = await files.next() - if (docs.length === 0) { - break - } - await workspaceStorage.remove( - toolCtx, - { name: ws.workspace }, - docs.map((it) => it._id) - ) + while (true) { + const docs = await files.next() + if (docs.length === 0) { + break } - - const mongoDb = getWorkspaceMongoDB(mongoClient, { name: ws.workspace }) - await mongoDb.dropDatabase() + await workspaceStorage.remove( + toolCtx, + { name: ws.workspace }, + docs.map((it) => it._id) + ) } + + const destroyer = getWorkspaceDestroyAdapter(dbUrl) + + await destroyer.deleteWorkspace(toolCtx, { name: ws.workspace }) } - ) - } catch (err: any) { - toolCtx.error('Failed to backup/archive workspace', { workspace: ws.workspace }) - } + } + ) + } catch (err: any) { + toolCtx.error('Failed to backup/archive workspace', { workspace: ws.workspace }) } } - } finally { - client.close() } console.log('Processed unused workspaces', unused) }) @@ -669,8 +652,8 @@ export function devTool ( .option('--region [region]', 'Force backup of selected workspace', '') .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '') .action(async (cmd: { workspace: string, region: string }) => { - const { dbUrl, txes } = prepareTools() - await withDatabase(dbUrl, async (db) => { + const { txes } = prepareTools() + await withAccountDatabase(async (db) => { const workspaces = (await listWorkspacesPure(db)) .sort((a, b) => a.lastVisit - b.lastVisit) .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) @@ -714,28 +697,21 @@ export function devTool ( }) program - .command('drop-workspace-mongo ') + .command('drop-workspace ') .description('drop workspace') .option('--full [full]', 'Force remove all data', false) .action(async (workspace, cmd: { full: boolean }) => { const { dbUrl } = prepareTools() - const mongodbUri = getMongoDBUrl() await withStorage(async (storageAdapter) => { - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const ws = await getWorkspaceById(db, workspace) if (ws === null) { console.log('no workspace exists') return } if (cmd.full) { - const client = getMongoClient(mongodbUri ?? dbUrl) - const _client = await client.getClient() - try { - await dropWorkspaceFull(toolCtx, db, _client, null, workspace, storageAdapter) - } finally { - client.close() - } + await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace, storageAdapter) } else { await dropWorkspace(toolCtx, db, null, workspace) } @@ -744,26 +720,19 @@ export function devTool ( }) program - .command('drop-workspace-by-email-mongo ') + .command('drop-workspace-by-email ') .description('drop workspace') .option('--full [full]', 'Force remove all data', false) .action(async (email, cmd: { full: boolean }) => { const { dbUrl } = prepareTools() - const mongodbUri = getMongoDBUrl() await withStorage(async (storageAdapter) => { - await withDatabase(dbUrl, async (db) => { - const client = getMongoClient(mongodbUri ?? dbUrl) - const _client = await client.getClient() - try { - for (const workspace of await listWorkspacesByAccount(db, email)) { - if (cmd.full) { - await dropWorkspaceFull(toolCtx, db, _client, null, workspace.workspace, storageAdapter) - } else { - await dropWorkspace(toolCtx, db, null, workspace.workspace) - } + await withAccountDatabase(async (db) => { + for (const workspace of await listWorkspacesByAccount(db, email)) { + if (cmd.full) { + await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace.workspace, storageAdapter) + } else { + await dropWorkspace(toolCtx, db, null, workspace.workspace) } - } finally { - client.close() } }) }) @@ -773,8 +742,7 @@ export function devTool ( .description('drop workspace') .option('--full [full]', 'Force remove all data', false) .action(async (email, cmd: { full: boolean }) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { for (const workspace of await listWorkspacesByAccount(db, email)) { console.log(workspace.workspace, workspace.workspaceUrl, workspace.workspaceName) } @@ -782,26 +750,19 @@ export function devTool ( }) program - .command('drop-workspace-last-visit-mongo') + .command('drop-workspace-last-visit') .description('drop old workspaces') .action(async (cmd: any) => { const { dbUrl } = prepareTools() - const mongodbUri = getMongoDBUrl() await withStorage(async (storageAdapter) => { - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const workspacesJSON = await listWorkspacesPure(db) - const client = getMongoClient(mongodbUri ?? dbUrl) - const _client = await client.getClient() - try { - for (const ws of workspacesJSON) { - const lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - if (lastVisit > 60) { - await dropWorkspaceFull(toolCtx, db, _client, null, ws.workspace, storageAdapter) - } + for (const ws of workspacesJSON) { + const lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) + if (lastVisit > 60) { + await dropWorkspaceFull(toolCtx, db, dbUrl, null, ws.workspace, storageAdapter) } - } finally { - client.close() } }) }) @@ -812,8 +773,8 @@ export function devTool ( .description('List workspaces') .option('-e|--expired [expired]', 'Show only expired', false) .action(async (cmd: { expired: boolean }) => { - const { dbUrl, version } = prepareTools() - await withDatabase(dbUrl, async (db) => { + const { version } = prepareTools() + await withAccountDatabase(async (db) => { const workspacesJSON = await listWorkspacesPure(db) for (const ws of workspacesJSON) { let lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) @@ -853,9 +814,9 @@ export function devTool ( }) program.command('fix-person-accounts-mongo').action(async () => { - const { dbUrl, version } = prepareTools() + const { version } = prepareTools() const mongodbUri = getMongoDBUrl() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const ws = await listWorkspacesPure(db) const client = getMongoClient(mongodbUri) const _client = await client.getClient() @@ -882,8 +843,7 @@ export function devTool ( .command('show-accounts') .description('Show accounts') .action(async () => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const workspaces = await listWorkspacesPure(db) const accounts = await listAccounts(db) for (const a of accounts) { @@ -901,8 +861,7 @@ export function devTool ( .command('drop-account ') .description('drop account') .action(async (email: string, cmd) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { await dropAccount(toolCtx, db, null, email) }) }) @@ -1204,11 +1163,10 @@ export function devTool ( const datalake = createDatalakeClient(datalakeConfig as DatalakeConfig) let workspaces: Workspace[] = [] - const accountUrl = getAccountDBUrl() - await withDatabase(accountUrl, async (db) => { + await withAccountDatabase(async (db) => { workspaces = await listWorkspacesPure(db) workspaces = workspaces - .filter((p) => p.mode !== 'archived') + .filter((p) => isActiveMode(p.mode) || isArchivingMode(p.mode)) .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) .sort((a, b) => b.lastVisit - a.lastVisit) }) @@ -1237,14 +1195,13 @@ export function devTool ( dryRun: cmd.dryrun } - const { dbUrl, version } = prepareTools() + const { version } = prepareTools() let workspaces: Workspace[] = [] - const accountUrl = getAccountDBUrl() - await withDatabase(accountUrl, async (db) => { + await withAccountDatabase(async (db) => { workspaces = await listWorkspacesPure(db) workspaces = workspaces - .filter((p) => p.mode !== 'archived') + .filter((p) => isActiveMode(p.mode)) .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) .sort((a, b) => b.lastVisit - a.lastVisit) }) @@ -1252,32 +1209,30 @@ export function devTool ( console.log('found workspaces', workspaces.length) await withStorage(async (storageAdapter) => { - await withDatabase(dbUrl, async (db) => { - const mongodbUri = getMongoDBUrl() - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() + const mongodbUri = getMongoDBUrl() + const client = getMongoClient(mongodbUri) + const _client = await client.getClient() - try { - const count = workspaces.length - let index = 0 - for (const workspace of workspaces) { - index++ + try { + const count = workspaces.length + let index = 0 + for (const workspace of workspaces) { + index++ - toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) - if (workspace.version === undefined || !deepEqual(workspace.version, version)) { - console.log(`upgrade to ${versionToString(version)} is required`) - continue - } - - const workspaceId = getWorkspaceId(workspace.workspace) - const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - await restoreWikiContentMongo(toolCtx, wsDb, workspaceId, storageAdapter, params) + toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) + if (workspace.version === undefined || !deepEqual(workspace.version, version)) { + console.log(`upgrade to ${versionToString(version)} is required`) + continue } - } finally { - client.close() + + const workspaceId = getWorkspaceId(workspace.workspace) + const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) + + await restoreWikiContentMongo(toolCtx, wsDb, workspaceId, storageAdapter, params) } - }) + } finally { + client.close() + } }) }) @@ -1292,11 +1247,10 @@ export function devTool ( dryRun: cmd.dryrun } - const { dbUrl, version } = prepareTools() + const { version } = prepareTools() let workspaces: Workspace[] = [] - const accountUrl = getAccountDBUrl() - await withDatabase(accountUrl, async (db) => { + await withAccountDatabase(async (db) => { workspaces = await listWorkspacesPure(db) workspaces = workspaces .filter((p) => p.mode !== 'archived') @@ -1307,7 +1261,7 @@ export function devTool ( console.log('found workspaces', workspaces.length) await withStorage(async (storageAdapter) => { - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const mongodbUri = getMongoDBUrl() const client = getMongoClient(mongodbUri) const _client = await client.getClient() @@ -1341,8 +1295,7 @@ export function devTool ( .command('confirm-email ') .description('confirm user email') .action(async (email: string, cmd) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const account = await getAccount(db, email) if (account?.confirmed === true) { console.log(`Already confirmed:${email}`) @@ -1367,16 +1320,14 @@ export function devTool ( .action(async (workspace: string, cmd) => { const { dbUrl } = prepareTools() await withStorage(async (adapter) => { - await withDatabase(dbUrl, async (db) => { - const telegramDB = process.env.TELEGRAM_DATABASE - if (telegramDB === undefined) { - console.error('please provide TELEGRAM_DATABASE.') - process.exit(1) - } + const telegramDB = process.env.TELEGRAM_DATABASE + if (telegramDB === undefined) { + console.error('please provide TELEGRAM_DATABASE.') + process.exit(1) + } - console.log(`clearing ${workspace} history:`) - await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(workspace), telegramDB, adapter) - }) + console.log(`clearing ${workspace} history:`) + await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(workspace), telegramDB, adapter) }) }) @@ -1386,7 +1337,7 @@ export function devTool ( .action(async (cmd) => { const { dbUrl } = prepareTools() await withStorage(async (adapter) => { - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const telegramDB = process.env.TELEGRAM_DATABASE if (telegramDB === undefined) { console.error('please provide TELEGRAM_DATABASE.') @@ -1426,11 +1377,9 @@ export function devTool ( .action(async (workspace: string, cmd: { recruit: boolean, tracker: boolean, removedTx: boolean }) => { const { dbUrl } = prepareTools() await withStorage(async (adapter) => { - await withDatabase(dbUrl, async (db) => { - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await cleanWorkspace(toolCtx, dbUrl, wsid, adapter, endpoint, cmd) - }) + const wsid = getWorkspaceId(workspace) + const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') + await cleanWorkspace(toolCtx, dbUrl, wsid, adapter, endpoint, cmd) }) }) program @@ -1503,8 +1452,7 @@ export function devTool ( move: cmd.move === 'true' } - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { await withStorage(async (adapter) => { try { const exAdapter = adapter as StorageAdapterEx @@ -1523,8 +1471,8 @@ export function devTool ( if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { continue } - if (workspace.mode === 'archived') { - console.log('ignore archived workspace', workspace.workspace) + if (!isActiveMode(workspace.mode)) { + console.log('ignore non active workspace', workspace.workspace, workspace.mode) continue } if (workspace.disabled === true && !cmd.disabled) { @@ -1554,8 +1502,7 @@ export function devTool ( .option('--disabled', 'Include disabled workspaces', false) .option('--all', 'Show all files', false) .action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { await withStorage(async (adapter) => { const mongodbUri = getMongoDBUrl() const client = getMongoClient(mongodbUri) @@ -1566,8 +1513,8 @@ export function devTool ( workspaces.sort((a, b) => b.lastVisit - a.lastVisit) for (const workspace of workspaces) { - if (workspace.mode === 'archived') { - console.log('ignore archived workspace', workspace.workspace) + if (!isActiveMode(workspace.mode)) { + console.log('ignore non active workspace', workspace.workspace, workspace.mode) continue } if (workspace.disabled === true && !cmd.disabled) { @@ -1682,8 +1629,7 @@ export function devTool ( .option('--disable ', 'Disable plugin configuration', '') .option('--list', 'List plugin states', false) .action(async (cmd: { enable: string, disable: string, list: boolean }) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { console.log('configure all workspaces') console.log(JSON.stringify(cmd)) const workspaces = await listWorkspacesRaw(db) @@ -1730,8 +1676,7 @@ export function devTool ( write: string mode: 'find-all' | 'connect-only' }) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { console.log(JSON.stringify(cmd)) if (!['find-all', 'connect-only'].includes(cmd.mode)) { console.log('wrong mode') @@ -1858,7 +1803,7 @@ export function devTool ( const { dbUrl } = prepareTools() const mongodbUri = getMongoDBUrl() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const workspaces = await listWorkspacesRaw(db) workspaces.sort((a, b) => b.lastVisit - a.lastVisit) for (const workspace of workspaces) { @@ -1882,7 +1827,7 @@ export function devTool ( const { dbUrl } = prepareTools() const mongodbUri = getMongoDBUrl() - await withDatabase(mongodbUri, async (db) => { + await withAccountDatabase(async (db) => { const workspaces = await listWorkspacesRaw(db) workspaces.sort((a, b) => b.lastVisit - a.lastVisit) await moveFromMongoToPG( @@ -1911,7 +1856,7 @@ export function devTool ( const { dbUrl } = prepareTools() const mongodbUri = getMongoDBUrl() - await withDatabase(mongodbUri, async (db) => { + await withAccountDatabase(async (db) => { const workspaceInfo = await getWorkspaceById(db, workspace) if (workspaceInfo === null) { throw new Error(`workspace ${workspace} not found`) @@ -1940,19 +1885,19 @@ export function devTool ( throw new Error('MONGO_URL and DB_URL are the same') } - await withDatabase(dbUrl, async (pgDb) => { - await withDatabase(mongodbUri, async (mongoDb) => { + await withAccountDatabase(async (pgDb) => { + await withAccountDatabase(async (mongoDb) => { await moveAccountDbFromMongoToPG(toolCtx, mongoDb, pgDb) - }) - }) + }, mongodbUri) + }, dbUrl) }) program .command('perfomance') .option('-p, --parallel', '', false) .action(async (cmd: { parallel: boolean }) => { - const { dbUrl, txes, version, migrateOperations } = prepareTools() - await withDatabase(dbUrl, async (db) => { + const { txes, version, migrateOperations } = prepareTools() + await withAccountDatabase(async (db) => { const email = generateId() const ws = generateId() const wsid = getWorkspaceId(ws) @@ -1990,8 +1935,7 @@ export function devTool ( .command('reset-ws-attempts ') .description('Reset workspace creation/upgrade attempts counter') .action(async (workspace) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { + await withAccountDatabase(async (db) => { const info = await getWorkspaceById(db, workspace) if (info === null) { throw new Error(`workspace ${workspace} not found`) diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index e3f8043059..419256610c 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -655,13 +655,70 @@ export interface DomainIndexConfiguration extends Doc { export type WorkspaceMode = | 'manual-creation' - | 'pending-creation' - | 'creating' - | 'upgrading' - | 'pending-deletion' - | 'deleting' + | 'pending-creation' // -> 'creating' + | 'creating' // -> 'active + | 'upgrading' // -> 'active' + | 'pending-deletion' // -> 'deleting' + | 'deleting' // -> "deleted" | 'active' + | 'archiving-pending-backup' // -> 'cleaning' + | 'archiving-backup' // -> 'archiving-pending-clean' + | 'archiving-pending-clean' // -> 'archiving-clean' + | 'archiving-clean' // -> 'archived' | 'archived' + | 'migration-pending-backup' // -> 'migration-backup' + | 'migration-backup' // -> 'migration-pending-cleanup' + | 'migration-pending-clean' // -> 'migration-pending-cleaning' + | 'migration-clean' // -> 'pending-restoring' + | 'pending-restore' // -> 'restoring' + | 'restoring' // -> 'active' + +export function isActiveMode (mode?: WorkspaceMode): boolean { + return mode === 'active' +} +export function isDeletingMode (mode: WorkspaceMode): boolean { + return mode === 'pending-deletion' || mode === 'deleting' +} +export function isArchivingMode (mode?: WorkspaceMode): boolean { + return ( + mode === 'archiving-pending-backup' || + mode === 'archiving-backup' || + mode === 'archiving-pending-clean' || + mode === 'archiving-clean' || + mode === 'archived' + ) +} + +export function isMigrationMode (mode?: WorkspaceMode): boolean { + return ( + mode === 'migration-pending-backup' || + mode === 'migration-backup' || + mode === 'migration-pending-clean' || + mode === 'migration-clean' + ) +} +export function isRestoringMode (mode?: WorkspaceMode): boolean { + return mode === 'restoring' || mode === 'pending-restore' +} + +export type WorkspaceUpdateEvent = + | 'ping' + | 'create-started' + | 'create-done' + | 'upgrade-started' + | 'upgrade-done' + | 'restore-started' + | 'restore-done' + | 'progress' + | 'migrate-backup-started' // -> state = 'migration-backup' + | 'migrate-backup-done' // -> state = 'migration-pending-cleaning' + | 'migrate-clean-started' // -> state = 'migration-cleaning' + | 'migrate-clean-done' // -> state = 'pending-restoring' + | 'archiving-backup-started' // -> state = 'archiving' + | 'archiving-backup-done' // -> state = 'archiving-pending-cleaning' + | 'archiving-clean-started' + | 'archiving-clean-done' + | 'archiving-done' export interface BackupStatus { dataSize: number @@ -689,5 +746,8 @@ export interface BaseWorkspaceInfo { endpoint: string + region?: string // Transactor group name + targetRegion?: string // Transactor region to move to + backupInfo?: BackupStatus } diff --git a/plugins/login-resources/src/components/AdminWorkspaces.svelte b/plugins/login-resources/src/components/AdminWorkspaces.svelte new file mode 100644 index 0000000000..9325943a65 --- /dev/null +++ b/plugins/login-resources/src/components/AdminWorkspaces.svelte @@ -0,0 +1,264 @@ + + +{#if isAdmin} +
+
Workspaces administration panel
+
+ +
+ +
+ Migration region selector: + it.region === selectedRegionId)?.name} + items={regionInfo.map((it) => ({ id: it.region === '' ? '#' : it.region, label: getEmbeddedLabel(it.name) }))} + on:selected={(it) => { + selectedRegionId = it.detail === '#' ? '' : it.detail + }} + /> +
+
+ +
+ {#each Object.keys(dayRanges) as k} + {@const v = groupped.get(k) ?? []} + {@const activeV = v.filter((it) => it.mode === 'active' && (it.region ?? '') !== selectedRegionId)} + {@const archiveV = v.filter((it) => it.mode === 'active')} + {@const archivedD = v.filter((it) => it.mode === 'archived')} + {@const av = v.length - archiveV.length - archivedD.length} + {#if v.length > 0} + + + + {k} - {v.length} + {#if av > 0} + - maitenance: {av} + {/if} + + + + {#if archiveV.length > 0} +
+ + + {workspace.region ?? ''} + + + {lastUsageDays} days + + + + {workspace.mode} + + + + {workspace.attempts} + + + + + {#if workspace.progress !== 100 && workspace.progress !== 0} + ({workspace.progress}%) + {/if} + + + {#if workspace.backupInfo != null} + {@const sz = workspace.backupInfo.dataSize + workspace.backupInfo.blobsSize} + {@const szGb = Math.round((sz * 100) / 1024) / 100} + {#if szGb > 0} + {Math.round((sz * 100) / 1024) / 100}Gb + {:else} + {Math.round(sz * 100) / 100}Mb + {/if} + {/if} + + + {#if workspace.backupInfo != null} + {@const hours = Math.round((now - workspace.backupInfo.lastBackup) / (1000 * 3600))} + + {#if hours > 24} + {Math.round(hours / 24)} days + {:else} + {hours} hours + {/if} + {/if} + +
+
+
+ {#if workspace.mode === 'active'} +
+
+
+ {/each} + + {/if} + {/each} + + + + + +{/if} diff --git a/plugins/login-resources/src/components/LoginApp.svelte b/plugins/login-resources/src/components/LoginApp.svelte index d0c9b64ef7..f4957893cc 100644 --- a/plugins/login-resources/src/components/LoginApp.svelte +++ b/plugins/login-resources/src/components/LoginApp.svelte @@ -51,6 +51,7 @@ import loginBackWebp from '../../img/login_back.webp' import loginBack2xWebp from '../../img/login_back_2x.webp' import login from '../plugin' + import AdminWorkspaces from './AdminWorkspaces.svelte' export let page: Pages = 'signup' @@ -107,61 +108,69 @@ onMount(chooseToken) -
-
768} class:p-4={$deviceInfo.docWidth > 768}> - - - +{#if page === 'admin'} + +{:else} +
+
768} class:p-4={$deviceInfo.docWidth > 768}> + + + - - + + -
- {getMetadata(workbench.metadata.PlatformTitle)} +
+ {getMetadata(workbench.metadata.PlatformTitle)} +
+ +
768} class:white={!$themeStore.dark}> + +
+ {#if page === 'login'} + + {:else if page === 'signup'} + + {:else if page === 'createWorkspace'} + + {:else if page === 'password'} + + {:else if page === 'recovery'} + + {:else if page === 'selectWorkspace'} + + {:else if page === 'join'} + + {:else if page === 'confirm'} + + {:else if page === 'confirmationSend'} + + {:else if page === 'auth'} + + {/if} +
+ +
+ +
- -
768} class:white={!$themeStore.dark}> - -
- {#if page === 'login'} - - {:else if page === 'signup'} - - {:else if page === 'createWorkspace'} - - {:else if page === 'password'} - - {:else if page === 'recovery'} - - {:else if page === 'selectWorkspace'} - - {:else if page === 'join'} - - {:else if page === 'confirm'} - - {:else if page === 'confirmationSend'} - - {:else if page === 'auth'} - - {/if} -
- -
- -
-
+{/if}