WEBDIENSTE/huly-platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2026-09-11 20:27:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

UBERF-7670: Per region moves (#7444)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev
2024-12-13 00:20:35 +07:00
committed by GitHub
parent 1eefb143cc
commit 7b497c93c1
35 changed files with 1597 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2 -1
View File
@@ -103,4 +103,5 @@ bundle.js.map
tests/profiles
**/bundle/model.json
.wrangler
dump
dump
**/logs/**
.vscode/launch.json
Vendored
+42 -6
View File
@@ -107,9 +107,9 @@
"DB_URL": "mongodb://localhost:27017",
// "DB_URL": "postgresql://postgres:example@localhost:5432",
"SERVER_SECRET": "secret",
"TRANSACTOR_URL": "ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg",
"REGION_INFO":"|Mongo;pg|Postgres;cockroach|CockroachDB",
"TRANSACTOR_URL": "ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg,ws://host.docker.internal:3332;;cockroach",
"ACCOUNTS_URL": "http://localhost:3000",
"REGION_INFO": "|Mongo;pg|Postgree",
"ACCOUNT_PORT": "3000",
"FRONT_URL": "http://localhost:8080",
"STATS_URL": "http://host.docker.internal:4900",
@@ -147,15 +147,16 @@
"protocol": "inspector"
},
{
"name": "Debug Workspace",
"name": "Debug Workspace(mongo)",
"type": "node",
"request": "launch",
"args": ["src/__start.ts"],
"env": {
"MONGO_URL": "mongodb://localhost:27017",
"DB_URL": "mongodb://localhost:27017",
"REGION": "",
// "DB_URL": "postgresql://postgres:example@localhost:5432",
// "DB_URL": "postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable",
"REGION": "",
"SERVER_SECRET": "secret",
"TRANSACTOR_URL": "ws://localhost:3333",
"ACCOUNTS_URL": "http://localhost:3000",
@@ -164,7 +165,42 @@
"MINIO_ACCESS_KEY": "minioadmin",
"MINIO_SECRET_KEY": "minioadmin",
"MINIO_ENDPOINT": "localhost",
"MODEL_VERSION": "v0.6.287"
"MODEL_VERSION": "v0.6.287",
"WS_OPERATION": "all+backup",
"BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
"BACKUP_BUCKET": "dev-backups",
// "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml",
// "INIT_WORKSPACE": "onboarding",
},
"runtimeVersion": "20",
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"sourceMaps": true,
"outputCapture": "std",
"cwd": "${workspaceRoot}/pods/workspace",
"protocol": "inspector"
},
{
"name": "Debug Workspace(cockroach)",
"type": "node",
"request": "launch",
"args": ["src/__start.ts"],
"env": {
// "DB_URL": "mongodb://localhost:27017",
// "DB_URL": "postgresql://postgres:example@localhost:5432",
"DB_URL": "postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable",
"REGION": "cockroach",
"SERVER_SECRET": "secret",
"TRANSACTOR_URL": "ws://localhost:3332",
"ACCOUNTS_URL": "http://localhost:3000",
"FRONT_URL": "http://localhost:8080",
"SES_URL": "",
"MINIO_ACCESS_KEY": "minioadmin",
"MINIO_SECRET_KEY": "minioadmin",
"MINIO_ENDPOINT": "localhost",
"MODEL_VERSION": "v0.6.287",
"WS_OPERATION": "all+backup",
"BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
"BACKUP_BUCKET": "dev-backups",
// "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml",
// "INIT_WORKSPACE": "onboarding",
},
desktop/src/main/backup.ts
+1 -1
View File
@@ -53,7 +53,7 @@ async function doBackup (dirName: string, token: string, endpoint: string, works
isCanceled: (): boolean => {
return runningBackup == null
},
progress: (value: number): void => {
progress: async (value: number): Promise<void> => {
notify('backup', value)
},
getConnection: async () => client
dev/.env
+2
View File
@@ -1,3 +1,5 @@
STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
MONGO_URL=mongodb://mongodb:27017?compressors=snappy
DB_URL_PG=postgresql://postgres:example@postgres:5432
BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
BACKUP_BUCKET_NAME=dev-backups
dev/docker-compose.yaml
+57 -53
View File
@@ -3,7 +3,7 @@ services:
image: 'mongo:7-jammy'
container_name: mongodb
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
healthcheck:
test: echo "try { db.currentOp().ok } catch (err) { }" | mongosh --port 27017 --quiet
interval: 5s
@@ -30,14 +30,14 @@ services:
- 5432:5432
restart: unless-stopped
cockroach:
image: cockroachdb/cockroach:latest-v24.2
ports:
- "26257:26257"
- "8089:8080"
command: start-single-node --insecure
volumes:
- cockroach_db:/cockroach/cockroach-data
restart: unless-stopped
image: cockroachdb/cockroach:latest-v24.2
ports:
- '26257:26257'
- '8089:8080'
command: start-single-node --insecure
volumes:
- cockroach_db:/cockroach/cockroach-data
restart: unless-stopped
minio:
image: 'minio/minio'
command: server /data --address ":9000" --console-address ":9001"
@@ -62,7 +62,7 @@ services:
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- http.cors.enabled=true
- http.cors.allow-origin=http://localhost:8082
- http.cors.allow-origin=http://localhost:8082
healthcheck:
interval: 20s
retries: 10
@@ -71,7 +71,7 @@ services:
account:
image: hardcoreeng/account
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
links:
- mongodb
- minio
@@ -87,7 +87,9 @@ services:
# - DB_URL=postgresql://postgres:example@postgres:5432
- DB_URL=${MONGO_URL}
# - DB_NS=account-2
# Pass only one region to disallow selection for new workspaces.Ø
- REGION_INFO=|Mongo;pg|Postgres;cockroach|CockroachDB
# - REGION_INFO=cockroach|CockroachDB
- TRANSACTOR_URL=ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg,ws://host.docker.internal:3332;;cockroach,
- SES_URL=
- STORAGE_CONFIG=${STORAGE_CONFIG}
@@ -100,12 +102,12 @@ services:
- BRANDING_PATH=/var/cfg/branding.json
# - DISABLE_SIGNUP=true
# - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml
# - INIT_WORKSPACE=onboarding
# - INIT_WORKSPACE=onboarding
restart: unless-stopped
stats:
image: hardcoreeng/stats
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
ports:
- 4900:4900
environment:
@@ -115,10 +117,7 @@ services:
workspace:
image: hardcoreeng/workspace
extra_hosts:
- "host.docker.internal:host-gateway"
# deploy:
# mode: replicated
# replicas: 3
- 'host.docker.internal:host-gateway'
links:
- mongodb
- minio
@@ -126,11 +125,10 @@ services:
volumes:
- ./branding.json:/var/cfg/branding.json
environment:
# - WS_OPERATION=create
- WS_OPERATION=all+backup
- SERVER_SECRET=secret
- DB_URL=${MONGO_URL}
- STATS_URL=http://host.docker.internal:4900
# - DB_URL=postgresql://postgres:example@postgres:5432
- SES_URL=
- STORAGE_CONFIG=${STORAGE_CONFIG}
- RESERVED_DB_NAMES=telegram,gmail,github
@@ -139,11 +137,13 @@ services:
- BRANDING_PATH=/var/cfg/branding.json
# - PARALLEL=2
- INIT_WORKSPACE=test
- BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
- BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
restart: unless-stopped
workspacepg:
image: hardcoreeng/workspace
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
links:
- postgres
- minio
@@ -151,7 +151,7 @@ services:
volumes:
- ./branding.json:/var/cfg/branding.json
environment:
# - WS_OPERATION=create
- WS_OPERATION=all+backup
- SERVER_SECRET=secret
- DB_URL=postgresql://postgres:example@postgres:5432
- STATS_URL=http://host.docker.internal:4900
@@ -165,11 +165,13 @@ services:
- BRANDING_PATH=/var/cfg/branding.json
# - PARALLEL=2
# - INIT_WORKSPACE=onboarding
restart: unless-stopped
- BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
- BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
restart: unless-stopped
workspace_cockroach:
image: hardcoreeng/workspace
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
links:
- cockroach
- minio
@@ -177,7 +179,7 @@ services:
volumes:
- ./branding.json:/var/cfg/branding.json
environment:
# - WS_OPERATION=create
- WS_OPERATION=all+backup
- SERVER_SECRET=secret
- DB_URL=postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable
- STATS_URL=http://host.docker.internal:4900
@@ -190,11 +192,13 @@ services:
- BRANDING_PATH=/var/cfg/branding.json
# - PARALLEL=2
# - INIT_WORKSPACE=onboarding
restart: unless-stopped
- BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
- BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
restart: unless-stopped
collaborator:
image: hardcoreeng/collaborator
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
links:
- mongodb
- minio
@@ -212,7 +216,7 @@ services:
front:
image: hardcoreeng/front
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
links:
- mongodb
- minio
@@ -247,7 +251,7 @@ services:
transactor:
image: hardcoreeng/transactor
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
links:
- mongodb
- minio
@@ -257,11 +261,11 @@ services:
ports:
- 3333:3333
volumes:
- ./branding.json:/var/cfg/branding.json
- ./branding.json:/var/cfg/branding.json
environment:
# - SERVER_PROVIDER=uweb
# - UWS_HTTP_MAX_HEADERS_SIZE="32768"
- UV_THREADPOOL_SIZE=10
- UV_THREADPOOL_SIZE=10
- SERVER_PORT=3333
- SERVER_SECRET=secret
- ENABLE_COMPRESSION=true
@@ -273,12 +277,12 @@ services:
- 'MONGO_OPTIONS={"appName": "transactor", "maxPoolSize": 10}'
- METRICS_CONSOLE=false
- METRICS_FILE=metrics.txt
- STORAGE_CONFIG=${STORAGE_CONFIG}
- STORAGE_CONFIG=${STORAGE_CONFIG}
- FRONT_URL=http://host.docker.internal:8087
# - APM_SERVER_URL=http://apm-server:8200
- SES_URL=''
- ACCOUNTS_URL=http://host.docker.internal:3000
- LAST_NAME_FIRST=true
- LAST_NAME_FIRST=true
- BRANDING_PATH=/var/cfg/branding.json
- SUPPORT_WORKSPACE=support
- AI_BOT_URL=http://host.docker.internal:4010
@@ -286,7 +290,7 @@ services:
transactor_pg:
image: hardcoreeng/transactor
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
links:
- postgres
- minio
@@ -296,11 +300,11 @@ services:
ports:
- 3331:3331
volumes:
- ./branding.json:/var/cfg/branding.json
- ./branding.json:/var/cfg/branding.json
environment:
# - SERVER_PROVIDER=uweb
# - UWS_HTTP_MAX_HEADERS_SIZE="32768"
# - UV_THREADPOOL_SIZE=10
# - UV_THREADPOOL_SIZE=10
- SERVER_PORT=3331
- SERVER_SECRET=secret
- ENABLE_COMPRESSION=true
@@ -314,13 +318,13 @@ services:
# - APM_SERVER_URL=http://apm-server:8200
- SES_URL=''
- ACCOUNTS_URL=http://host.docker.internal:3000
- LAST_NAME_FIRST=true
- BRANDING_PATH=/var/cfg/branding.json
- LAST_NAME_FIRST=true
- BRANDING_PATH=/var/cfg/branding.json
restart: unless-stopped
transactor_cockroach:
image: hardcoreeng/transactor
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
links:
- cockroach
- minio
@@ -330,7 +334,7 @@ services:
ports:
- 3332:3332
volumes:
- ./branding.json:/var/cfg/branding.json
- ./branding.json:/var/cfg/branding.json
environment:
# - SERVER_PROVIDER=uweb
# - UWS_HTTP_MAX_HEADERS_SIZE="32768"
@@ -348,9 +352,9 @@ services:
# - APM_SERVER_URL=http://apm-server:8200
- SES_URL=''
- ACCOUNTS_URL=http://host.docker.internal:3000
- LAST_NAME_FIRST=true
- BRANDING_PATH=/var/cfg/branding.json
restart: unless-stopped
- LAST_NAME_FIRST=true
- BRANDING_PATH=/var/cfg/branding.json
restart: unless-stopped
rekoni:
image: hardcoreeng/rekoni-service
restart: unless-stopped
@@ -359,7 +363,7 @@ services:
fulltext:
image: hardcoreeng/fulltext
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
restart: unless-stopped
links:
- elastic
@@ -378,11 +382,11 @@ services:
fulltext_pg:
image: hardcoreeng/fulltext
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
restart: unless-stopped
links:
- elastic
- postgres
- postgres
ports:
- 4701:4701
environment:
@@ -398,11 +402,11 @@ services:
fulltext_cockroach:
image: hardcoreeng/fulltext
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
restart: unless-stopped
links:
- elastic
- cockroach
- cockroach
ports:
- 4702:4702
environment:
@@ -414,11 +418,11 @@ services:
- STORAGE_CONFIG=${STORAGE_CONFIG}
- STATS_URL=http://host.docker.internal:4900
- REKONI_URL=http://host.docker.internal:4004
- ACCOUNTS_URL=http://host.docker.internal:3000
- ACCOUNTS_URL=http://host.docker.internal:3000
print:
image: hardcoreeng/print
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
restart: unless-stopped
ports:
- 4005:4005
@@ -429,7 +433,7 @@ services:
sign:
image: hardcoreeng/sign
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
restart: unless-stopped
ports:
- 4006:4006
@@ -443,13 +447,13 @@ services:
- ACCOUNTS_URL=http://host.docker.internal:3000
- MINIO_SECRET_KEY=minioadmin
- CERTIFICATE_PATH=/var/cfg/certificate.p12
- SERVICE_ID=sign-service
- SERVICE_ID=sign-service
- BRANDING_PATH=/var/cfg/branding.json
- STATS_URL=http://host.docker.internal:4900
analytics:
image: hardcoreeng/analytics-collector
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
restart: unless-stopped
ports:
- 4017:4017
@@ -467,7 +471,7 @@ services:
ports:
- 4010:4010
extra_hosts:
- "host.docker.internal:host-gateway"
- 'host.docker.internal:host-gateway'
restart: unless-stopped
environment:
- SERVER_SECRET=secret
dev/tool/src/index.ts
+163 -219
View File
@@ -58,7 +58,7 @@ import serverClientPlugin, {
listAccountWorkspaces,
updateBackupInfo
} from '@hcengineering/server-client'
import { createBackupPipeline, getConfig } from '@hcengineering/server-pipeline'
import { createBackupPipeline, getConfig, getWorkspaceDestroyAdapter } from '@hcengineering/server-pipeline'
import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
import { FileModelLogger } from '@hcengineering/server-tool'
import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
@@ -73,6 +73,8 @@ import core, {
AccountRole,
generateId,
getWorkspaceId,
isActiveMode,
isArchivingMode,
MeasureMetricsContext,
metricsToString,
RateLimiter,
@@ -90,6 +92,9 @@ import contact from '@hcengineering/model-contact'
import { getMongoClient, getWorkspaceMongoDB, shutdown } from '@hcengineering/mongo'
import { backupDownload } from '@hcengineering/server-backup/src/backup'
import { createDatalakeClient, DatalakeService, type DatalakeConfig } from '@hcengineering/datalake'
import { getModelVersion } from '@hcengineering/model-all'
import { S3Service, type S3Config } from '@hcengineering/s3'
import type { PipelineFactory, StorageAdapter, StorageAdapterEx } from '@hcengineering/server-core'
import { deepEqual } from 'fast-equals'
import { createWriteStream, readFileSync } from 'fs'
@@ -116,13 +121,10 @@ import {
} from './clean'
import { changeConfiguration } from './configuration'
import { moveAccountDbFromMongoToPG, moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db'
import { restoreControlledDocContentMongo, restoreWikiContentMongo } from './markup'
import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
import { fixAccountEmails, renameAccount } from './renameAccount'
import { copyToDatalake, moveFiles, showLostFiles } from './storage'
import { getModelVersion } from '@hcengineering/model-all'
import { type DatalakeConfig, DatalakeService, createDatalakeClient } from '@hcengineering/datalake'
import { S3Service, type S3Config } from '@hcengineering/s3'
import { restoreControlledDocContentMongo, restoreWikiContentMongo } from './markup'
const colorConstants = {
colorRed: '\u001b[31m',
@@ -171,7 +173,8 @@ export function devTool (
setMetadata(serverClientPlugin.metadata.Endpoint, accountsUrl)
setMetadata(serverToken.metadata.Secret, serverSecret)
async function withDatabase (uri: string, f: (db: AccountDB) => Promise<any>): Promise<void> {
async function withAccountDatabase (f: (db: AccountDB) => Promise<any>, dbOverride?: string): Promise<void> {
const uri = dbOverride ?? getAccountDBUrl()
console.log(`connecting to database '${uri}'...`)
const [accountDb, closeAccountsDb] = await getAccountDB(uri)
@@ -211,8 +214,7 @@ export function devTool (
.requiredOption('-f, --first <first>', 'first name')
.requiredOption('-l, --last <last>', 'last name')
.action(async (email: string, cmd) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
console.log(`creating account ${cmd.first as string} ${cmd.last as string} (${email})...`)
await createAcc(toolCtx, db, null, email, cmd.password, cmd.first, cmd.last, true)
})
@@ -223,8 +225,7 @@ export function devTool (
.description('create user and corresponding account in master database')
.option('-p, --password <password>', 'new user password')
.action(async (email: string, cmd) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
console.log(`update account ${email} ${cmd.first as string} ${cmd.last as string}...`)
await replacePassword(db, email, cmd.password)
})
@@ -234,8 +235,7 @@ export function devTool (
.command('reset-email <email> <newEmail>')
.description('rename account in accounts and all workspaces')
.action(async (email: string, newEmail: string, cmd) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
console.log(`update account ${email} to ${newEmail}`)
await renameAccount(toolCtx, db, accountsUrl, email, newEmail)
})
@@ -245,8 +245,7 @@ export function devTool (
.command('fix-email <email> <newEmail>')
.description('fix email in all workspaces to be proper one')
.action(async (email: string, newEmail: string, cmd) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
console.log(`update account ${email} to ${newEmail}`)
await fixAccountEmails(toolCtx, db, accountsUrl, email, newEmail)
})
@@ -257,12 +256,11 @@ export function devTool (
.description('compact all db collections')
.option('-w, --workspace <workspace>', 'A selected "workspace" only', '')
.action(async (cmd: { workspace: string }) => {
const { dbUrl } = prepareTools()
const mongodbUri = getMongoDBUrl()
await withDatabase(dbUrl, async (db) => {
const dbUrl = getMongoDBUrl()
await withAccountDatabase(async (db) => {
console.log('compacting db ...')
let gtotal: number = 0
const client = getMongoClient(mongodbUri ?? dbUrl)
const client = getMongoClient(dbUrl)
const _client = await client.getClient()
try {
const workspaces = await listWorkspacesPure(db)
@@ -297,8 +295,7 @@ export function devTool (
.command('assign-workspace <email> <workspace>')
.description('assign workspace')
.action(async (email: string, workspace: string, cmd) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
console.log(`assigning user ${email} to ${workspace}...`)
try {
const workspaceInfo = await getWorkspaceById(db, workspace)
@@ -332,8 +329,7 @@ export function devTool (
.command('show-user <email>')
.description('show user')
.action(async (email) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const info = await getAccount(db, email)
console.log(info)
})
@@ -352,8 +348,8 @@ export function devTool (
workspace,
cmd: { email: string, workspaceName: string, init?: string, branding?: string, region?: string }
) => {
const { dbUrl, txes, version, migrateOperations } = prepareTools()
await withDatabase(dbUrl, async (db) => {
const { txes, version, migrateOperations } = prepareTools()
await withAccountDatabase(async (db) => {
const measureCtx = new MeasureMetricsContext('create-workspace', {})
const brandingObj =
cmd.branding !== undefined || cmd.init !== undefined ? { key: cmd.branding, initWorkspace: cmd.init } : null
@@ -386,9 +382,8 @@ export function devTool (
.command('set-user-role <email> <workspace> <role>')
.description('set user role')
.action(async (email: string, workspace: string, role: AccountRole, cmd) => {
const { dbUrl } = prepareTools()
console.log(`set user ${email} role for ${workspace}...`)
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const workspaceInfo = await getWorkspaceById(db, workspace)
if (workspaceInfo === null) {
throw new Error(`workspace ${workspace} not found`)
@@ -406,9 +401,8 @@ export function devTool (
.command('set-user-admin <email> <role>')
.description('set user role')
.action(async (email: string, role: string) => {
const { dbUrl } = prepareTools()
console.log(`set user ${email} admin...`)
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
await setAccountAdmin(db, email, role === 'true')
})
})
@@ -421,8 +415,7 @@ export function devTool (
.action(async (workspace, cmd: { force: boolean, indexes: boolean }) => {
const { version, txes, migrateOperations } = prepareTools()
const accountUrl = getAccountDBUrl()
await withDatabase(accountUrl, async (db) => {
await withAccountDatabase(async (db) => {
const info = await getWorkspaceById(db, workspace)
if (info === null) {
throw new Error(`workspace ${workspace} not found`)
@@ -469,8 +462,7 @@ export function devTool (
.option('-f|--force [force]', 'Force update', false)
.action(async (cmd: { logs: string, force: boolean, console: boolean, ignore: string, region: string }) => {
const { version, txes, migrateOperations } = prepareTools()
const accountUrl = getAccountDBUrl()
await withDatabase(accountUrl, async (db) => {
await withAccountDatabase(async (db) => {
const workspaces = (await listWorkspacesRaw(db, cmd.region)).filter((ws) => !cmd.ignore.includes(ws.workspace))
workspaces.sort((a, b) => b.lastVisit - a.lastVisit)
const measureCtx = new MeasureMetricsContext('upgrade', {})
@@ -512,11 +504,10 @@ export function devTool (
program
.command('list-unused-workspaces')
.description('remove unused workspaces. Without it will only mark them disabled')
.description('list unused workspaces. Without it will only mark them disabled')
.option('-t|--timeout [timeout]', 'Timeout in days', '60')
.action(async (cmd: { disable: boolean, exclude: string, timeout: string }) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const workspaces = new Map((await listWorkspacesPure(db)).map((p) => [p._id.toString(), p]))
const accounts = await listAccounts(db)
@@ -566,7 +557,7 @@ export function devTool (
})
})
program
.command('archive-workspaces-mongo')
.command('archive-workspaces')
.description('Archive and delete non visited workspaces...')
.option('-r|--remove [remove]', 'Pass to remove all data', false)
.option('--region [region]', 'Pass to remove all data', '')
@@ -582,8 +573,7 @@ export function devTool (
region: string
}) => {
const { dbUrl, txes } = prepareTools()
const mongodbUri = getMongoDBUrl()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const workspaces = (await listWorkspacesPure(db))
.sort((a, b) => a.lastVisit - b.lastVisit)
.filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace)
@@ -591,72 +581,65 @@ export function devTool (
const _timeout = parseInt(cmd.timeout) ?? 7
let unused = 0
for (const ws of workspaces) {
const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24)
// We need to update workspaces with missing workspaceUrl
const client = getMongoClient(mongodbUri ?? dbUrl)
const mongoClient = await client.getClient()
try {
for (const ws of workspaces) {
const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24)
if (lastVisitDays > _timeout && isActiveMode(ws.mode)) {
unused++
toolCtx.warn('--- unused', {
url: ws.workspaceUrl,
id: ws.workspace,
lastVisitDays,
mode: ws.mode
})
try {
await backupWorkspace(
toolCtx,
ws,
(dbUrl, storageAdapter) => {
const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, {
externalStorage: storageAdapter,
usePassedCtx: true
})
return factory
},
(ctx, dbUrls, workspace, branding, externalStorage) => {
return getConfig(ctx, dbUrls, ctx, {
externalStorage,
disableTriggers: true
})
},
cmd.region,
true,
true,
5000, // 5 gigabytes per blob
async (storage, workspaceStorage) => {
if (cmd.remove) {
await updateArchiveInfo(toolCtx, db, ws.workspace, true)
const files = await workspaceStorage.listStream(toolCtx, { name: ws.workspace })
if (lastVisitDays > _timeout && ws.mode !== 'archived') {
unused++
toolCtx.warn('--- unused', {
url: ws.workspaceUrl,
id: ws.workspace,
lastVisitDays,
mode: ws.mode
})
try {
await backupWorkspace(
toolCtx,
ws,
(dbUrl, storageAdapter) => {
const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, {
externalStorage: storageAdapter,
usePassedCtx: true
})
return factory
},
(ctx, dbUrls, workspace, branding, externalStorage) => {
return getConfig(ctx, dbUrls, ctx, {
externalStorage,
disableTriggers: true
})
},
cmd.region,
true,
true,
5000, // 5 gigabytes per blob
async (storage, workspaceStorage) => {
if (cmd.remove) {
await updateArchiveInfo(toolCtx, db, ws.workspace, true)
const files = await workspaceStorage.listStream(toolCtx, { name: ws.workspace })
while (true) {
const docs = await files.next()
if (docs.length === 0) {
break
}
await workspaceStorage.remove(
toolCtx,
{ name: ws.workspace },
docs.map((it) => it._id)
)
while (true) {
const docs = await files.next()
if (docs.length === 0) {
break
}
const mongoDb = getWorkspaceMongoDB(mongoClient, { name: ws.workspace })
await mongoDb.dropDatabase()
await workspaceStorage.remove(
toolCtx,
{ name: ws.workspace },
docs.map((it) => it._id)
)
}
const destroyer = getWorkspaceDestroyAdapter(dbUrl)
await destroyer.deleteWorkspace(toolCtx, { name: ws.workspace })
}
)
} catch (err: any) {
toolCtx.error('Failed to backup/archive workspace', { workspace: ws.workspace })
}
}
)
} catch (err: any) {
toolCtx.error('Failed to backup/archive workspace', { workspace: ws.workspace })
}
}
} finally {
client.close()
}
console.log('Processed unused workspaces', unused)
})
@@ -669,8 +652,8 @@ export function devTool (
.option('--region [region]', 'Force backup of selected workspace', '')
.option('-w|--workspace [workspace]', 'Force backup of selected workspace', '')
.action(async (cmd: { workspace: string, region: string }) => {
const { dbUrl, txes } = prepareTools()
await withDatabase(dbUrl, async (db) => {
const { txes } = prepareTools()
await withAccountDatabase(async (db) => {
const workspaces = (await listWorkspacesPure(db))
.sort((a, b) => a.lastVisit - b.lastVisit)
.filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace)
@@ -714,28 +697,21 @@ export function devTool (
})
program
.command('drop-workspace-mongo <name>')
.command('drop-workspace <name>')
.description('drop workspace')
.option('--full [full]', 'Force remove all data', false)
.action(async (workspace, cmd: { full: boolean }) => {
const { dbUrl } = prepareTools()
const mongodbUri = getMongoDBUrl()
await withStorage(async (storageAdapter) => {
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const ws = await getWorkspaceById(db, workspace)
if (ws === null) {
console.log('no workspace exists')
return
}
if (cmd.full) {
const client = getMongoClient(mongodbUri ?? dbUrl)
const _client = await client.getClient()
try {
await dropWorkspaceFull(toolCtx, db, _client, null, workspace, storageAdapter)
} finally {
client.close()
}
await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace, storageAdapter)
} else {
await dropWorkspace(toolCtx, db, null, workspace)
}
@@ -744,26 +720,19 @@ export function devTool (
})
program
.command('drop-workspace-by-email-mongo <email>')
.command('drop-workspace-by-email <email>')
.description('drop workspace')
.option('--full [full]', 'Force remove all data', false)
.action(async (email, cmd: { full: boolean }) => {
const { dbUrl } = prepareTools()
const mongodbUri = getMongoDBUrl()
await withStorage(async (storageAdapter) => {
await withDatabase(dbUrl, async (db) => {
const client = getMongoClient(mongodbUri ?? dbUrl)
const _client = await client.getClient()
try {
for (const workspace of await listWorkspacesByAccount(db, email)) {
if (cmd.full) {
await dropWorkspaceFull(toolCtx, db, _client, null, workspace.workspace, storageAdapter)
} else {
await dropWorkspace(toolCtx, db, null, workspace.workspace)
}
await withAccountDatabase(async (db) => {
for (const workspace of await listWorkspacesByAccount(db, email)) {
if (cmd.full) {
await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace.workspace, storageAdapter)
} else {
await dropWorkspace(toolCtx, db, null, workspace.workspace)
}
} finally {
client.close()
}
})
})
@@ -773,8 +742,7 @@ export function devTool (
.description('drop workspace')
.option('--full [full]', 'Force remove all data', false)
.action(async (email, cmd: { full: boolean }) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
for (const workspace of await listWorkspacesByAccount(db, email)) {
console.log(workspace.workspace, workspace.workspaceUrl, workspace.workspaceName)
}
@@ -782,26 +750,19 @@ export function devTool (
})
program
.command('drop-workspace-last-visit-mongo')
.command('drop-workspace-last-visit')
.description('drop old workspaces')
.action(async (cmd: any) => {
const { dbUrl } = prepareTools()
const mongodbUri = getMongoDBUrl()
await withStorage(async (storageAdapter) => {
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const workspacesJSON = await listWorkspacesPure(db)
const client = getMongoClient(mongodbUri ?? dbUrl)
const _client = await client.getClient()
try {
for (const ws of workspacesJSON) {
const lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24)
if (lastVisit > 60) {
await dropWorkspaceFull(toolCtx, db, _client, null, ws.workspace, storageAdapter)
}
for (const ws of workspacesJSON) {
const lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24)
if (lastVisit > 60) {
await dropWorkspaceFull(toolCtx, db, dbUrl, null, ws.workspace, storageAdapter)
}
} finally {
client.close()
}
})
})
@@ -812,8 +773,8 @@ export function devTool (
.description('List workspaces')
.option('-e|--expired [expired]', 'Show only expired', false)
.action(async (cmd: { expired: boolean }) => {
const { dbUrl, version } = prepareTools()
await withDatabase(dbUrl, async (db) => {
const { version } = prepareTools()
await withAccountDatabase(async (db) => {
const workspacesJSON = await listWorkspacesPure(db)
for (const ws of workspacesJSON) {
let lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24)
@@ -853,9 +814,9 @@ export function devTool (
})
program.command('fix-person-accounts-mongo').action(async () => {
const { dbUrl, version } = prepareTools()
const { version } = prepareTools()
const mongodbUri = getMongoDBUrl()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const ws = await listWorkspacesPure(db)
const client = getMongoClient(mongodbUri)
const _client = await client.getClient()
@@ -882,8 +843,7 @@ export function devTool (
.command('show-accounts')
.description('Show accounts')
.action(async () => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const workspaces = await listWorkspacesPure(db)
const accounts = await listAccounts(db)
for (const a of accounts) {
@@ -901,8 +861,7 @@ export function devTool (
.command('drop-account <name>')
.description('drop account')
.action(async (email: string, cmd) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
await dropAccount(toolCtx, db, null, email)
})
})
@@ -1204,11 +1163,10 @@ export function devTool (
const datalake = createDatalakeClient(datalakeConfig as DatalakeConfig)
let workspaces: Workspace[] = []
const accountUrl = getAccountDBUrl()
await withDatabase(accountUrl, async (db) => {
await withAccountDatabase(async (db) => {
workspaces = await listWorkspacesPure(db)
workspaces = workspaces
.filter((p) => p.mode !== 'archived')
.filter((p) => isActiveMode(p.mode) || isArchivingMode(p.mode))
.filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace)
.sort((a, b) => b.lastVisit - a.lastVisit)
})
@@ -1237,14 +1195,13 @@ export function devTool (
dryRun: cmd.dryrun
}
const { dbUrl, version } = prepareTools()
const { version } = prepareTools()
let workspaces: Workspace[] = []
const accountUrl = getAccountDBUrl()
await withDatabase(accountUrl, async (db) => {
await withAccountDatabase(async (db) => {
workspaces = await listWorkspacesPure(db)
workspaces = workspaces
.filter((p) => p.mode !== 'archived')
.filter((p) => isActiveMode(p.mode))
.filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace)
.sort((a, b) => b.lastVisit - a.lastVisit)
})
@@ -1252,32 +1209,30 @@ export function devTool (
console.log('found workspaces', workspaces.length)
await withStorage(async (storageAdapter) => {
await withDatabase(dbUrl, async (db) => {
const mongodbUri = getMongoDBUrl()
const client = getMongoClient(mongodbUri)
const _client = await client.getClient()
const mongodbUri = getMongoDBUrl()
const client = getMongoClient(mongodbUri)
const _client = await client.getClient()
try {
const count = workspaces.length
let index = 0
for (const workspace of workspaces) {
index++
try {
const count = workspaces.length
let index = 0
for (const workspace of workspaces) {
index++
toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count })
if (workspace.version === undefined || !deepEqual(workspace.version, version)) {
console.log(`upgrade to ${versionToString(version)} is required`)
continue
}
const workspaceId = getWorkspaceId(workspace.workspace)
const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace })
await restoreWikiContentMongo(toolCtx, wsDb, workspaceId, storageAdapter, params)
toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count })
if (workspace.version === undefined || !deepEqual(workspace.version, version)) {
console.log(`upgrade to ${versionToString(version)} is required`)
continue
}
} finally {
client.close()
const workspaceId = getWorkspaceId(workspace.workspace)
const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace })
await restoreWikiContentMongo(toolCtx, wsDb, workspaceId, storageAdapter, params)
}
})
} finally {
client.close()
}
})
})
@@ -1292,11 +1247,10 @@ export function devTool (
dryRun: cmd.dryrun
}
const { dbUrl, version } = prepareTools()
const { version } = prepareTools()
let workspaces: Workspace[] = []
const accountUrl = getAccountDBUrl()
await withDatabase(accountUrl, async (db) => {
await withAccountDatabase(async (db) => {
workspaces = await listWorkspacesPure(db)
workspaces = workspaces
.filter((p) => p.mode !== 'archived')
@@ -1307,7 +1261,7 @@ export function devTool (
console.log('found workspaces', workspaces.length)
await withStorage(async (storageAdapter) => {
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const mongodbUri = getMongoDBUrl()
const client = getMongoClient(mongodbUri)
const _client = await client.getClient()
@@ -1341,8 +1295,7 @@ export function devTool (
.command('confirm-email <email>')
.description('confirm user email')
.action(async (email: string, cmd) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const account = await getAccount(db, email)
if (account?.confirmed === true) {
console.log(`Already confirmed:${email}`)
@@ -1367,16 +1320,14 @@ export function devTool (
.action(async (workspace: string, cmd) => {
const { dbUrl } = prepareTools()
await withStorage(async (adapter) => {
await withDatabase(dbUrl, async (db) => {
const telegramDB = process.env.TELEGRAM_DATABASE
if (telegramDB === undefined) {
console.error('please provide TELEGRAM_DATABASE.')
process.exit(1)
}
const telegramDB = process.env.TELEGRAM_DATABASE
if (telegramDB === undefined) {
console.error('please provide TELEGRAM_DATABASE.')
process.exit(1)
}
console.log(`clearing ${workspace} history:`)
await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(workspace), telegramDB, adapter)
})
console.log(`clearing ${workspace} history:`)
await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(workspace), telegramDB, adapter)
})
})
@@ -1386,7 +1337,7 @@ export function devTool (
.action(async (cmd) => {
const { dbUrl } = prepareTools()
await withStorage(async (adapter) => {
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const telegramDB = process.env.TELEGRAM_DATABASE
if (telegramDB === undefined) {
console.error('please provide TELEGRAM_DATABASE.')
@@ -1426,11 +1377,9 @@ export function devTool (
.action(async (workspace: string, cmd: { recruit: boolean, tracker: boolean, removedTx: boolean }) => {
const { dbUrl } = prepareTools()
await withStorage(async (adapter) => {
await withDatabase(dbUrl, async (db) => {
const wsid = getWorkspaceId(workspace)
const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external')
await cleanWorkspace(toolCtx, dbUrl, wsid, adapter, endpoint, cmd)
})
const wsid = getWorkspaceId(workspace)
const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external')
await cleanWorkspace(toolCtx, dbUrl, wsid, adapter, endpoint, cmd)
})
})
program
@@ -1503,8 +1452,7 @@ export function devTool (
move: cmd.move === 'true'
}
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
await withStorage(async (adapter) => {
try {
const exAdapter = adapter as StorageAdapterEx
@@ -1523,8 +1471,8 @@ export function devTool (
if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) {
continue
}
if (workspace.mode === 'archived') {
console.log('ignore archived workspace', workspace.workspace)
if (!isActiveMode(workspace.mode)) {
console.log('ignore non active workspace', workspace.workspace, workspace.mode)
continue
}
if (workspace.disabled === true && !cmd.disabled) {
@@ -1554,8 +1502,7 @@ export function devTool (
.option('--disabled', 'Include disabled workspaces', false)
.option('--all', 'Show all files', false)
.action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
await withStorage(async (adapter) => {
const mongodbUri = getMongoDBUrl()
const client = getMongoClient(mongodbUri)
@@ -1566,8 +1513,8 @@ export function devTool (
workspaces.sort((a, b) => b.lastVisit - a.lastVisit)
for (const workspace of workspaces) {
if (workspace.mode === 'archived') {
console.log('ignore archived workspace', workspace.workspace)
if (!isActiveMode(workspace.mode)) {
console.log('ignore non active workspace', workspace.workspace, workspace.mode)
continue
}
if (workspace.disabled === true && !cmd.disabled) {
@@ -1682,8 +1629,7 @@ export function devTool (
.option('--disable <disable>', 'Disable plugin configuration', '')
.option('--list', 'List plugin states', false)
.action(async (cmd: { enable: string, disable: string, list: boolean }) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
console.log('configure all workspaces')
console.log(JSON.stringify(cmd))
const workspaces = await listWorkspacesRaw(db)
@@ -1730,8 +1676,7 @@ export function devTool (
write: string
mode: 'find-all' | 'connect-only'
}) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
console.log(JSON.stringify(cmd))
if (!['find-all', 'connect-only'].includes(cmd.mode)) {
console.log('wrong mode')
@@ -1858,7 +1803,7 @@ export function devTool (
const { dbUrl } = prepareTools()
const mongodbUri = getMongoDBUrl()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const workspaces = await listWorkspacesRaw(db)
workspaces.sort((a, b) => b.lastVisit - a.lastVisit)
for (const workspace of workspaces) {
@@ -1882,7 +1827,7 @@ export function devTool (
const { dbUrl } = prepareTools()
const mongodbUri = getMongoDBUrl()
await withDatabase(mongodbUri, async (db) => {
await withAccountDatabase(async (db) => {
const workspaces = await listWorkspacesRaw(db)
workspaces.sort((a, b) => b.lastVisit - a.lastVisit)
await moveFromMongoToPG(
@@ -1911,7 +1856,7 @@ export function devTool (
const { dbUrl } = prepareTools()
const mongodbUri = getMongoDBUrl()
await withDatabase(mongodbUri, async (db) => {
await withAccountDatabase(async (db) => {
const workspaceInfo = await getWorkspaceById(db, workspace)
if (workspaceInfo === null) {
throw new Error(`workspace ${workspace} not found`)
@@ -1940,19 +1885,19 @@ export function devTool (
throw new Error('MONGO_URL and DB_URL are the same')
}
await withDatabase(dbUrl, async (pgDb) => {
await withDatabase(mongodbUri, async (mongoDb) => {
await withAccountDatabase(async (pgDb) => {
await withAccountDatabase(async (mongoDb) => {
await moveAccountDbFromMongoToPG(toolCtx, mongoDb, pgDb)
})
})
}, mongodbUri)
}, dbUrl)
})
program
.command('perfomance')
.option('-p, --parallel', '', false)
.action(async (cmd: { parallel: boolean }) => {
const { dbUrl, txes, version, migrateOperations } = prepareTools()
await withDatabase(dbUrl, async (db) => {
const { txes, version, migrateOperations } = prepareTools()
await withAccountDatabase(async (db) => {
const email = generateId()
const ws = generateId()
const wsid = getWorkspaceId(ws)
@@ -1990,8 +1935,7 @@ export function devTool (
.command('reset-ws-attempts <name>')
.description('Reset workspace creation/upgrade attempts counter')
.action(async (workspace) => {
const { dbUrl } = prepareTools()
await withDatabase(dbUrl, async (db) => {
await withAccountDatabase(async (db) => {
const info = await getWorkspaceById(db, workspace)
if (info === null) {
throw new Error(`workspace ${workspace} not found`)
packages/core/src/classes.ts
+65 -5
View File
@@ -655,13 +655,70 @@ export interface DomainIndexConfiguration extends Doc {
export type WorkspaceMode =
| 'manual-creation'
| 'pending-creation'
| 'creating'
| 'upgrading'
| 'pending-deletion'
| 'deleting'
| 'pending-creation' // -> 'creating'
| 'creating' // -> 'active
| 'upgrading' // -> 'active'
| 'pending-deletion' // -> 'deleting'
| 'deleting' // -> "deleted"
| 'active'
| 'archiving-pending-backup' // -> 'cleaning'
| 'archiving-backup' // -> 'archiving-pending-clean'
| 'archiving-pending-clean' // -> 'archiving-clean'
| 'archiving-clean' // -> 'archived'
| 'archived'
| 'migration-pending-backup' // -> 'migration-backup'
| 'migration-backup' // -> 'migration-pending-cleanup'
| 'migration-pending-clean' // -> 'migration-pending-cleaning'
| 'migration-clean' // -> 'pending-restoring'
| 'pending-restore' // -> 'restoring'
| 'restoring' // -> 'active'
export function isActiveMode (mode?: WorkspaceMode): boolean {
return mode === 'active'
}
export function isDeletingMode (mode: WorkspaceMode): boolean {
return mode === 'pending-deletion' || mode === 'deleting'
}
export function isArchivingMode (mode?: WorkspaceMode): boolean {
return (
mode === 'archiving-pending-backup' ||
mode === 'archiving-backup' ||
mode === 'archiving-pending-clean' ||
mode === 'archiving-clean' ||
mode === 'archived'
)
}
export function isMigrationMode (mode?: WorkspaceMode): boolean {
return (
mode === 'migration-pending-backup' ||
mode === 'migration-backup' ||
mode === 'migration-pending-clean' ||
mode === 'migration-clean'
)
}
export function isRestoringMode (mode?: WorkspaceMode): boolean {
return mode === 'restoring' || mode === 'pending-restore'
}
export type WorkspaceUpdateEvent =
| 'ping'
| 'create-started'
| 'create-done'
| 'upgrade-started'
| 'upgrade-done'
| 'restore-started'
| 'restore-done'
| 'progress'
| 'migrate-backup-started' // -> state = 'migration-backup'
| 'migrate-backup-done' // -> state = 'migration-pending-cleaning'
| 'migrate-clean-started' // -> state = 'migration-cleaning'
| 'migrate-clean-done' // -> state = 'pending-restoring'
| 'archiving-backup-started' // -> state = 'archiving'
| 'archiving-backup-done' // -> state = 'archiving-pending-cleaning'
| 'archiving-clean-started'
| 'archiving-clean-done'
| 'archiving-done'
export interface BackupStatus {
dataSize: number
@@ -689,5 +746,8 @@ export interface BaseWorkspaceInfo {
endpoint: string
region?: string // Transactor group name
targetRegion?: string // Transactor region to move to
backupInfo?: BackupStatus
}
plugins/login-resources/src/components/AdminWorkspaces.svelte
+264
View File
@@ -0,0 +1,264 @@
<script lang="ts">
import { groupByArray, type BaseWorkspaceInfo } from '@hcengineering/core'
import { getEmbeddedLabel } from '@hcengineering/platform'
import { isAdminUser } from '@hcengineering/presentation'
import {
Button,
ButtonMenu,
Expandable,
IconArrowRight,
IconOpen,
IconStart,
IconStop,
locationToUrl,
Popup,
Scroller,
SearchEdit,
ticker
} from '@hcengineering/ui'
import { workbenchId } from '@hcengineering/workbench'
import { getAllWorkspaces, getRegionInfo, performWorkspaceOperation, type RegionInfo } from '../utils'
$: isAdmin = isAdminUser()
let search: string = ''
async function select (workspace: string): Promise<void> {
const url = locationToUrl({ path: [workbenchId, workspace] })
window.open(url, '_blank')
}
type WorkspaceInfo = BaseWorkspaceInfo & { attempts: number }
let workspaces: WorkspaceInfo[] = []
$: if ($ticker > 0) {
void getAllWorkspaces().then((res) => {
workspaces = res.sort((a, b) =>
(b.workspaceUrl ?? b.workspace).localeCompare(a.workspaceUrl ?? a.workspace)
) as WorkspaceInfo[]
})
}
const now = Date.now()
const dayRanges = {
Today: 1,
'Tree Days': 3,
Week: 7,
Month: 30,
'Two Months': 60,
'Tree Months': 90,
'Six Month': 182,
'Nine Months': 270,
Year: 365,
FewOrMoreYears: 10000000
}
$: groupped = groupByArray(
workspaces.filter(
(it) =>
(it.workspaceName?.includes(search) ?? false) ||
(it.workspaceUrl?.includes(search) ?? false) ||
it.workspace?.includes(search)
),
(it) => {
const lastUsageDays = Math.round((now - it.lastVisit) / (1000 * 3600 * 24))
return Object.entries(dayRanges).find(([_k, v]) => lastUsageDays <= v)?.[0] ?? 'Other'
}
)
let regionInfo: RegionInfo[] = []
let selectedRegionId: string = ''
void getRegionInfo().then((_regionInfo) => {
regionInfo = _regionInfo ?? []
if (selectedRegionId === '' && regionInfo.length > 0) {
selectedRegionId = regionInfo[0].region
}
})
$: selectedRegionName = regionInfo.find((it) => it.region === selectedRegionId)?.name
</script>
{#if isAdmin}
<div class="anticrm-panel flex-row flex-grow p-5">
<div class="fs-title p-3">Workspaces administration panel</div>
<div class="fs-title p-3 flex-no-shrink">
<SearchEdit bind:value={search} width={'100%'} />
</div>
<div class="fs-title p-3 flex-row-center">
<span class="mr-2"> Migration region selector: </span>
<ButtonMenu
selected={selectedRegionId}
autoSelectionIfOne
title={regionInfo.find((it) => it.region === selectedRegionId)?.name}
items={regionInfo.map((it) => ({ id: it.region === '' ? '#' : it.region, label: getEmbeddedLabel(it.name) }))}
on:selected={(it) => {
selectedRegionId = it.detail === '#' ? '' : it.detail
}}
/>
</div>
<div class="fs-title p-1">
<Scroller maxHeight={40} noStretch={true}>
<div class="mr-4">
{#each Object.keys(dayRanges) as k}
{@const v = groupped.get(k) ?? []}
{@const activeV = v.filter((it) => it.mode === 'active' && (it.region ?? '') !== selectedRegionId)}
{@const archiveV = v.filter((it) => it.mode === 'active')}
{@const archivedD = v.filter((it) => it.mode === 'archived')}
{@const av = v.length - archiveV.length - archivedD.length}
{#if v.length > 0}
<Expandable expandable={true} bordered={true}>
<svelte:fragment slot="title">
<span class="fs-title focused-button">
{k} - {v.length}
{#if av > 0}
- maitenance: {av}
{/if}
</span>
</svelte:fragment>
<svelte:fragment slot="tools">
{#if archiveV.length > 0}
<Button
icon={IconStop}
label={getEmbeddedLabel(`Mass Archive ${archiveV.length}`)}
kind={'ghost'}
on:click={() => {
void performWorkspaceOperation(
archiveV.map((it) => it.workspace),
'archive'
)
}}
/>
{/if}
{#if regionInfo.length > 0 && activeV.length > 0}
<Button
icon={IconArrowRight}
kind={'positive'}
label={getEmbeddedLabel(`Mass Migrate ${activeV.length} to ${selectedRegionName ?? ''}`)}
on:click={() => {
void performWorkspaceOperation(
activeV.map((it) => it.workspace),
'migrate-to',
selectedRegionId
)
}}
/>
{/if}
</svelte:fragment>
{#each v as workspace}
{@const wsName = workspace.workspaceName ?? workspace.workspace}
{@const lastUsageDays = Math.round((Date.now() - workspace.lastVisit) / (1000 * 3600 * 24))}
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<div class="flex fs-title cursor-pointer focused-button bordered">
<div class="flex p-2">
<span class="label overflow-label flex-row-center" style:width={'12rem'}>
{wsName}
<div class="ml-1">
<Button
icon={IconOpen}
size={'small'}
on:click={() => select(workspace.workspaceUrl ?? workspace.workspace)}
/>
</div>
</span>
<span class="label overflow-label" style:width={'8rem'}>
{workspace.region ?? ''}
</span>
<span class="label overflow-label" style:width={'5rem'}>
{lastUsageDays} days
</span>
<span class="label overflow-label" style:width={'10rem'}>
{workspace.mode}
</span>
<span class="label overflow-label" style:width={'2rem'}>
{workspace.attempts}
</span>
<!-- <span class="flex flex-between select-text overflow-label" style:width={'25rem'}>
{workspace.workspace}
</span> -->
<span class="flex flex-between" style:width={'5rem'}>
{#if workspace.progress !== 100 && workspace.progress !== 0}
({workspace.progress}%)
{/if}
</span>
<span class="flex flex-between" style:width={'5rem'}>
{#if workspace.backupInfo != null}
{@const sz = workspace.backupInfo.dataSize + workspace.backupInfo.blobsSize}
{@const szGb = Math.round((sz * 100) / 1024) / 100}
{#if szGb > 0}
{Math.round((sz * 100) / 1024) / 100}Gb
{:else}
{Math.round(sz * 100) / 100}Mb
{/if}
{/if}
</span>
<span class="flex flex-between" style:width={'5rem'}>
{#if workspace.backupInfo != null}
{@const hours = Math.round((now - workspace.backupInfo.lastBackup) / (1000 * 3600))}
{#if hours > 24}
{Math.round(hours / 24)} days
{:else}
{hours} hours
{/if}
{/if}
</span>
</div>
<div class="flex flex-grow gap-1-5 flex-between">
<div class="flex flex-row-center gap-1-5">
{#if workspace.mode === 'active'}
<Button
icon={IconStop}
size={'small'}
label={getEmbeddedLabel('Archive')}
kind={'ghost'}
on:click={() => {
void performWorkspaceOperation(workspace.workspace, 'archive')
}}
/>
{/if}
{#if workspace.mode === 'archived'}
<Button
icon={IconStart}
size={'small'}
kind={'ghost'}
label={getEmbeddedLabel('Unarchive')}
on:click={() => {
void performWorkspaceOperation(workspace.workspace, 'unarchive')
}}
/>
{/if}
{#if regionInfo.length > 0 && workspace.mode === 'active' && (workspace.region ?? '') !== selectedRegionId}
<Button
icon={IconArrowRight}
size={'small'}
kind={'positive'}
label={getEmbeddedLabel('Migrate ' + (selectedRegionName ?? ''))}
on:click={() => {
void performWorkspaceOperation(workspace.workspace, 'migrate-to', selectedRegionId)
}}
/>
{/if}
</div>
</div>
</div>
{/each}
</Expandable>
{/if}
{/each}
</div>
</Scroller>
</div>
</div>
<Popup />
{/if}
plugins/login-resources/src/components/LoginApp.svelte
+60 -51
View File
@@ -51,6 +51,7 @@
import loginBackWebp from '../../img/login_back.webp'
import loginBack2xWebp from '../../img/login_back_2x.webp'
import login from '../plugin'
import AdminWorkspaces from './AdminWorkspaces.svelte'
export let page: Pages = 'signup'
@@ -107,61 +108,69 @@
onMount(chooseToken)
</script>
<div class="theme-dark w-full h-full backd" class:paneld={$deviceInfo.docWidth <= 768} class:white={!$themeStore.dark}>
<div class="bg-image clear-mins" class:back={$deviceInfo.docWidth > 768} class:p-4={$deviceInfo.docWidth > 768}>
<picture>
<source srcset={`${loginBackAvif}, ${loginBack2xAvif} 2x`} type="image/avif" />
<source srcset={`${loginBackWebp}, ${loginBack2xWebp} 2x`} type="image/webp" />
{#if page === 'admin'}
<AdminWorkspaces />
{:else}
<div
class="theme-dark w-full h-full backd"
class:paneld={$deviceInfo.docWidth <= 768}
class:white={!$themeStore.dark}
>
<div class="bg-image clear-mins" class:back={$deviceInfo.docWidth > 768} class:p-4={$deviceInfo.docWidth > 768}>
<picture>
<source srcset={`${loginBackAvif}, ${loginBack2xAvif} 2x`} type="image/avif" />
<source srcset={`${loginBackWebp}, ${loginBack2xWebp} 2x`} type="image/webp" />
<img
class="back-image"
src={loginBack}
style:display={$deviceInfo.docWidth <= 768 ? 'none' : 'block'}
srcset={`${loginBack} 1x, ${loginBack2x} 2x`}
alt=""
/>
</picture>
<img
class="back-image"
src={loginBack}
style:display={$deviceInfo.docWidth <= 768 ? 'none' : 'block'}
srcset={`${loginBack} 1x, ${loginBack2x} 2x`}
alt=""
/>
</picture>
<div
style:position="fixed"
style:left={$deviceInfo.docWidth <= 480 ? '.75rem' : '1.75rem'}
style:top={'3rem'}
class="flex-row-center"
>
<LoginIcon /><span class="fs-title ml-2">{getMetadata(workbench.metadata.PlatformTitle)}</span>
<div
style:position="fixed"
style:left={$deviceInfo.docWidth <= 480 ? '.75rem' : '1.75rem'}
style:top={'3rem'}
class="flex-row-center"
>
<LoginIcon /><span class="fs-title ml-2">{getMetadata(workbench.metadata.PlatformTitle)}</span>
</div>
<div class="panel-base" class:panel={$deviceInfo.docWidth > 768} class:white={!$themeStore.dark}>
<Scroller padding={'1rem 0'}>
<div class="form-content">
{#if page === 'login'}
<LoginForm {navigateUrl} {signUpDisabled} />
{:else if page === 'signup'}
<SignupForm {signUpDisabled} />
{:else if page === 'createWorkspace'}
<CreateWorkspaceForm />
{:else if page === 'password'}
<PasswordRequest {signUpDisabled} />
{:else if page === 'recovery'}
<PasswordRestore />
{:else if page === 'selectWorkspace'}
<SelectWorkspace {navigateUrl} />
{:else if page === 'join'}
<Join />
{:else if page === 'confirm'}
<Confirmation />
{:else if page === 'confirmationSend'}
<ConfirmationSend />
{:else if page === 'auth'}
<Auth />
{/if}
</div>
</Scroller>
</div>
<Popup />
</div>
<div class="panel-base" class:panel={$deviceInfo.docWidth > 768} class:white={!$themeStore.dark}>
<Scroller padding={'1rem 0'}>
<div class="form-content">
{#if page === 'login'}
<LoginForm {navigateUrl} {signUpDisabled} />
{:else if page === 'signup'}
<SignupForm {signUpDisabled} />
{:else if page === 'createWorkspace'}
<CreateWorkspaceForm />
{:else if page === 'password'}
<PasswordRequest {signUpDisabled} />
{:else if page === 'recovery'}
<PasswordRestore />
{:else if page === 'selectWorkspace'}
<SelectWorkspace {navigateUrl} />
{:else if page === 'join'}
<Join />
{:else if page === 'confirm'}
<Confirmation />
{:else if page === 'confirmationSend'}
<ConfirmationSend />
{:else if page === 'auth'}
<Auth />
{/if}
</div>
</Scroller>
</div>
<Popup />
</div>
</div>
{/if}
<style lang="scss">
.back-image {
plugins/login-resources/src/components/SelectWorkspace.svelte
+9 -5
View File
@@ -31,6 +31,7 @@
import login from '../plugin'
import { getAccount, getHref, getWorkspaces, goTo, navigateToWorkspace, selectWorkspace } from '../utils'
import StatusControl from './StatusControl.svelte'
import { isArchivingMode } from '@hcengineering/core'
export let navigateUrl: string | undefined = undefined
let workspaces: Workspace[] = []
@@ -45,7 +46,8 @@
account = await getAccount()
}
const updateWorkspaces = reduceCalls(async function updateWorkspaces (time: number): Promise<void> {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const updateWorkspaces = reduceCalls(async function updateWorkspaces (_time: number): Promise<void> {
try {
workspaces = await getWorkspaces()
} catch (e) {
@@ -53,7 +55,9 @@
}
})
$: if (flagToUpdateWorkspaces) updateWorkspaces($ticker)
$: if (flagToUpdateWorkspaces) {
void updateWorkspaces($ticker)
}
onMount(() => {
void loadAccount()
@@ -132,10 +136,10 @@
<div class="flex flex-col flex-grow">
<span class="label overflow-label flex-center">
{wsName}
{#if workspace.mode === 'archived'}
{#if isArchivingMode(workspace.mode)}
- <Label label={presentation.string.Archived} />
{/if}
{#if workspace.mode === 'creating'}
{#if workspace.mode !== 'active'}
({workspace.progress}%)
{/if}
</span>
@@ -178,7 +182,7 @@
</Scroller>
<div class="grow-separator" />
<div class="footer">
{#if workspaces.length}
{#if workspaces.length > 0}
<div>
<span><Label label={login.string.WantAnotherWorkspace} /></span>
<NavLink
plugins/login-resources/src/index.ts
+1
View File
@@ -58,6 +58,7 @@ export const pages = [
'password',
'recovery',
'selectWorkspace',
'admin',
'join',
'confirm',
'confirmationSend',
plugins/login-resources/src/utils.ts
+106 -5
View File
@@ -14,7 +14,7 @@
//
import { Analytics } from '@hcengineering/analytics'
import { AccountRole, concatLink, type Doc, type Ref } from '@hcengineering/core'
import { AccountRole, concatLink, type BaseWorkspaceInfo, type Doc, type Ref } from '@hcengineering/core'
import { loginId, type LoginInfo, type OtpInfo, type Workspace, type WorkspaceLoginInfo } from '@hcengineering/login'
import {
OK,
@@ -27,7 +27,7 @@ import {
unknownError,
unknownStatus
} from '@hcengineering/platform'
import presentation from '@hcengineering/presentation'
import presentation, { isAdminUser } from '@hcengineering/presentation'
import {
fetchMetadataLocalStorage,
getCurrentLocation,
@@ -208,10 +208,10 @@ export async function createWorkspace (
}
}
function getLastVisitDays (it: Workspace): number {
function getLastVisitDays (it: Pick<Workspace, 'lastVisit'>): number {
return Math.floor((Date.now() - it.lastVisit) / (1000 * 3600 * 24))
}
function getWorkspaceSize (it: Workspace): number {
function getWorkspaceSize (it: Pick<Workspace, 'backupInfo'>): number {
let sz = 0
sz += it.backupInfo?.dataSize ?? 0
sz += it.backupInfo?.blobsSize ?? 0
@@ -264,7 +264,108 @@ export async function getWorkspaces (): Promise<Workspace[]> {
})
return workspaces
} catch (err) {
} catch (err: any) {
return []
}
}
// performWorkspaceOperation
export async function performWorkspaceOperation (
workspace: string | string[],
operation: 'archive' | 'migrate-to' | 'unarchive',
...params: any[]
): Promise<boolean> {
const accountsUrl = getMetadata(login.metadata.AccountsUrl)
if (accountsUrl === undefined) {
throw new Error('accounts url not specified')
}
if (!isAdminUser()) {
throw new PlatformError(unknownError('Non admin user'))
}
const token = getMetadata(presentation.metadata.Token)
if (token === undefined) {
const loc = getCurrentLocation()
loc.path[1] = 'login'
loc.path.length = 2
navigate(loc)
return true
}
const request = {
method: 'performWorkspaceOperation',
params: [workspace, operation, ...params] as any[]
}
try {
const response = await fetch(accountsUrl, {
method: 'POST',
headers: {
Authorization: 'Bearer ' + token,
'Content-Type': 'application/json'
},
body: JSON.stringify(request)
})
const result = await response.json()
if (result.error != null) {
throw new PlatformError(result.error)
}
return (result.result as boolean) ?? false
} catch (err: any) {
return false
}
}
export async function getAllWorkspaces (): Promise<BaseWorkspaceInfo[]> {
const accountsUrl = getMetadata(login.metadata.AccountsUrl)
if (accountsUrl === undefined) {
throw new Error('accounts url not specified')
}
const token = getMetadata(presentation.metadata.Token)
if (token === undefined) {
const loc = getCurrentLocation()
loc.path[1] = 'login'
loc.path.length = 2
navigate(loc)
return []
}
const request = {
method: 'getAllWorkspaces',
params: [] as any[]
}
try {
const response = await fetch(accountsUrl, {
method: 'POST',
headers: {
Authorization: 'Bearer ' + token,
'Content-Type': 'application/json'
},
body: JSON.stringify(request)
})
const result = await response.json()
if (result.error != null) {
throw new PlatformError(result.error)
}
const workspaces: BaseWorkspaceInfo[] = result.result
workspaces.sort((a, b) => {
const adays = getLastVisitDays(a)
const bdays = getLastVisitDays(b)
if (adays === bdays) {
return getWorkspaceSize(b) - getWorkspaceSize(a)
}
return b.lastVisit - a.lastVisit
})
return workspaces
} catch (err: any) {
return []
}
}
plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte
+2 -1
View File
@@ -14,6 +14,7 @@
-->
<script lang="ts">
import contact from '@hcengineering/contact'
import { isArchivingMode } from '@hcengineering/core'
import login, { Workspace } from '@hcengineering/login'
import { getMetadata, getResource } from '@hcengineering/platform'
import presentation, { decodeTokenPayload, isAdminUser } from '@hcengineering/presentation'
@@ -179,7 +180,7 @@
<div class="flex-col flex-grow">
<span class="label overflow-label flex flex-grow flex-between">
{wsName}
{#if ws.mode === 'archived'}
{#if isArchivingMode(ws.mode)}
- <Label label={presentation.string.Archived} />
{/if}
{#if ws.region != null && ws.region !== ''}
server/account/src/collections/mongo.ts
+27 -7
View File
@@ -217,6 +217,16 @@ export class WorkspaceMongoDbCollection extends MongoDbCollection<Workspace> imp
): Promise<WorkspaceInfo | undefined> {
const pendingCreationQuery: Filter<Workspace>['$or'] = [{ mode: { $in: ['pending-creation', 'creating'] } }]
const migrationQuery: Filter<Workspace>['$or'] = [
{ mode: { $in: ['migration-backup', 'migration-pending-backup', 'migration-clean', 'migration-pending-clean'] } }
]
const archivingQuery: Filter<Workspace>['$or'] = [
{ mode: { $in: ['archiving-pending-backup', 'archiving-backup', 'archiving-pending-clean', 'archiving-clean'] } }
]
const restoreQuery: Filter<Workspace>['$or'] = [{ mode: { $in: ['pending-restore', 'restoring'] } }]
const versionQuery = {
$or: [
{ 'version.major': { $lt: version.major } },
@@ -252,13 +262,23 @@ export class WorkspaceMongoDbCollection extends MongoDbCollection<Workspace> imp
// to clear them with the worker.
const defaultRegionQuery = { $or: [{ region: { $exists: false } }, { region: '' }] }
const operationQuery = {
$or:
operation === 'create'
? pendingCreationQuery
: operation === 'upgrade'
? pendingUpgradeQuery
: [...pendingCreationQuery, ...pendingUpgradeQuery]
let operationQuery: Filter<Workspace> = {}
switch (operation) {
case 'create':
operationQuery = { $or: pendingCreationQuery }
break
case 'upgrade':
operationQuery = { $or: pendingUpgradeQuery }
break
case 'all':
operationQuery = { $or: [...pendingCreationQuery, ...pendingUpgradeQuery] }
break
case 'all+backup':
operationQuery = {
$or: [...pendingCreationQuery, ...pendingUpgradeQuery, ...migrationQuery, ...archivingQuery, ...restoreQuery]
}
break
}
const attemptsQuery = { $or: [{ attempts: { $exists: false } }, { attempts: { $lte: 3 } }] }
server/account/src/collections/postgres.ts
+22 -6
View File
@@ -377,15 +377,31 @@ export class WorkspacePostgresDbCollection extends PostgresDbCollection<Workspac
const values: any[] = []
const pendingCreationSql = "mode IN ('pending-creation', 'creating')"
const migrationSql =
"mode IN ('migration-backup', 'migration-pending-backup', 'migration-clean', 'migration-pending-clean')"
const restoringSql = "mode IN ('pending-restore', 'restoring')"
const archivingSql = "'archiving-pending-backup', 'archiving-backup', 'archiving-pending-clean', 'archiving-clean')"
const versionSql =
'("versionMajor" < $1) OR ("versionMajor" = $1 AND "versionMinor" < $2) OR ("versionMajor" = $1 AND "versionMinor" = $2 AND "versionPatch" < $3)'
const pendingUpgradeSql = `(((disabled = FALSE OR disabled IS NULL) AND (mode = 'active' OR mode IS NULL) AND ${versionSql} ${wsLivenessMs !== undefined ? 'AND "lastVisit" > $4' : ''}) OR ((disabled = FALSE OR disabled IS NULL) AND mode = 'upgrading'))`
const operationSql =
operation === 'create'
? pendingCreationSql
: operation === 'upgrade'
? pendingUpgradeSql
: `(${pendingCreationSql} OR ${pendingUpgradeSql})`
let operationSql: string = ''
switch (operation) {
case 'create':
operationSql = pendingCreationSql
break
case 'upgrade':
operationSql = pendingUpgradeSql
break
case 'all':
operationSql = `(${pendingCreationSql} OR ${pendingUpgradeSql})`
break
case 'all+backup':
operationSql = `(${pendingCreationSql} OR ${pendingUpgradeSql} OR ${migrationSql} OR ${archivingSql} OR ${restoringSql})`
break
}
if (operation === 'upgrade' || operation === 'all') {
values.push(version.major, version.minor, version.patch)
server/account/src/operations.ts
+218 -18
View File
@@ -31,6 +31,8 @@ import core, {
Data,
generateId,
getWorkspaceId,
isArchivingMode,
isMigrationMode,
isWorkspaceCreating,
MeasureContext,
RateLimiter,
@@ -42,17 +44,26 @@ import core, {
versionToString,
WorkspaceId,
type BackupStatus,
type BaseWorkspaceInfo,
type Branding,
type WorkspaceMode
type WorkspaceMode,
type WorkspaceUpdateEvent
} from '@hcengineering/core'
import platform, { getMetadata, PlatformError, Severity, Status, translate } from '@hcengineering/platform'
import platform, {
getMetadata,
PlatformError,
Severity,
Status,
translate,
unknownError
} from '@hcengineering/platform'
import { type StorageAdapter } from '@hcengineering/server-core'
import { decodeToken as decodeTokenRaw, generateToken, type Token } from '@hcengineering/server-token'
import { connect } from '@hcengineering/server-tool'
import { randomBytes } from 'crypto'
import { type MongoClient } from 'mongodb'
import otpGenerator from 'otp-generator'
import { getWorkspaceDestroyAdapter } from '@hcengineering/server-pipeline'
import { accountPlugin } from './plugin'
import type {
Account,
@@ -67,7 +78,6 @@ import type {
RegionInfo,
UpgradeStatistic,
Workspace,
WorkspaceEvent,
WorkspaceInfo,
WorkspaceLoginInfo,
WorkspaceOperation
@@ -132,6 +142,16 @@ export async function getWorkspaceById (db: AccountDB, workspace: string): Promi
return await db.workspace.findOne({ workspace })
}
/**
* @public
* @param db -
* @param workspace -
* @returns
*/
export async function getWorkspacesById (db: AccountDB, workspace: string | string[]): Promise<Workspace[]> {
return await db.workspace.find(Array.isArray(workspace) ? { workspace: { $in: workspace } } : { workspace })
}
async function getAccountInfo (
ctx: MeasureContext,
db: AccountDB,
@@ -466,7 +486,7 @@ export async function selectWorkspace (
}
if (workspaceInfo !== null) {
if (workspaceInfo.mode === 'archived') {
if (isArchivingMode(workspaceInfo.mode) || isMigrationMode(workspaceInfo.mode)) {
const result: WorkspaceLoginInfo = {
endpoint: '',
email,
@@ -1112,13 +1132,13 @@ export async function updateWorkspaceInfo (
branding: Branding | null,
token: string,
workspaceId: string,
event: WorkspaceEvent,
event: WorkspaceUpdateEvent,
version: Data<Version>, // A worker version
progress: number,
message?: string
): Promise<void> {
const decodedToken = decodeToken(ctx, token)
if (decodedToken.extra?.service !== 'workspace') {
if (decodedToken.extra?.service !== 'workspace' && decodedToken.email !== systemAccountEmail) {
throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {}))
}
const workspaceInfo = await getWorkspaceById(db, workspaceId)
@@ -1127,7 +1147,9 @@ export async function updateWorkspaceInfo (
}
progress = Math.round(progress)
const update: Partial<WorkspaceInfo> = {}
const update: Partial<WorkspaceInfo> = {
lastProcessingTime: Date.now()
}
switch (event) {
case 'create-started':
update.mode = 'creating'
@@ -1167,6 +1189,56 @@ export async function updateWorkspaceInfo (
case 'progress':
update.progress = progress
break
case 'migrate-backup-started':
update.mode = 'migration-backup'
update.progress = progress
break
case 'migrate-backup-done':
update.mode = 'migration-pending-clean'
update.progress = progress
update.lastProcessingTime = Date.now() - processingTimeoutMs // To not wait for next step
break
case 'migrate-clean-started':
update.mode = 'migration-clean'
update.attempts = 0
update.progress = progress
break
case 'migrate-clean-done':
update.region = workspaceInfo.targetRegion ?? ''
update.mode = 'pending-restore'
update.progress = progress
update.lastProcessingTime = Date.now() - processingTimeoutMs // To not wait for next step
break
case 'restore-started':
update.mode = 'restoring'
update.attempts = 0
update.progress = progress
break
case 'restore-done':
update.mode = 'active'
update.progress = 100
break
case 'archiving-backup-started':
update.mode = 'archiving-backup'
update.attempts = 0
update.progress = progress
break
case 'archiving-backup-done':
update.mode = 'archiving-pending-clean'
update.progress = progress
update.lastProcessingTime = Date.now() - processingTimeoutMs // To not wait for next step
break
case 'archiving-clean-started':
update.mode = 'archiving-clean'
update.attempts = 0
update.progress = progress
break
case 'archiving-clean-done':
update.mode = 'archived'
update.progress = 100
break
case 'ping':
default:
break
@@ -1179,12 +1251,104 @@ export async function updateWorkspaceInfo (
await db.workspace.updateOne(
{ _id: workspaceInfo._id },
{
...update,
lastProcessingTime: Date.now()
...update
}
)
}
/**
* @public
*/
export async function performWorkspaceOperation (
ctx: MeasureContext,
db: AccountDB,
branding: Branding | null,
token: string,
workspaceId: string | string[],
event: 'archive' | 'migrate-to' | 'unarchive',
...params: any
): Promise<boolean> {
const decodedToken = decodeToken(ctx, token)
const account = await getAccount(db, decodedToken.email)
if (account === null) {
ctx.error('account not found', { email: decodedToken.email })
return false
}
if (account.admin !== true) {
return false
}
const workspaceInfos = await getWorkspacesById(db, workspaceId)
if (workspaceInfos.length === 0) {
throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspace: workspaceId }))
}
let ops = 0
for (const workspaceInfo of workspaceInfos) {
const update: Partial<WorkspaceInfo> = {}
switch (event) {
case 'archive':
if (workspaceInfo.mode !== 'active') {
throw new PlatformError(unknownError('Archive allowed only for active workspaces'))
}
update.mode = 'archiving-pending-backup'
update.attempts = 0
update.progress = 0
update.lastProcessingTime = Date.now() - processingTimeoutMs // To not wait for next step
break
case 'unarchive':
if (event === 'unarchive') {
if (workspaceInfo.mode !== 'archived') {
throw new PlatformError(unknownError('Unarchive allowed only for archived workspaces'))
}
}
update.mode = 'pending-restore'
update.attempts = 0
update.progress = 0
update.lastProcessingTime = Date.now() - processingTimeoutMs // To not wait for next step
break
case 'migrate-to': {
if (workspaceInfo.mode !== 'active') {
return false
}
if (params.length !== 1 && params[0] == null) {
throw new PlatformError(unknownError('Invalid region passed to migrate operation'))
}
const regions = getRegions()
if (regions.find((it) => it.region === params[0]) === undefined) {
throw new PlatformError(unknownError('Invalid region passed to migrate operation'))
}
if ((workspaceInfo.region ?? '') === params[0]) {
throw new PlatformError(unknownError('Invalid region passed to migrate operation'))
}
update.mode = 'migration-pending-backup'
update.targetRegion = params[0]
update.attempts = 0
update.progress = 0
update.lastProcessingTime = Date.now() - processingTimeoutMs // To not wait for next step
break
}
default:
break
}
if (Object.keys(update).length !== 0) {
await db.workspace.updateOne(
{ _id: workspaceInfo._id },
{
...update
}
)
ops++
}
}
return ops > 0
}
/**
* @public
*/
@@ -1294,6 +1458,8 @@ async function postUpgradeUserWorkspace (
)
}
const processingTimeoutMs = 30 * 1000
/**
* Retrieves one workspace for which there are things to process.
*
@@ -1316,7 +1482,6 @@ export async function getPendingWorkspace (
throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {}))
}
// Move to config?
const processingTimeoutMs = 30 * 1000
const wsLivenessDays = getMetadata(accountPlugin.metadata.WsLivenessDays)
const wsLivenessMs = wsLivenessDays !== undefined ? wsLivenessDays * 24 * 60 * 60 * 1000 : undefined
@@ -1448,19 +1613,50 @@ export async function getUserWorkspaces (
return []
}
if (account.admin !== true && account.workspaces.length === 0) {
if (account.workspaces.length === 0) {
return []
}
return (
await db.workspace.find(account.admin === true ? {} : { _id: { $in: account.workspaces } }, {
lastVisit: 'descending'
})
await db.workspace.find(
{ _id: { $in: account.workspaces } },
{
lastVisit: 'descending'
}
)
)
.filter((it) => it.disabled !== true || isWorkspaceCreating(it.mode))
.map(mapToClientWorkspace)
}
/**
* Admin only operation to list workspaced based on last visit, for admin purposes.
*
* @public
*/
export async function getAllWorkspaces (
ctx: MeasureContext,
db: AccountDB,
branding: Branding | null,
token: string
): Promise<BaseWorkspaceInfo[]> {
const { email } = decodeToken(ctx, token)
const account = await getAccount(db, email)
if (account === null) {
ctx.error('account not found', { email })
return []
}
if (account.admin !== true) {
return []
}
return (await db.workspace.find({})).map((it) => {
it.accounts = it.accounts.map((it) => it.toString())
return it
})
}
export type ClientWSInfoWithUpgrade = ClientWorkspaceInfo & {
upgrade?: {
toProcess: number
@@ -2097,14 +2293,16 @@ export async function dropWorkspace (
export async function dropWorkspaceFull (
ctx: MeasureContext,
db: AccountDB,
client: MongoClient,
dbUrl: string,
branding: Branding | null,
workspaceId: string,
storageAdapter?: StorageAdapter
): Promise<void> {
const ws = await dropWorkspace(ctx, db, branding, workspaceId)
const workspaceDb = client.db(ws.workspace)
await workspaceDb.dropDatabase()
const adapter = getWorkspaceDestroyAdapter(dbUrl)
await adapter.deleteWorkspace(ctx, { name: ws.workspace })
const wspace = getWorkspaceId(workspaceId)
const hasBucket = await storageAdapter?.exists(ctx, wspace)
if (storageAdapter !== undefined && hasBucket === true) {
@@ -2562,6 +2760,8 @@ export function getMethods (hasSignUp: boolean = true): Record<string, AccountMe
selectWorkspace: wrap(selectWorkspace),
getRegionInfo: wrap(getRegionInfo),
getUserWorkspaces: wrap(getUserWorkspaces),
performWorkspaceOperation: wrap(performWorkspaceOperation),
getAllWorkspaces: wrap(getAllWorkspaces),
getInviteLink: wrap(getInviteLink),
getAccountInfo: wrap(getAccountInfo),
getWorkspaceInfo: wrap(getWorkspaceInfo),
server/account/src/types.ts
+1 -7
View File
@@ -56,7 +56,6 @@ export interface Workspace extends BaseWorkspaceInfo {
_id: ObjectId
accounts: ObjectId[]
region?: string // Transactor group name
lastProcessingTime?: number
attempts?: number
message?: string
@@ -131,12 +130,7 @@ export type AccountInfo = Omit<Account, 'hash' | 'salt'>
/**
* @public
*/
export type WorkspaceEvent = 'ping' | 'create-started' | 'upgrade-started' | 'progress' | 'create-done' | 'upgrade-done'
/**
* @public
*/
export type WorkspaceOperation = 'create' | 'upgrade' | 'all'
export type WorkspaceOperation = 'create' | 'upgrade' | 'all' | 'all+backup'
/**
* @public
server/account/src/utils.ts
+21 -7
View File
@@ -119,15 +119,29 @@ const getEndpoints = (): string[] => {
return endpoints
}
// Info is static, so no need to calculate it every time.
let regionInfo: RegionInfo[] = []
export const getRegions = (): RegionInfo[] => {
if (process.env.REGION_INFO !== undefined) {
return process.env.REGION_INFO.split(';')
.map((it) => it.split('|'))
.map((it) => ({ region: it[0].trim(), name: it[1].trim() }))
if (regionInfo.length === 0) {
const endpoints = getEndpoints()
.map(toTransactor)
.map((it) => ({ region: it.region.trim(), name: '' }))
if (process.env.REGION_INFO !== undefined) {
regionInfo = process.env.REGION_INFO.split(';')
.map((it) => it.split('|'))
.map((it) => ({ region: it[0].trim(), name: it[1].trim() }))
// We need to add all endpoints if they are not in info.
for (const endpoint of endpoints) {
if (regionInfo.find((it) => it.region === endpoint.region) === undefined) {
regionInfo.push(endpoint)
}
}
} else {
regionInfo = endpoints
}
}
return getEndpoints()
.map(toTransactor)
.map((it) => ({ region: it.region.trim(), name: '' }))
return regionInfo
}
export const getEndpoint = (ctx: MeasureContext, workspaceInfo: WorkspaceInfo, kind: EndpointKind): string => {
server/backup-service/src/index.ts
+2 -1
View File
@@ -129,7 +129,8 @@ export async function backupWorkspace (
region,
freshBackup,
clean,
downloadLimit
downloadLimit,
[]
)
if (result && onFinish !== undefined) {
await onFinish(storageAdapter, workspaceStorageAdapter)
server/backup/src/backup.ts
+84 -31
View File
@@ -33,6 +33,7 @@ import core, {
SortingOrder,
systemAccountEmail,
toIdMap,
toWorkspaceString,
TxProcessor,
WorkspaceId,
type BackupStatus,
@@ -41,7 +42,7 @@ import core, {
type Tx,
type TxCUD
} from '@hcengineering/core'
import { BlobClient, createClient } from '@hcengineering/server-client'
import { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client'
import { estimateDocSize, type StorageAdapter } from '@hcengineering/server-core'
import { generateToken } from '@hcengineering/server-token'
import { connect } from '@hcengineering/server-tool'
@@ -672,7 +673,7 @@ export async function backup (
storageAdapter?: StorageAdapter
// Return true in case
isCanceled?: () => boolean
progress?: (progress: number) => void
progress?: (progress: number) => Promise<void>
token?: string
} = {
force: false,
@@ -765,12 +766,7 @@ export async function backup (
connection =
options.getConnection !== undefined
? await options.getConnection()
: ((await createClient(
transactorUrl,
options.token ?? token,
undefined,
options.connectTimeout
)) as CoreClient & BackupClient)
: ((await createClient(transactorUrl, token, undefined, options.connectTimeout)) as CoreClient & BackupClient)
if (!lastTxChecked && !options.freshBackup) {
lastTx = await connection.findOne(
@@ -975,7 +971,7 @@ export async function backup (
async function processDomain (
ctx: MeasureContext,
domain: Domain,
progress: (value: number) => void
progress: (value: number) => Promise<void>
): Promise<void> {
const changes: Snapshot = {
added: new Map(),
@@ -1007,13 +1003,18 @@ export async function backup (
let _packClose = async (): Promise<void> => {}
let addedDocuments = (): number => 0
progress(0)
if (progress !== undefined) {
await progress(0)
}
let { changed, needRetrieveChunks } = await ctx.with('load-chunks', { domain }, (ctx) =>
loadChangesFromServer(ctx, domain, digest, changes)
)
processedChanges.removed = Array.from(digest.keys())
digest.clear()
progress(10)
if (progress !== undefined) {
await progress(10)
}
if (needRetrieveChunks.length > 0) {
ctx.info('dumping domain...', { workspace: workspaceId.name, domain })
@@ -1149,7 +1150,6 @@ export async function backup (
function processChanges (d: Doc, error: boolean = false): void {
processed++
progress(10 + (processed / totalChunks) * 90)
// Move processed document to processedChanges
if (changes.added.has(d._id)) {
if (!error) {
@@ -1175,6 +1175,9 @@ export async function backup (
limit: options.blobDownloadLimit
})
processChanges(d, true)
if (progress !== undefined) {
await progress(10 + (processed / totalChunks) * 90)
}
continue
}
@@ -1188,6 +1191,9 @@ export async function backup (
size: blob.size / (1024 * 1024)
})
processChanges(d, true)
if (progress !== undefined) {
await progress(10 + (processed / totalChunks) * 90)
}
continue
}
@@ -1298,12 +1304,15 @@ export async function backup (
current: Math.round(process.memoryUsage().heapUsed / (1024 * 1024))
})
await ctx.with('process-domain', { domain }, async (ctx) => {
await processDomain(ctx, domain, (value) => {
options.progress?.(Math.round(((domainProgress + value / 100) / domains.length) * 100))
})
await processDomain(
ctx,
domain,
(value) =>
options.progress?.(Math.round(((domainProgress + value / 100) / domains.length) * 100)) ?? Promise.resolve()
)
})
domainProgress++
options.progress?.(Math.round((domainProgress / domains.length) * 10000) / 100)
await options.progress?.(Math.round((domainProgress / domains.length) * 10000) / 100)
}
if (!canceled()) {
backupInfo.lastTxId = lastTx?._id ?? '0' // We could store last tx, since full backup is complete
@@ -1639,8 +1648,12 @@ export async function restore (
recheck?: boolean
include?: Set<string>
skip?: Set<string>
getConnection?: () => Promise<CoreClient & BackupClient>
storageAdapter?: StorageAdapter
token?: string
progress?: (progress: number) => Promise<void>
}
): Promise<void> {
): Promise<boolean> {
const infoFile = 'backup.json.gz'
if (!(await storage.exists(infoFile))) {
@@ -1670,14 +1683,34 @@ export async function restore (
ctx.info('connecting:', { transactorUrl, workspace: workspaceId.name })
const token = generateToken(systemAccountEmail, workspaceId, {
mode: 'backup',
model: 'upgrade'
})
const token =
opt.token ??
generateToken(systemAccountEmail, workspaceId, {
mode: 'backup',
model: 'upgrade'
})
const connection = (await createClient(transactorUrl, token)) as CoreClient & BackupClient
const connection =
opt.getConnection !== undefined
? await opt.getConnection()
: ((await createClient(transactorUrl, token)) as CoreClient & BackupClient)
const blobClient = new BlobClient(transactorUrl, token, workspaceId)
if (opt.getConnection === undefined) {
try {
let serverEndpoint = await getTransactorEndpoint(token, 'external')
serverEndpoint = serverEndpoint.replaceAll('wss://', 'https://').replace('ws://', 'http://')
await fetch(
serverEndpoint + `/api/v1/manage?token=${token}&operation=force-close&wsId=${toWorkspaceString(workspaceId)}`,
{
method: 'PUT'
}
)
} catch (err: any) {
// Ignore
}
}
const blobClient = new BlobClient(transactorUrl, token, workspaceId, { storageAdapter: opt.storageAdapter })
console.log('connected')
// We need to find empty domains and clean them.
@@ -1692,6 +1725,8 @@ export async function restore (
let uploadedMb = 0
let uploaded = 0
let domainProgress = 0
const printUploaded = (msg: string, size: number): void => {
if (size == null) {
return
@@ -1726,6 +1761,9 @@ export async function restore (
let chunks = 0
try {
while (true) {
if (opt.progress !== undefined) {
await opt.progress?.(domainProgress)
}
const st = Date.now()
const it = await connection.loadChunk(c, idx)
chunks++
@@ -1776,6 +1814,9 @@ export async function restore (
let sendSize = 0
let totalSend = 0
async function sendChunk (doc: Doc | undefined, len: number): Promise<void> {
if (opt.progress !== undefined) {
await opt.progress?.(domainProgress)
}
if (doc !== undefined) {
docsToAdd.delete(doc._id)
docs.push(doc)
@@ -1824,13 +1865,13 @@ export async function restore (
}
return true
})
} else {
try {
await connection.upload(c, docsToSend)
} catch (err: any) {
ctx.error('error during upload', { err, docs: JSON.stringify(docs) })
}
}
try {
await connection.upload(c, docsToSend)
} catch (err: any) {
ctx.error('error during upload', { err, docs: JSON.stringify(docs) })
}
docs.length = 0
sendSize = 0
}
@@ -2004,7 +2045,11 @@ export async function restore (
const limiter = new RateLimiter(opt.parallel ?? 1)
try {
let i = 0
for (const c of domains) {
if (opt.progress !== undefined) {
await opt.progress?.(domainProgress)
}
if (opt.include !== undefined && !opt.include.has(c)) {
continue
}
@@ -2032,13 +2077,21 @@ export async function restore (
}
}
}
domainProgress = Math.round(i / domains.size) * 100
i++
})
}
await limiter.waitProcessing()
} catch (err: any) {
Analytics.handleError(err)
return false
} finally {
await connection.sendForceClose()
await connection.close()
if (opt.getConnection === undefined && connection !== undefined) {
await connection.sendForceClose()
await connection.close()
}
}
return true
}
/**
server/backup/src/service.ts
+78 -7
View File
@@ -19,6 +19,7 @@ import core, {
DOMAIN_TX,
getWorkspaceId,
Hierarchy,
isActiveMode,
ModelDb,
SortingOrder,
systemAccountEmail,
@@ -37,7 +38,7 @@ import {
type StorageAdapter
} from '@hcengineering/server-core'
import { generateToken } from '@hcengineering/server-token'
import { backup } from '.'
import { backup, restore } from '.'
import { createStorageBackupStorage } from './storage'
export interface BackupConfig {
AccountsURL: string
@@ -66,7 +67,8 @@ class BackupWorker {
) => DbConfiguration,
readonly region: string,
readonly freshWorkspace: boolean = false,
readonly clean: boolean = false
readonly clean: boolean = false,
readonly skipDomains: string[] = []
) {}
canceled = false
@@ -115,6 +117,12 @@ class BackupWorker {
let skipped = 0
const allWorkspaces = await listAccountWorkspaces(this.config.Token, this.region)
const workspaces = allWorkspaces.filter((it) => {
if (!isActiveMode(it.mode)) {
// We should backup only active workspaces
skipped++
return false
}
const lastBackup = it.backupInfo?.lastBackup ?? 0
if ((Date.now() - lastBackup) / 1000 < this.config.Interval) {
// No backup required, interval not elapsed
@@ -151,7 +159,8 @@ class BackupWorker {
async doBackup (
rootCtx: MeasureContext,
workspaces: BaseWorkspaceInfo[],
recheckTimeout: number
recheckTimeout: number,
notify?: (progress: number) => Promise<void>
): Promise<{ failedWorkspaces: BaseWorkspaceInfo[], processed: number, skipped: number }> {
let index = 0
@@ -185,7 +194,7 @@ class BackupWorker {
}
const result = await ctx.with('backup', { workspace: ws.workspace }, (ctx) =>
backup(ctx, '', getWorkspaceId(ws.workspace), storage, {
skipDomains: [],
skipDomains: this.skipDomains,
force: true,
freshBackup: this.freshWorkspace,
clean: this.clean,
@@ -226,6 +235,9 @@ class BackupWorker {
pipeline = await this.pipelineFactory(ctx, wsUrl, true, () => {}, null)
}
return wrapPipeline(ctx, pipeline, wsUrl)
},
progress: (progress) => {
return notify?.(progress) ?? Promise.resolve()
}
})
)
@@ -307,7 +319,9 @@ export async function doBackupWorkspace (
region: string,
freshWorkspace: boolean,
clean: boolean,
downloadLimit: number
downloadLimit: number,
skipDomains: string[],
notify?: (progress: number) => Promise<void>
): Promise<boolean> {
const backupWorker = new BackupWorker(
storage,
@@ -317,10 +331,67 @@ export async function doBackupWorkspace (
getConfig,
region,
freshWorkspace,
clean
clean,
skipDomains
)
backupWorker.downloadLimit = downloadLimit
const { processed } = await backupWorker.doBackup(ctx, [workspace], Number.MAX_VALUE)
const { processed } = await backupWorker.doBackup(ctx, [workspace], Number.MAX_VALUE, notify)
await backupWorker.close()
return processed === 1
}
export async function doRestoreWorkspace (
rootCtx: MeasureContext,
ws: BaseWorkspaceInfo,
backupAdapter: StorageAdapter,
bucketName: string,
pipelineFactory: PipelineFactory,
workspaceStorageAdapter: StorageAdapter,
getConfig: (
ctx: MeasureContext,
workspace: WorkspaceIdWithUrl,
branding: Branding | null,
externalStorage: StorageAdapter
) => DbConfiguration,
skipDomains: string[],
notify?: (progress: number) => Promise<void>
): Promise<boolean> {
rootCtx.warn('\nRESTORE WORKSPACE ', {
workspace: ws.workspace
})
const ctx = rootCtx.newChild(ws.workspace, { workspace: ws.workspace })
let pipeline: Pipeline | undefined
try {
const storage = await createStorageBackupStorage(ctx, backupAdapter, getWorkspaceId(bucketName), ws.workspace)
const wsUrl: WorkspaceIdWithUrl = {
name: ws.workspace,
workspaceName: ws.workspaceName ?? '',
workspaceUrl: ws.workspaceUrl ?? ''
}
const result: boolean = await ctx.with('restore', { workspace: ws.workspace }, (ctx) =>
restore(ctx, '', getWorkspaceId(ws.workspace), storage, {
date: -1,
skip: new Set(skipDomains),
recheck: true,
storageAdapter: workspaceStorageAdapter,
getConnection: async () => {
if (pipeline === undefined) {
pipeline = await pipelineFactory(ctx, wsUrl, true, () => {}, null)
}
return wrapPipeline(ctx, pipeline, wsUrl)
},
progress: (progress) => {
return notify?.(progress) ?? Promise.resolve()
}
})
)
return result
} catch (err: any) {
rootCtx.error('\n\nFAILED to RESTORE', { workspace: ws.workspace, err })
return false
} finally {
if (pipeline !== undefined) {
await pipeline.close()
}
}
}
server/client/src/account.ts
+8 -7
View File
@@ -14,13 +14,14 @@
//
import {
AccountRole,
BackupStatus,
Doc,
Ref,
type BaseWorkspaceInfo,
type Data,
type Version,
BackupStatus,
AccountRole,
Ref,
Doc
type WorkspaceUpdateEvent
} from '@hcengineering/core'
import { getMetadata, PlatformError, unknownError } from '@hcengineering/platform'
@@ -164,7 +165,7 @@ export async function getPendingWorkspace (
token: string,
region: string,
version: Data<Version>,
operation: 'create' | 'upgrade' | 'all'
operation: 'create' | 'upgrade' | 'all' | 'all+backup'
): Promise<BaseWorkspaceInfo | undefined> {
const accountsUrl = getAccoutsUrlOrFail()
const workspaces = await (
@@ -186,7 +187,7 @@ export async function getPendingWorkspace (
export async function updateWorkspaceInfo (
token: string,
workspaceId: string,
event: 'ping' | 'create-started' | 'upgrade-started' | 'progress' | 'create-done' | 'upgrade-done',
event: WorkspaceUpdateEvent,
version: Data<Version>,
progress: number,
message?: string
@@ -210,7 +211,7 @@ export async function workerHandshake (
token: string,
region: string,
version: Data<Version>,
operation: 'create' | 'upgrade' | 'all'
operation: 'create' | 'upgrade' | 'all' | 'all+backup'
): Promise<void> {
const accountsUrl = getAccoutsUrlOrFail()
await fetch(accountsUrl, {
server/core/src/adapter.ts
+8
View File
@@ -98,6 +98,14 @@ export interface TxAdapter extends DbAdapter {
getModel: (ctx: MeasureContext) => Promise<Tx[]>
}
/**
* Adpater to delete a selected workspace and all its data.
* @public
*/
export interface WorkspaceDestroyAdapter {
deleteWorkspace: (ctx: MeasureContext, workspace: WorkspaceId) => Promise<void>
}
/**
* @public
*/
server/mongo/src/index.ts
+20
View File
@@ -14,5 +14,25 @@
// limitations under the License.
//
import type { WorkspaceDestroyAdapter } from '@hcengineering/server-core'
import { getMongoClient, getWorkspaceMongoDB } from './utils'
export * from './storage'
export * from './utils'
export function createMongoDestroyAdapter (url: string): WorkspaceDestroyAdapter {
return {
deleteWorkspace: async (ctx, workspace): Promise<void> => {
const client = getMongoClient(url)
try {
await ctx.with('delete-workspace', {}, async () => {
const dbClient = await client.getClient()
const db = getWorkspaceMongoDB(dbClient, workspace)
await db.dropDatabase()
})
} finally {
client.close()
}
}
}
}
server/postgres/src/index.ts
+32 -2
View File
@@ -13,6 +13,36 @@
// limitations under the License.
//
export * from './storage'
export { getDBClient, convertDoc, createTables, retryTxn } from './utils'
import type { WorkspaceDestroyAdapter } from '@hcengineering/server-core'
import { domainSchemas } from './schemas'
import { getDBClient, retryTxn } from './utils'
export { getDocFieldsByDomains, translateDomain } from './schemas'
export * from './storage'
export { convertDoc, createTables, getDBClient, retryTxn } from './utils'
export function createPostgreeDestroyAdapter (url: string): WorkspaceDestroyAdapter {
return {
deleteWorkspace: async (ctx, workspace): Promise<void> => {
const client = getDBClient(url)
try {
const connection = await client.getClient()
await ctx.with('delete-workspace', {}, async () => {
// We need to clear information about workspace from all collections in schema
for (const [domain] of Object.entries(domainSchemas)) {
await ctx.with('delete-workspace-domain', {}, async () => {
await retryTxn(connection, async (client) => {
await client`delete from ${connection(domain)} where "workspaceId" = '${connection(workspace.name)}'`
})
})
}
})
} catch (err: any) {
ctx.error('failed to clean workspace data', { err })
} finally {
client.close()
}
}
}
}
server/server-pipeline/src/pipeline.ts
+10 -5
View File
@@ -37,8 +37,8 @@ import {
TriggersMiddleware,
TxMiddleware
} from '@hcengineering/middleware'
import { createMongoAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
import { createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
import { createMongoAdapter, createMongoDestroyAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
import { createPostgreeDestroyAdapter, createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
import {
createBenchmarkAdapter,
createInMemoryAdapter,
@@ -52,7 +52,8 @@ import {
type PipelineContext,
type PipelineFactory,
type StorageAdapter,
type StorageConfiguration
type StorageConfiguration,
type WorkspaceDestroyAdapter
} from '@hcengineering/server-core'
import { buildStorageFromConfig, createStorageDataAdapter, storageConfigFromEnv } from '@hcengineering/server-storage'
import { generateToken } from '@hcengineering/server-token'
@@ -223,6 +224,10 @@ export async function getServerPipeline (
}
}
export function getWorkspaceDestroyAdapter (dbUrl: string): WorkspaceDestroyAdapter {
return dbUrl.startsWith('mongodb') ? createMongoDestroyAdapter(dbUrl) : createPostgreeDestroyAdapter(dbUrl)
}
export function getConfig (
metrics: MeasureContext,
dbUrl: string,
@@ -250,11 +255,11 @@ export function getConfig (
defaultAdapter: extensions?.defaultAdapter ?? 'Main',
adapters: {
Tx: {
factory: dbUrl.startsWith('postgresql') ? createPostgresTxAdapter : createMongoTxAdapter,
factory: dbUrl.startsWith('mongodb') ? createMongoTxAdapter : createPostgresTxAdapter,
url: dbUrl
},
Main: {
factory: dbUrl.startsWith('postgresql') ? createPostgresAdapter : createMongoAdapter,
factory: dbUrl.startsWith('mongodb') ? createMongoAdapter : createPostgresAdapter,
url: dbUrl
},
Null: {
server/server/src/sessionManager.ts
+12 -1
View File
@@ -19,6 +19,9 @@ import core, {
WorkspaceEvent,
cutObjectArray,
generateId,
isArchivingMode,
isMigrationMode,
isRestoringMode,
isWorkspaceCreating,
systemAccountEmail,
toWorkspaceString,
@@ -346,10 +349,18 @@ class TSessionManager implements SessionManager {
return { upgrade: true }
}
if (workspaceInfo.mode === 'archived') {
if (isArchivingMode(workspaceInfo.mode)) {
// No access to disabled workspaces for regular users
return { error: new Error('Workspace is archived'), terminate: true, archived: true }
}
if (isMigrationMode(workspaceInfo.mode)) {
// No access to disabled workspaces for regular users
return { error: new Error('Workspace is in region migration'), terminate: true, archived: false }
}
if (isRestoringMode(workspaceInfo.mode)) {
// No access to disabled workspaces for regular users
return { error: new Error('Workspace is in backup restore'), terminate: true, archived: false }
}
if (workspaceInfo.disabled === true && token.email !== systemAccountEmail && token.extra?.admin !== 'true') {
// No access to disabled workspaces for regular users
server/workspace-service/package.json
+2 -1
View File
@@ -55,6 +55,7 @@
"@hcengineering/server-client": "^0.6.0",
"@hcengineering/server-token": "^0.6.11",
"@hcengineering/server-notification": "^0.6.1",
"@hcengineering/analytics": "^0.6.0"
"@hcengineering/analytics": "^0.6.0",
"@hcengineering/server-backup": "^0.6.0"
}
}
server/workspace-service/src/index.ts
+28 -6
View File
@@ -26,12 +26,12 @@ import { type MigrateOperation } from '@hcengineering/model'
import { setMetadata } from '@hcengineering/platform'
import serverClientPlugin from '@hcengineering/server-client'
import serverNotification from '@hcengineering/server-notification'
import { createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import serverToken from '@hcengineering/server-token'
import toolPlugin from '@hcengineering/server-tool'
import { WorkspaceWorker } from './service'
import { WorkspaceWorker, type WorkspaceOperation } from './service'
export * from './ws-operations'
/**
* @public
*/
@@ -44,12 +44,33 @@ export function serveWorkspaceAccount (
onClose?: () => void
): void {
const region = process.env.REGION ?? ''
const wsOperation = process.env.WS_OPERATION ?? 'all'
if (wsOperation !== 'all' && wsOperation !== 'create' && wsOperation !== 'upgrade') {
console.log(`Invalid operation provided: ${wsOperation}. Must be one of 'all', 'create', 'upgrade'`)
const wsOperation: WorkspaceOperation = (process.env.WS_OPERATION as WorkspaceOperation) ?? 'all'
if (wsOperation !== 'all' && wsOperation !== 'create' && wsOperation !== 'upgrade' && wsOperation !== 'all+backup') {
console.log(
`Invalid operation provided: ${wsOperation as string}.
Must be one of 'all', 'create', 'upgrade', 'all+backup'`
)
process.exit(1)
}
if (wsOperation === 'all+backup' && process.env.BACKUP_STORAGE === undefined) {
console.log('BACKUP_STORAGE is required for all operation')
process.exit(1)
}
if (wsOperation === 'all+backup' && process.env.BACKUP_BUCKET === undefined) {
console.log('BACKUP_BUCKET is required for all operation')
process.exit(1)
}
const backup =
wsOperation === 'all+backup'
? {
backupStorage: createStorageFromConfig(storageConfigFromEnv(process.env.BACKUP_STORAGE ?? '').storages[0]),
bucketName: process.env.BACKUP_BUCKET ?? 'backup'
}
: undefined
console.log(
'Starting workspace service in region:',
region === '' ? 'DEFAULT' : region,
@@ -110,7 +131,8 @@ export function serveWorkspaceAccount (
force: false,
console: false,
logs: 'upgrade-logs',
waitTimeout
waitTimeout,
backup
},
() => canceled
)
server/workspace-service/src/service.ts
+241 -4
View File
@@ -19,8 +19,12 @@ import {
type MeasureContext,
type Tx,
type Version,
type WorkspaceUpdateEvent,
getBranding,
getWorkspaceId,
isArchivingMode,
isMigrationMode,
isRestoringMode,
systemAccountEmail
} from '@hcengineering/core'
import { type MigrateOperation, type ModelLogger } from '@hcengineering/model'
@@ -32,9 +36,14 @@ import {
workerHandshake
} from '@hcengineering/server-client'
import { generateToken } from '@hcengineering/server-token'
import { FileModelLogger } from '@hcengineering/server-tool'
import { FileModelLogger, prepareTools } from '@hcengineering/server-tool'
import path from 'path'
import { Analytics } from '@hcengineering/analytics'
import { doBackupWorkspace, doRestoreWorkspace } from '@hcengineering/server-backup'
import type { PipelineFactory, StorageAdapter } from '@hcengineering/server-core'
import { createBackupPipeline, getConfig, getWorkspaceDestroyAdapter } from '@hcengineering/server-pipeline'
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import { createWorkspace, upgradeWorkspace } from './ws-operations'
export interface WorkspaceOptions {
@@ -45,8 +54,15 @@ export interface WorkspaceOptions {
ignore?: string
waitTimeout: number
backup?: {
backupStorage: StorageAdapter
bucketName: string
}
}
export type WorkspaceOperation = 'create' | 'upgrade' | 'all' | 'all+backup'
export class WorkspaceWorker {
runningTasks: number = 0
resolveBusy: (() => void) | null = null
@@ -57,7 +73,7 @@ export class WorkspaceWorker {
readonly migrationOperation: [string, MigrateOperation][],
readonly region: string,
readonly limit: number,
readonly operation: 'create' | 'upgrade' | 'all',
readonly operation: WorkspaceOperation,
readonly brandings: BrandingMap
) {}
@@ -214,7 +230,13 @@ export class WorkspaceWorker {
}
private async _upgradeWorkspace (ctx: MeasureContext, ws: BaseWorkspaceInfo, opt: WorkspaceOptions): Promise<void> {
if (ws.disabled === true || ws.mode === 'archived' || (opt.ignore ?? '').includes(ws.workspace)) {
if (
ws.disabled === true ||
isArchivingMode(ws.mode) ||
isMigrationMode(ws.mode) ||
isRestoringMode(ws.mode) ||
(opt.ignore ?? '').includes(ws.workspace)
) {
return
}
const t = Date.now()
@@ -292,14 +314,27 @@ export class WorkspaceWorker {
}
}
async doCleanup (ctx: MeasureContext, workspace: BaseWorkspaceInfo): Promise<void> {
const { dbUrl } = prepareTools([])
const adapter = getWorkspaceDestroyAdapter(dbUrl)
await adapter.deleteWorkspace(ctx, { name: workspace.workspace })
}
private async doWorkspaceOperation (
ctx: MeasureContext,
workspace: BaseWorkspaceInfo,
opt: WorkspaceOptions
): Promise<void> {
const token = generateToken(systemAccountEmail, { name: workspace.workspace }, { service: 'workspace' })
const sendEvent = (event: WorkspaceUpdateEvent, progress: number): Promise<void> =>
withRetryConnUntilSuccess(() =>
updateWorkspaceInfo(token, workspace.workspace, event, this.version, progress, `${event} done`)
)()
switch (workspace.mode ?? 'active') {
case 'creating':
case 'pending-creation':
case 'creating':
// We need to either start workspace creation
// or see if we need to restart it
await this._createWorkspace(ctx, workspace, opt)
@@ -310,6 +345,55 @@ export class WorkspaceWorker {
// It's safe to upgrade the workspace again as the procedure allows re-trying.
await this._upgradeWorkspace(ctx, workspace, opt)
break
case 'archiving-pending-backup':
case 'archiving-backup': {
await sendEvent('archiving-backup-started', 0)
if (await this.doBackup(ctx, workspace, opt, true)) {
await sendEvent('archiving-backup-done', 100)
}
break
}
case 'archiving-pending-clean':
case 'archiving-clean': {
// We should remove DB, not storages.
await sendEvent('archiving-clean-started', 0)
try {
await this.doCleanup(ctx, workspace)
} catch (err: any) {
Analytics.handleError(err)
return
}
await sendEvent('archiving-clean-done', 100)
break
}
case 'migration-pending-backup':
case 'migration-backup':
await sendEvent('migrate-backup-started', 0)
if (await this.doBackup(ctx, workspace, opt, false)) {
await sendEvent('migrate-backup-done', 100)
}
break
case 'migration-pending-clean':
case 'migration-clean': {
// We should remove DB, not storages.
await sendEvent('migrate-clean-started', 0)
try {
await this.doCleanup(ctx, workspace)
} catch (err: any) {
Analytics.handleError(err)
return
}
await sendEvent('migrate-clean-done', 0)
break
}
case 'pending-restore':
case 'restoring':
await sendEvent('restore-started', 0)
if (await this.doRestore(ctx, workspace, opt)) {
await sendEvent('restore-done', 100)
}
break
case 'deleting':
// Seems we failed to delete, so let's restore deletion.
// TODO: move from account
@@ -319,6 +403,159 @@ export class WorkspaceWorker {
}
}
private async doBackup (
ctx: MeasureContext,
workspace: BaseWorkspaceInfo,
opt: WorkspaceOptions,
archive: boolean
): Promise<boolean> {
if (opt.backup === undefined) {
return false
}
const { dbUrl } = prepareTools([])
const workspaceStorageConfig = storageConfigFromEnv()
const workspaceStorageAdapter = buildStorageFromConfig(workspaceStorageConfig)
const pipelineFactory: PipelineFactory = createBackupPipeline(ctx, dbUrl, this.txes, {
externalStorage: workspaceStorageAdapter,
usePassedCtx: true
})
// A token to access account service
const token = generateToken(systemAccountEmail, { name: 'workspace' })
const handleWsEventWithRetry = (
event: 'ping' | 'progress',
version: Data<Version>,
progress: number,
message?: string
): Promise<void> => {
return withRetryConnUntilTimeout(
() => updateWorkspaceInfo(token, workspace.workspace, event, version, progress, message),
5000
)()
}
let progress = 0
const notifyInt = setInterval(() => {
void handleWsEventWithRetry('ping', this.version, progress, '')
}, 5000)
try {
const result: boolean = await doBackupWorkspace(
ctx,
workspace,
opt.backup.backupStorage,
{
Token: token,
BucketName: opt.backup.bucketName,
CoolDown: 0,
Timeout: 0,
SkipWorkspaces: '',
AccountsURL: '',
Interval: 0
},
pipelineFactory,
workspaceStorageAdapter,
(ctx, workspace, branding, externalStorage) => {
return getConfig(ctx, dbUrl, ctx, {
externalStorage,
disableTriggers: true
})
},
this.region,
archive,
archive,
50000,
['blob'],
(_p: number) => {
if (progress !== Math.round(_p)) {
progress = Math.round(_p)
return handleWsEventWithRetry('progress', this.version, progress, '')
}
return Promise.resolve()
}
)
if (result) {
console.log('backup completed')
return true
}
} finally {
clearInterval(notifyInt)
await workspaceStorageAdapter.close()
}
return false
}
private async doRestore (ctx: MeasureContext, workspace: BaseWorkspaceInfo, opt: WorkspaceOptions): Promise<boolean> {
if (opt.backup === undefined) {
return false
}
const { dbUrl } = prepareTools([])
const workspaceStorageConfig = storageConfigFromEnv()
const workspaceStorageAdapter = buildStorageFromConfig(workspaceStorageConfig)
const pipelineFactory: PipelineFactory = createBackupPipeline(ctx, dbUrl, this.txes, {
externalStorage: workspaceStorageAdapter,
usePassedCtx: true
})
// A token to access account service
const token = generateToken(systemAccountEmail, { name: 'workspace' })
const handleWsEventWithRetry = (
event: 'ping' | 'progress',
version: Data<Version>,
progress: number,
message?: string
): Promise<void> => {
return withRetryConnUntilTimeout(
() => updateWorkspaceInfo(token, workspace.workspace, event, version, progress, message),
5000
)()
}
let progress = 0
const notifyInt = setInterval(() => {
void handleWsEventWithRetry('ping', this.version, progress, '')
}, 5000)
try {
const result: boolean = await doRestoreWorkspace(
ctx,
workspace,
opt.backup.backupStorage,
opt.backup.bucketName,
pipelineFactory,
workspaceStorageAdapter,
(ctx, workspace, branding, externalStorage) => {
return getConfig(ctx, dbUrl, ctx, {
externalStorage,
disableTriggers: true
})
},
['blob'],
(_p: number) => {
if (progress !== Math.round(_p)) {
progress = Math.round(_p)
return handleWsEventWithRetry('progress', this.version, progress, '')
}
return Promise.resolve()
}
)
if (result) {
console.log('backup completed')
return true
}
} finally {
clearInterval(notifyInt)
await workspaceStorageAdapter.close()
}
return false
}
private async doSleep (ctx: MeasureContext, opt: WorkspaceOptions): Promise<void> {
await new Promise<void>((resolve) => {
const wakeup: () => void = () => {
server/ws/src/server_http.ts
+2 -2
View File
@@ -14,7 +14,7 @@
//
import { Analytics } from '@hcengineering/analytics'
import { generateId, toWorkspaceString, type MeasureContext, type Tx } from '@hcengineering/core'
import { generateId, systemAccountEmail, toWorkspaceString, type MeasureContext, type Tx } from '@hcengineering/core'
import platform, { Severity, Status, UNAUTHORIZED, unknownStatus } from '@hcengineering/platform'
import { RPCHandler, type Response } from '@hcengineering/rpc'
import {
@@ -127,7 +127,7 @@ export function startHttpServer (
try {
const token = req.query.token as string
const payload = decodeToken(token)
if (payload.extra?.admin !== 'true') {
if (payload.extra?.admin !== 'true' && payload.email !== systemAccountEmail) {
console.warn('Non admin attempt to maintenance action', { payload })
res.writeHead(404, {})
res.end()
services/github/pod-github/src/platform.ts
+3 -2
View File
@@ -10,6 +10,7 @@ import core, {
Client,
ClientConnectEvent,
DocumentUpdate,
isActiveMode,
MeasureContext,
RateLimiter,
Ref,
@@ -748,8 +749,8 @@ export class PlatformWorker {
errors++
return
}
if (workspaceInfo?.mode === 'archived') {
this.ctx.warn('Workspace is archived.', { workspace })
if (!isActiveMode(workspaceInfo?.mode)) {
this.ctx.warn('Workspace is in maitenance, skipping for now.', { workspace })
return
}
if (workspaceInfo?.disabled === true) {
tests/.env
+2 -1
View File
@@ -1,2 +1,3 @@
STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
SERVER_PROVIDER="ws"
BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
BACKUP_BUCKET_NAME=dev-backups
tests/docker-compose.yaml
-1
View File
@@ -126,7 +126,6 @@ services:
volumes:
- ./branding-test.json:/var/cfg/branding.json
environment:
- SERVER_PROVIDER=${SERVER_PROVIDER}
- SERVER_PORT=3334
- SERVER_SECRET=secret
- DB_URL=mongodb://mongodb:27018
tests/shutdown.sh
Executable
+2
View File
@@ -0,0 +1,2 @@
docker compose -p sanity kill
docker compose -p sanity down --volumes