WEBDIENSTE/huly-platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2026-09-10 03:37:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update sanity DB restore (#2836)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev
2023-06-09 16:36:55 +06:00
committed by GitHub
parent 55fee1f8c4
commit c8c06cb0dc
77 changed files with 162 additions and 6107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/tool/src/__start.ts
+2
View File
@@ -70,4 +70,6 @@ function prepareTools (): {
return { ...prepareToolsRaw(builder(enabled, disabled).getTxes()), version, migrateOperations }
}
console.log(`tools git_version: ${process.env.GIT_REVISION ?? ''} model_version: ${process.env.MODEL_VERSION ?? ''}`)
devTool(prepareTools, '')
dev/tool/src/index.ts
+1 -46
View File
@@ -45,7 +45,7 @@ import toolPlugin from '@hcengineering/server-tool'
import { program } from 'commander'
import { Db, MongoClient } from 'mongodb'
import { clearTelegramHistory } from './telegram'
import { diffWorkspace, dumpWorkspace, restoreWorkspace } from './workspace'
import { diffWorkspace } from './workspace'
import { Data, getWorkspaceId, Tx, Version } from '@hcengineering/core'
import { MinioService } from '@hcengineering/minio'
@@ -54,7 +54,6 @@ import { openAIConfigDefaults } from '@hcengineering/openai'
import { benchmark } from './benchmark'
import { cleanArchivedSpaces, cleanRemovedTransactions, cleanWorkspace } from './clean'
import { changeConfiguration } from './configuration'
import { rebuildElastic } from './elastic'
import { openAIConfig } from './openai'
/**
@@ -293,14 +292,6 @@ export function devTool (
})
})
program
.command('dump-workspace <workspace> <dirName>')
.description('dump workspace transactions and minio resources')
.action(async (workspace: string, dirName: string, cmd) => {
const { mongodbUri, minio } = prepareTools()
return await dumpWorkspace(mongodbUri, getWorkspaceId(workspace, productId), dirName, minio)
})
program
.command('backup <dirName> <workspace>')
.description('dump workspace transactions and minio resources')
@@ -360,23 +351,6 @@ export function devTool (
return await backupList(storage)
})
program
.command('restore-workspace <workspace> <dirName>')
.description('restore workspace transactions and minio resources from previous dump.')
.action(async (workspace: string, dirName: string, cmd) => {
const { mongodbUri, minio, txes, migrateOperations } = prepareTools()
return await restoreWorkspace(
mongodbUri,
getWorkspaceId(workspace, productId),
dirName,
minio,
getElasticUrl(),
transactorUrl,
txes,
migrateOperations
)
})
program
.command('confirm-email <email>')
.description('confirm user email')
@@ -434,25 +408,6 @@ export function devTool (
})
})
program
.command('rebuild-elastic [workspace]')
.description('rebuild elastic index')
.action(async (workspace: string, cmd) => {
const { mongodbUri, minio } = prepareTools()
return await withDatabase(mongodbUri, async (db) => {
if (workspace === undefined) {
const workspaces = await listWorkspaces(db, productId)
for (const w of workspaces) {
await rebuildElastic(mongodbUri, getWorkspaceId(w.workspace, productId), minio, getElasticUrl())
}
} else {
await rebuildElastic(mongodbUri, getWorkspaceId(workspace, productId), minio, getElasticUrl())
console.log('rebuild end')
}
})
})
program
.command('generate-token <name> <workspace> <productId>')
.description('generate token')
dev/tool/src/workspace.ts
+1 -146
View File
@@ -16,155 +16,10 @@
import contact from '@hcengineering/contact'
import core, { DOMAIN_TX, Tx, WorkspaceId } from '@hcengineering/core'
import { MinioService, MinioWorkspaceItem } from '@hcengineering/minio'
import { MigrateOperation } from '@hcengineering/model'
import { getWorkspaceDB } from '@hcengineering/mongo'
import { upgradeModel } from '@hcengineering/server-tool'
import { existsSync } from 'fs'
import { mkdir, open, readFile, writeFile } from 'fs/promises'
import { Document, MongoClient } from 'mongodb'
import { join } from 'path'
import { rebuildElastic } from './elastic'
import { MongoClient } from 'mongodb'
import { generateModelDiff, printDiff } from './mdiff'
interface CollectionInfo {
name: string
file: string
}
interface WorkspaceInfo {
version: string
collections: CollectionInfo[]
minioData: MinioWorkspaceItem[]
}
/**
* @public
*/
export async function dumpWorkspace (
mongoUrl: string,
workspaceId: WorkspaceId,
fileName: string,
minio: MinioService
): Promise<void> {
const client = new MongoClient(mongoUrl)
try {
await client.connect()
const db = getWorkspaceDB(client, workspaceId)
console.log('dumping transactions...')
if (!existsSync(fileName)) {
await mkdir(fileName, { recursive: true })
}
const workspaceInfo: WorkspaceInfo = {
version: '0.0.0',
collections: [],
minioData: []
}
const collections = await db.collections()
for (const c of collections) {
const docs = await c.find().toArray()
workspaceInfo.collections.push({ name: c.collectionName, file: c.collectionName + '.json' })
await writeFile(fileName + c.collectionName + '.json', JSON.stringify(docs, undefined, 2))
}
console.log('Dump minio objects')
if (await minio.exists(workspaceId)) {
workspaceInfo.minioData.push(...(await minio.list(workspaceId)))
const minioDbLocation = fileName + '.minio'
if (!existsSync(minioDbLocation)) {
await mkdir(minioDbLocation)
}
for (const d of workspaceInfo.minioData) {
const stat = await minio.stat(workspaceId, d.name)
d.metaData = stat.metaData
const fileHandle = await open(join(minioDbLocation, d.name), 'w')
const chunks: Buffer[] = await minio.read(workspaceId, d.name)
for (const b of chunks) {
await fileHandle.write(b)
}
await fileHandle.close()
}
}
await writeFile(fileName + '.workspace.json', JSON.stringify(workspaceInfo, undefined, 2))
} finally {
await client.close()
}
}
export async function restoreWorkspace (
mongoUrl: string,
workspaceId: WorkspaceId,
fileName: string,
minio: MinioService,
elasticUrl: string,
transactorUrl: string,
rawTxes: Tx[],
migrateOperations: [string, MigrateOperation][]
): Promise<void> {
console.log('Restoring workspace', mongoUrl, workspaceId, fileName)
const client = new MongoClient(mongoUrl)
try {
await client.connect()
const db = getWorkspaceDB(client, workspaceId)
const workspaceInfo = JSON.parse((await readFile(fileName + '.workspace.json')).toString()) as WorkspaceInfo
// Drop existing collections
const cols = await db.collections()
for (const c of cols) {
console.log('dropping existing table', c.collectionName)
await db.dropCollection(c.collectionName)
}
// Restore collections.
for (const c of workspaceInfo.collections) {
const collection = db.collection(c.name)
await collection.deleteMany({})
const data = JSON.parse((await readFile(fileName + c.name + '.json')).toString()) as Document[]
if (data.length > 0) {
console.log('restore existing collection', c.name, data.length)
await collection.insertMany(data)
}
}
if (await minio.exists(workspaceId)) {
const objectNames = (await minio.list(workspaceId)).map((i) => i.name)
await minio.remove(workspaceId, objectNames)
await minio.delete(workspaceId)
}
await minio.make(workspaceId)
const minioDbLocation = fileName + '.minio'
console.log('Restore minio objects', workspaceInfo.minioData.length)
let promises: Promise<void>[] = []
for (const d of workspaceInfo.minioData) {
const file = await open(join(minioDbLocation, d.name), 'r')
const stream = file.createReadStream()
promises.push(
minio.put(workspaceId, d.name, stream, d.size, d.metaData).then(async () => {
await file.close()
})
)
if (promises.length > 10) {
await Promise.all(promises)
promises = []
}
}
await upgradeModel(transactorUrl, workspaceId, rawTxes, migrateOperations)
await rebuildElastic(mongoUrl, workspaceId, minio, elasticUrl)
} finally {
await client.close()
}
}
export async function diffWorkspace (mongoUrl: string, workspace: WorkspaceId, rawTxes: Tx[]): Promise<void> {
const client = new MongoClient(mongoUrl)
try {