WEBDIENSTE/huly-platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2026-09-11 04:07:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

UBERF-9429: Provide workspace ids to storage adapters (#7956)

Browse Source 
Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com>
This commit is contained in:
Alexey Zinoviev
2025-02-14 15:59:36 +07:00
committed by GitHub
parent ddc8230890
commit e8df531654
53 changed files with 740 additions and 688 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/tool/src/clean.ts
+5 -4
View File
@@ -54,6 +54,7 @@ import core, {
type TxUpdateDoc,
type WorkspaceUuid,
type WorkspaceDataId,
type WorkspaceIds,
generateId,
getObjectValue,
toIdMap,
@@ -172,12 +173,12 @@ export async function cleanWorkspace (
export async function fixMinioBW (
ctx: MeasureContext,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
storageService: StorageAdapter
): Promise<void> {
console.log('try clean bw miniature for ', workspaceId)
console.log('try clean bw miniature for ', wsIds)
const from = new Date(new Date().setDate(new Date().getDate() - 7)).getTime()
const list = await storageService.listStream(ctx, workspaceId)
const list = await storageService.listStream(ctx, wsIds)
let removed = 0
while (true) {
const objs = await list.next()
@@ -187,7 +188,7 @@ export async function fixMinioBW (
for (const obj of objs) {
if (obj.modifiedOn < from) continue
if ((obj._id as string).includes('%preview%')) {
await storageService.remove(ctx, workspaceId, [obj._id])
await storageService.remove(ctx, wsIds, [obj._id])
removed++
if (removed % 100 === 0) {
console.log('removed: ', removed)
dev/tool/src/index.ts
+3 -6
View File
@@ -47,6 +47,7 @@ import {
MeasureMetricsContext,
metricsToString,
type PersonId,
type WorkspaceUuid,
type Data,
type Tx,
type Version,
@@ -1136,13 +1137,9 @@ export function devTool (
.action(async (bucketName: string, dirName: string, storeIn: string, cmd) => {
const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE)
const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0])
const backupIds = { uuid: bucketName as WorkspaceUuid, dataId: bucketName as WorkspaceDataId, url: '' }
try {
const storage = await createStorageBackupStorage(
toolCtx,
storageAdapter,
bucketName as WorkspaceDataId,
dirName
)
const storage = await createStorageBackupStorage(toolCtx, storageAdapter, backupIds, dirName)
await backupDownload(storage, storeIn)
} catch (err: any) {
toolCtx.error('failed to size backup', { err })
dev/tool/src/markup.ts
+17 -26
View File
@@ -28,9 +28,9 @@ import core, {
type Ref,
type TxCreateDoc,
type TxUpdateDoc,
type WorkspaceDataId,
DOMAIN_TX,
SortingOrder,
type WorkspaceIds,
makeCollabId,
makeCollabYdocId,
makeDocCollabId
@@ -49,7 +49,7 @@ export interface RestoreWikiContentParams {
export async function restoreWikiContentMongo (
ctx: MeasureContext,
db: Db,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
storageAdapter: StorageAdapter,
params: RestoreWikiContentParams
): Promise<void> {
@@ -74,17 +74,17 @@ export async function restoreWikiContentMongo (
const correctCollabId = { objectClass: doc._class, objectId: doc._id, objectAttr: 'content' }
const wrongYdocId = await findWikiDocYdocName(ctx, db, workspaceId, doc._id)
const wrongYdocId = await findWikiDocYdocName(ctx, db, doc._id)
if (wrongYdocId === undefined) {
console.log('current ydoc not found', doc._id)
continue
}
const stat = storageAdapter.stat(ctx, workspaceId, wrongYdocId)
const stat = storageAdapter.stat(ctx, wsIds, wrongYdocId)
if (stat === undefined) continue
const ydoc1 = await loadCollabYdoc(ctx, storageAdapter, workspaceId, correctCollabId)
const ydoc2 = await loadCollabYdoc(ctx, storageAdapter, workspaceId, wrongYdocId)
const ydoc1 = await loadCollabYdoc(ctx, storageAdapter, wsIds, correctCollabId)
const ydoc2 = await loadCollabYdoc(ctx, storageAdapter, wsIds, wrongYdocId)
if (ydoc1 !== undefined && ydoc1.share.has('content')) {
// There already is content, we should skip the document
@@ -103,7 +103,7 @@ export async function restoreWikiContentMongo (
yDocCopyXmlField(ydoc2, 'description', 'content')
}
await saveCollabYdoc(ctx, storageAdapter, workspaceId, correctCollabId, ydoc2)
await saveCollabYdoc(ctx, storageAdapter, wsIds, correctCollabId, ydoc2)
}
restoredCnt++
} catch (err: any) {
@@ -119,7 +119,6 @@ export async function restoreWikiContentMongo (
export async function findWikiDocYdocName (
ctx: MeasureContext,
db: Db,
workspaceId: WorkspaceDataId,
doc: Ref<Document>
): Promise<Ref<Blob> | undefined> {
const updateContentTx = await db.collection<TxUpdateDoc<Document & { content: string }>>(DOMAIN_TX).findOne(
@@ -198,7 +197,7 @@ export interface RestoreControlledDocContentParams {
export async function restoreControlledDocContentMongo (
ctx: MeasureContext,
db: Db,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
storageAdapter: StorageAdapter,
params: RestoreWikiContentParams
): Promise<void> {
@@ -220,15 +219,7 @@ export async function restoreControlledDocContentMongo (
const doc = await iterator.next()
if (doc === null) break
const restored = await restoreControlledDocContentForDoc(
ctx,
db,
workspaceId,
storageAdapter,
params,
doc,
'content'
)
const restored = await restoreControlledDocContentForDoc(ctx, db, wsIds, storageAdapter, params, doc, 'content')
if (restored) {
restoredCnt++
}
@@ -247,7 +238,7 @@ export async function restoreControlledDocContentMongo (
export async function restoreControlledDocContentForDoc (
ctx: MeasureContext,
db: Db,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
storageAdapter: StorageAdapter,
params: RestoreWikiContentParams,
doc: Doc,
@@ -272,7 +263,7 @@ export async function restoreControlledDocContentForDoc (
const ydocId = makeCollabYdocId(makeDocCollabId(doc, attribute))
// Ensure that we don't have new content in storage
const stat = await storageAdapter.stat(ctx, workspaceId, ydocId)
const stat = await storageAdapter.stat(ctx, wsIds, ydocId)
if (stat !== undefined) {
console.log('content already restored', doc._class, doc._id, ydocId)
return false
@@ -281,15 +272,15 @@ export async function restoreControlledDocContentForDoc (
console.log('restoring content', doc._id, currentYdocId, '-->', ydocId)
if (!params.dryRun) {
try {
const stat = await storageAdapter.stat(ctx, workspaceId, currentYdocId)
const stat = await storageAdapter.stat(ctx, wsIds, currentYdocId)
if (stat === undefined) {
console.log('no content to restore', doc._class, doc._id, ydocId)
return false
}
const data = await storageAdapter.read(ctx, workspaceId, currentYdocId)
const data = await storageAdapter.read(ctx, wsIds, currentYdocId)
const buffer = Buffer.concat(data as any)
await storageAdapter.put(ctx, workspaceId, ydocId, buffer, 'application/ydoc', buffer.length)
await storageAdapter.put(ctx, wsIds, ydocId, buffer, 'application/ydoc', buffer.length)
} catch (err: any) {
console.error('failed to restore content for', doc._class, doc._id, err)
return false
@@ -302,7 +293,7 @@ export async function restoreControlledDocContentForDoc (
export async function restoreMarkupRefsMongo (
ctx: MeasureContext,
db: Db,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
hierarchy: Hierarchy,
storageAdapter: StorageAdapter
): Promise<void> {
@@ -347,10 +338,10 @@ export async function restoreMarkupRefsMongo (
const ydocId = makeCollabYdocId(collabId)
try {
const buffer = await storageAdapter.read(ctx, workspaceId, ydocId)
const buffer = await storageAdapter.read(ctx, wsIds, ydocId)
const ydoc = yDocFromBuffer(Buffer.concat(buffer as any))
const jsonId = await saveCollabJson(ctx, storageAdapter, workspaceId, collabId, ydoc)
const jsonId = await saveCollabJson(ctx, storageAdapter, wsIds, collabId, ydoc)
await collection.updateOne({ _id: doc._id }, { $set: { [attributeName]: jsonId } })
} catch {}
}
dev/tool/src/storage.ts
+29 -28
View File
@@ -20,7 +20,8 @@ import {
type Ref,
concatLink,
RateLimiter,
type WorkspaceDataId
type WorkspaceIds,
type WorkspaceUuid
} from '@hcengineering/core'
import { type DatalakeClient } from '@hcengineering/datalake'
import { type UploadObjectParams } from '@hcengineering/datalake/types/client'
@@ -42,7 +43,7 @@ export interface MoveFilesParams {
export async function moveFiles (
ctx: MeasureContext,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
exAdapter: StorageAdapterEx,
params: MoveFilesParams
): Promise<void> {
@@ -52,7 +53,7 @@ export async function moveFiles (
if (target === undefined) return
// We assume that the adapter moves all new files to the default adapter
await target.make(ctx, workspaceId)
await target.make(ctx, wsIds)
for (const { name, adapter } of exAdapter.adapters.slice(1).reverse()) {
console.log('moving from', name, 'limit', 'concurrency', params.concurrency)
@@ -60,14 +61,14 @@ export async function moveFiles (
// we attempt retry the whole process in case of failure
// files that were already moved will be skipped
await retryOnFailure(ctx, 5, async () => {
await processAdapter(ctx, exAdapter, adapter, target, workspaceId, params)
await processAdapter(ctx, exAdapter, adapter, target, wsIds, params)
})
}
}
export async function showLostFiles (
ctx: MeasureContext,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
db: Db,
storageAdapter: StorageAdapter,
{ showAll }: { showAll: boolean }
@@ -81,7 +82,7 @@ export async function showLostFiles (
const { _id, _class, file, name, modifiedOn } = attachment
const date = new Date(modifiedOn).toISOString()
const stat = await storageAdapter.stat(ctx, workspaceId, file)
const stat = await storageAdapter.stat(ctx, wsIds, file)
if (stat === undefined) {
console.warn('-', date, _class, _id, file, name)
} else if (showAll) {
@@ -95,7 +96,7 @@ async function processAdapter (
exAdapter: StorageAdapterEx,
source: StorageAdapter,
target: StorageAdapter,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
params: MoveFilesParams
): Promise<void> {
if (source === target) {
@@ -128,9 +129,9 @@ async function processAdapter (
const rateLimiter = new RateLimiter(params.concurrency)
const iterator = await source.listStream(ctx, workspaceId)
const iterator = await source.listStream(ctx, wsIds)
const targetIterator = await target.listStream(ctx, workspaceId)
const targetIterator = await target.listStream(ctx, wsIds)
const targetBlobs = new Map<Ref<Blob>, ListBlobResult>()
@@ -165,7 +166,7 @@ async function processAdapter (
}
if (targetBlob === undefined) {
const sourceBlob = await source.stat(ctx, workspaceId, data._id)
const sourceBlob = await source.stat(ctx, wsIds, data._id)
if (sourceBlob === undefined) {
console.error('blob not found', data._id)
@@ -177,7 +178,7 @@ async function processAdapter (
ctx,
5,
async () => {
return await processFile(ctx, source, target, workspaceId, sourceBlob)
return await processFile(ctx, source, target, wsIds, sourceBlob)
},
50
)
@@ -209,7 +210,7 @@ async function processAdapter (
if (toRemove.length > 0 && params.move) {
while (toRemove.length > 0) {
const part = toRemove.splice(0, 500)
await source.remove(ctx, workspaceId, part)
await source.remove(ctx, wsIds, part)
}
}
printStats()
@@ -222,16 +223,16 @@ async function processFile (
ctx: MeasureContext,
source: Pick<StorageAdapter, 'get'>,
target: Pick<StorageAdapter, 'put'>,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
blob: Blob
): Promise<UploadedObjectInfo> {
const readable = await source.get(ctx, workspaceId, blob._id)
const readable = await source.get(ctx, wsIds, blob._id)
try {
readable.on('end', () => {
readable.destroy()
})
const stream = readable.pipe(new PassThrough())
return await target.put(ctx, workspaceId, blob._id, stream, blob.contentType, blob.size)
return await target.put(ctx, wsIds, blob._id, stream, blob.contentType, blob.size)
} finally {
readable.destroy()
}
@@ -267,7 +268,7 @@ export interface CopyDatalakeParams {
export async function copyToDatalake (
ctx: MeasureContext,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
config: S3Config,
adapter: S3Service,
datalake: DatalakeClient,
@@ -275,7 +276,7 @@ export async function copyToDatalake (
): Promise<void> {
console.log('copying from', config.name, 'concurrency:', params.concurrency)
const exists = await adapter.exists(ctx, workspaceId)
const exists = await adapter.exists(ctx, wsIds)
if (!exists) {
console.log('no files to copy')
return
@@ -311,7 +312,7 @@ export async function copyToDatalake (
let cursor: string | undefined = ''
let hasMore = true
while (hasMore) {
const res = await datalake.listObjects(ctx, workspaceId, cursor, 1000)
const res = await datalake.listObjects(ctx, wsIds.uuid, cursor, 1000)
cursor = res.cursor
hasMore = res.cursor !== undefined
for (const blob of res.blobs) {
@@ -323,7 +324,7 @@ export async function copyToDatalake (
const rateLimiter = new RateLimiter(params.concurrency)
const iterator = await adapter.listStream(ctx, workspaceId)
const iterator = await adapter.listStream(ctx, wsIds)
try {
while (true) {
@@ -349,7 +350,7 @@ export async function copyToDatalake (
ctx,
5,
async () => {
await copyBlobToDatalake(ctx, workspaceId, blob, config, adapter, datalake)
await copyBlobToDatalake(ctx, wsIds, blob, config, adapter, datalake)
processedCnt += 1
processedSize += blob.size
},
@@ -374,7 +375,7 @@ export async function copyToDatalake (
export async function copyBlobToDatalake (
ctx: MeasureContext,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
blob: ListBlobResult,
config: S3Config,
adapter: S3Service,
@@ -385,15 +386,15 @@ export async function copyBlobToDatalake (
// Handle small file
const { endpoint, accessKey: accessKeyId, secretKey: secretAccessKey, region } = config
const bucketId = adapter.getBucketId(workspaceId)
const objectId = adapter.getDocumentKey(workspaceId, encodeURIComponent(objectName))
const bucketId = adapter.getBucketId(wsIds)
const objectId = adapter.getDocumentKey(wsIds, encodeURIComponent(objectName))
const url = concatLink(endpoint, `${bucketId}/${objectId}`)
const params = { url, accessKeyId, secretAccessKey, region }
await datalake.uploadFromS3(ctx, workspaceId, objectName, params)
await datalake.uploadFromS3(ctx, wsIds.uuid, objectName, params)
} else {
// Handle huge file
const stat = await adapter.stat(ctx, workspaceId, objectName)
const stat = await adapter.stat(ctx, wsIds, objectName)
if (stat !== undefined) {
const metadata = {
lastModified: stat.modifiedOn,
@@ -402,10 +403,10 @@ export async function copyBlobToDatalake (
size: stat.size
}
const readable = await adapter.get(ctx, workspaceId, objectName)
const readable = await adapter.get(ctx, wsIds, objectName)
try {
console.log('uploading huge blob', objectName, Math.round(stat.size / 1024 / 1024), 'MB')
await uploadMultipart(ctx, datalake, workspaceId, objectName, readable, metadata)
await uploadMultipart(ctx, datalake, wsIds.uuid, objectName, readable, metadata)
console.log('done', objectName)
} finally {
readable.destroy()
@@ -417,7 +418,7 @@ export async function copyBlobToDatalake (
function uploadMultipart (
ctx: MeasureContext,
datalake: DatalakeClient,
workspaceId: WorkspaceDataId,
workspaceId: WorkspaceUuid,
objectName: string,
stream: Readable,
metadata: UploadObjectParams
dev/tool/src/telegram.ts
+6 -5
View File
@@ -14,7 +14,7 @@
// limitations under the License.
//
import { DOMAIN_TX, type WorkspaceDataId, type MeasureContext, type Ref } from '@hcengineering/core'
import { DOMAIN_TX, type WorkspaceDataId, type MeasureContext, type Ref, type WorkspaceIds } from '@hcengineering/core'
import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment'
import contact, { DOMAIN_CHANNEL } from '@hcengineering/model-contact'
import { DOMAIN_TELEGRAM } from '@hcengineering/model-telegram'
@@ -31,14 +31,15 @@ const LastMessages = 'last-msgs'
export async function clearTelegramHistory (
ctx: MeasureContext,
mongoUrl: string,
workspaceId: WorkspaceDataId,
wsIds: WorkspaceIds,
tgDb: string,
storageAdapter: StorageAdapter
): Promise<void> {
const client = getMongoClient(mongoUrl)
try {
const _client = await client.getClient()
const workspaceDB = getWorkspaceMongoDB(_client, workspaceId)
const dataId = wsIds.dataId ?? (wsIds.uuid as unknown as WorkspaceDataId)
const workspaceDB = getWorkspaceMongoDB(_client, dataId)
const telegramDB = _client.db(tgDb)
const sharedMessages = await workspaceDB
@@ -91,12 +92,12 @@ export async function clearTelegramHistory (
workspaceDB.collection(DOMAIN_ATTACHMENT).deleteMany({
attachedToClass: telegram.class.Message
}),
storageAdapter.remove(ctx, workspaceId, Array.from(attachments))
storageAdapter.remove(ctx, wsIds, Array.from(attachments))
])
console.log('clearing telegram service data...')
await telegramDB.collection(LastMessages).deleteMany({
workspace: workspaceId
workspace: wsIds.uuid
})
} finally {
client.close()