WEBDIENSTE/huly-platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2026-09-09 19:27:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

UBERF-8122: Fix backup service

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev
2024-09-17 02:25:31 +07:00
parent c3a41ea1bb
commit b06f433bab
20 changed files with 287 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/tool/src/clean.ts
+17 -12
View File
@@ -104,12 +104,15 @@ export async function cleanWorkspace (
const minioList = await storageAdapter.listStream(ctx, workspaceId)
const toClean: string[] = []
while (true) {
const mv = await minioList.next()
if (mv === undefined) {
const mvFiles = await minioList.next()
if (mvFiles.length === 0) {
break
}
if (!files.has(mv._id)) {
toClean.push(mv._id)
for (const mv of mvFiles) {
if (!files.has(mv._id)) {
toClean.push(mv._id)
}
}
}
await storageAdapter.remove(ctx, workspaceId, toClean)
@@ -192,16 +195,18 @@ export async function fixMinioBW (
const list = await storageService.listStream(ctx, workspaceId)
let removed = 0
while (true) {
const obj = await list.next()
if (obj === undefined) {
const objs = await list.next()
if (objs.length === 0) {
break
}
if (obj.modifiedOn < from) continue
if ((obj._id as string).includes('%preview%')) {
await storageService.remove(ctx, workspaceId, [obj._id])
removed++
if (removed % 100 === 0) {
console.log('removed: ', removed)
for (const obj of objs) {
if (obj.modifiedOn < from) continue
if ((obj._id as string).includes('%preview%')) {
await storageService.remove(ctx, workspaceId, [obj._id])
removed++
if (removed % 100 === 0) {
console.log('removed: ', removed)
}
}
}
}
dev/tool/src/storage.ts
+65 -60
View File
@@ -40,20 +40,22 @@ export async function syncFiles (
const iterator = await adapter.listStream(ctx, workspaceId)
try {
while (true) {
const data = await iterator.next()
if (data === undefined) break
const dataBulk = await iterator.next()
if (dataBulk.length === 0) break
const blob = await exAdapter.stat(ctx, workspaceId, data._id)
if (blob !== undefined) continue
for (const data of dataBulk) {
const blob = await exAdapter.stat(ctx, workspaceId, data._id)
if (blob !== undefined) continue
await exAdapter.syncBlobFromStorage(ctx, workspaceId, data._id, name)
await exAdapter.syncBlobFromStorage(ctx, workspaceId, data._id, name)
count += 1
if (count % 100 === 0) {
const duration = Date.now() - time
time = Date.now()
count += 1
if (count % 100 === 0) {
const duration = Date.now() - time
time = Date.now()
console.log('...processed', count, Math.round(duration / 1000) + 's')
console.log('...processed', count, Math.round(duration / 1000) + 's')
}
}
}
console.log('processed', count)
@@ -112,64 +114,67 @@ async function processAdapter (
const iterator = await source.listStream(ctx, workspaceId)
try {
while (true) {
const data = await iterator.next()
if (data === undefined) break
const dataBulk = await iterator.next()
if (dataBulk.length === 0) break
const blob = (await exAdapter.stat(ctx, workspaceId, data._id)) ?? (await source.stat(ctx, workspaceId, data._id))
for (const data of dataBulk) {
const blob =
(await exAdapter.stat(ctx, workspaceId, data._id)) ?? (await source.stat(ctx, workspaceId, data._id))
if (blob === undefined) {
console.error('blob not found', data._id)
continue
}
if (blob.provider !== exAdapter.defaultAdapter) {
if (blob.size <= params.blobSizeLimitMb * 1024 * 1024) {
await rateLimiter.exec(async () => {
try {
await retryOnFailure(
ctx,
5,
async () => {
await processFile(ctx, source, params.move ? exAdapter : target, workspaceId, blob)
},
50
)
movedCnt += 1
movedBytes += blob.size
batchBytes += blob.size
} catch (err) {
console.error('failed to process blob', data._id, err)
}
})
} else {
skippedCnt += 1
console.log('skipping large blob', data._id, Math.round(blob.size / 1024 / 1024))
if (blob === undefined) {
console.error('blob not found', data._id)
continue
}
}
processedCnt += 1
processedBytes += blob.size
if (blob.provider !== exAdapter.defaultAdapter) {
if (blob.size <= params.blobSizeLimitMb * 1024 * 1024) {
await rateLimiter.exec(async () => {
try {
await retryOnFailure(
ctx,
5,
async () => {
await processFile(ctx, source, params.move ? exAdapter : target, workspaceId, blob)
},
50
)
movedCnt += 1
movedBytes += blob.size
batchBytes += blob.size
} catch (err) {
console.error('failed to process blob', data._id, err)
}
})
} else {
skippedCnt += 1
console.log('skipping large blob', data._id, Math.round(blob.size / 1024 / 1024))
}
}
if (processedCnt % 100 === 0) {
await rateLimiter.waitProcessing()
processedCnt += 1
processedBytes += blob.size
const duration = Date.now() - time
if (processedCnt % 100 === 0) {
await rateLimiter.waitProcessing()
console.log(
'...processed',
processedCnt,
Math.round(processedBytes / 1024 / 1024) + 'MB',
'moved',
movedCnt,
Math.round(movedBytes / 1024 / 1024) + 'MB',
'+' + Math.round(batchBytes / 1024 / 1024) + 'MB',
'skipped',
skippedCnt,
Math.round(duration / 1000) + 's'
)
const duration = Date.now() - time
batchBytes = 0
time = Date.now()
console.log(
'...processed',
processedCnt,
Math.round(processedBytes / 1024 / 1024) + 'MB',
'moved',
movedCnt,
Math.round(movedBytes / 1024 / 1024) + 'MB',
'+' + Math.round(batchBytes / 1024 / 1024) + 'MB',
'skipped',
skippedCnt,
Math.round(duration / 1000) + 's'
)
batchBytes = 0
time = Date.now()
}
}
}