Fix cockroach (#6965)

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>
This commit is contained in:
Denis Bykhov
2024-10-23 13:29:44 +05:00
committed by GitHub
parent 498b390b10
commit 5eeb828f9c
88 changed files with 447 additions and 314 deletions
+35 -14
View File
@@ -18,7 +18,14 @@ import {
type MeasureMetricsContext
} from '@hcengineering/core'
import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo'
import { convertDoc, createTable, getDBClient, retryTxn, translateDomain } from '@hcengineering/postgres'
import {
convertDoc,
createTable,
getDBClient,
getDocFieldsByDomains,
retryTxn,
translateDomain
} from '@hcengineering/postgres'
import { getTransactorEndpoint } from '@hcengineering/server-client'
import { generateToken } from '@hcengineering/server-token'
import { connect } from '@hcengineering/server-tool'
@@ -54,10 +61,6 @@ export async function moveFromMongoToPG (
client.close()
}
function escapeBackticks (str: string): string {
return str.replaceAll("'", "''")
}
async function moveWorkspace (
accountDb: AccountDB,
mongo: MongoClient,
@@ -85,6 +88,13 @@ async function moveWorkspace (
const currentIds = new Set(current.rows.map((r) => r._id))
console.log('move domain', domain)
const docs: Doc[] = []
const fields = getDocFieldsByDomains(domain)
const filedsWithData = [...fields, 'data']
const insertFields: string[] = []
for (const field of filedsWithData) {
insertFields.push(`"${field}"`)
}
const insertStr = insertFields.join(', ')
while (true) {
while (docs.length < 50000) {
const doc = (await cursor.next()) as Doc | null
@@ -95,18 +105,29 @@ async function moveWorkspace (
if (docs.length === 0) break
while (docs.length > 0) {
const part = docs.splice(0, 500)
const vals = part
.map((doc) => {
const d = convertDoc(doc, ws.workspace)
return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', ${
d.attachedTo != null ? `'${d.attachedTo}'` : 'NULL'
}, '${escapeBackticks(JSON.stringify(d.data))}')`
})
.join(', ')
const values: any[] = []
const vars: string[] = []
let index = 1
for (let i = 0; i < part.length; i++) {
const doc = part[i]
const variables: string[] = []
const d = convertDoc(domain, doc, ws.workspace)
values.push(d.workspaceId)
variables.push(`$${index++}`)
for (const field of fields) {
values.push(d[field])
variables.push(`$${index++}`)
}
values.push(d.data)
variables.push(`$${index++}`)
vars.push(`(${variables.join(', ')})`)
}
const vals = vars.join(',')
try {
await retryTxn(pgClient, async (client) => {
await client.query(
`INSERT INTO ${translateDomain(domain)} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ${vals}`
`INSERT INTO ${translateDomain(domain)} ("workspaceId", ${insertStr}) VALUES ${vals}`,
values
)
})
} catch (err) {
+2 -2
View File
@@ -371,7 +371,7 @@ export function devTool (
lastProcessingTime: Date.now() + 1000 * 60
})
await createWorkspace(measureCtx, version, brandingObj, wsInfo, txes, migrateOperations)
await createWorkspace(measureCtx, version, brandingObj, wsInfo, txes, migrateOperations, undefined, true)
await updateWorkspace(db, wsInfo, {
mode: 'active',
@@ -1717,7 +1717,7 @@ export function devTool (
lastProcessingTime: Date.now() + 1000 * 60
})
await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations)
await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations, undefined, true)
await updateWorkspace(db, wsInfo, {
mode: 'active',