UBERF-6374: Improve server logging and improve startup performance (#5210)

This commit is contained in:
Andrey Sobolev
2024-04-06 15:14:06 +07:00
committed by GitHub
parent ceac67f3a3
commit 034700a65b
39 changed files with 631 additions and 405 deletions
+7 -6
View File
@@ -25,6 +25,7 @@ import type { DocumentQuery, FindResult, TxResult, SearchQuery, SearchOptions, S
import { Tx, TxFactory, TxProcessor } from '../tx'
import { connect } from './connection'
import { genMinModel } from './minmodel'
import { clone } from '../clone'
describe('client', () => {
it('should create client and spaces', async () => {
@@ -118,7 +119,7 @@ describe('client', () => {
loadDocs: async (domain: Domain, docs: Ref<Doc>[]) => [],
upload: async (domain: Domain, docs: Doc[]) => {},
clean: async (domain: Domain, docs: Ref<Doc>[]) => {},
loadModel: async (last: Timestamp) => txes,
loadModel: async (last: Timestamp) => clone(txes),
getAccount: async () => null as unknown as Account,
measure: async () => {
return async () => ({ time: 0, serverTime: 0 })
@@ -141,8 +142,8 @@ describe('client', () => {
expect(result1).toHaveLength(1)
expect(result1[0]._id).toStrictEqual(txCreateDoc1.objectId)
expect(spyCreate).toHaveBeenLastCalledWith(txCreateDoc1)
expect(spyUpdate).toBeCalledTimes(0)
expect(spyCreate).toHaveBeenLastCalledWith(txCreateDoc1, false)
expect(spyUpdate).toHaveBeenCalledTimes(0)
await client1.close()
const pluginData2 = {
@@ -159,8 +160,8 @@ describe('client', () => {
expect(result2).toHaveLength(2)
expect(result2[0]._id).toStrictEqual(txCreateDoc1.objectId)
expect(result2[1]._id).toStrictEqual(txCreateDoc2.objectId)
expect(spyCreate).toHaveBeenLastCalledWith(txCreateDoc2)
expect(spyUpdate).toBeCalledTimes(0)
expect(spyCreate).toHaveBeenLastCalledWith(txCreateDoc2, false)
expect(spyUpdate).toHaveBeenCalledTimes(0)
await client2.close()
const pluginData3 = {
@@ -181,7 +182,7 @@ describe('client', () => {
expect(result3).toHaveLength(1)
expect(result3[0]._id).toStrictEqual(txCreateDoc2.objectId)
expect(spyCreate).toHaveBeenLastCalledWith(txCreateDoc2)
expect(spyCreate).toHaveBeenLastCalledWith(txCreateDoc2, false)
expect(spyUpdate.mock.calls[1][1]).toStrictEqual(txUpdateDoc)
expect(spyUpdate).toBeCalledTimes(2)
await client3.close()
+94 -47
View File
@@ -18,11 +18,12 @@ import { BackupClient, DocChunk } from './backup'
import { Account, AttachedDoc, Class, DOMAIN_MODEL, Doc, Domain, PluginConfiguration, Ref, Timestamp } from './classes'
import core from './component'
import { Hierarchy } from './hierarchy'
import { MeasureContext, MeasureMetricsContext } from './measurements'
import { ModelDb } from './memdb'
import type { DocumentQuery, FindOptions, FindResult, FulltextStorage, Storage, TxResult, WithLookup } from './storage'
import { SearchOptions, SearchQuery, SearchResult, SortingOrder } from './storage'
import { Tx, TxCUD, TxCollectionCUD, TxCreateDoc, TxProcessor, TxUpdateDoc } from './tx'
import { toFindResult } from './utils'
import { toFindResult, toIdMap } from './utils'
const transactionThreshold = 500
@@ -222,8 +223,10 @@ export async function createClient (
connect: (txHandler: TxHandler) => Promise<ClientConnection>,
// If set will build model with only allowed plugins.
allowedPlugins?: Plugin[],
txPersistence?: TxPersistenceStore
txPersistence?: TxPersistenceStore,
_ctx?: MeasureContext
): Promise<AccountClient> {
const ctx = _ctx ?? new MeasureMetricsContext('createClient', {})
let client: ClientImpl | null = null
// Temporal buffer, while we apply model
@@ -248,9 +251,13 @@ export async function createClient (
}
const configs = new Map<Ref<PluginConfiguration>, PluginConfiguration>()
const conn = await connect(txHandler)
const conn = await ctx.with('connect', {}, async () => await connect(txHandler))
await loadModel(conn, allowedPlugins, configs, hierarchy, model, false, txPersistence)
await ctx.with(
'load-model',
{ reload: false },
async (ctx) => await loadModel(ctx, conn, allowedPlugins, configs, hierarchy, model, false, txPersistence)
)
txBuffer = txBuffer.filter((tx) => tx.space !== core.space.Model)
@@ -264,14 +271,20 @@ export async function createClient (
conn.onConnect = async (event) => {
console.log('Client: onConnect', event)
// Find all new transactions and apply
const loadModelResponse = await loadModel(conn, allowedPlugins, configs, hierarchy, model, true, txPersistence)
const loadModelResponse = await ctx.with(
'connect',
{ reload: true },
async (ctx) => await loadModel(ctx, conn, allowedPlugins, configs, hierarchy, model, true, txPersistence)
)
if (event === ClientConnectEvent.Reconnected && loadModelResponse.full) {
// We have upgrade procedure and need rebuild all stuff.
hierarchy = new Hierarchy()
model = new ModelDb(hierarchy)
await buildModel(loadModelResponse, allowedPlugins, configs, hierarchy, model)
await ctx.with('build-model', {}, async (ctx) => {
await buildModel(ctx, loadModelResponse, allowedPlugins, configs, hierarchy, model)
})
await oldOnConnect?.(ClientConnectEvent.Upgraded)
// No need to fetch more stuff since upgrade was happened.
@@ -285,10 +298,15 @@ export async function createClient (
}
// We need to look for last {transactionThreshold} transactions and if it is more since lastTx one we receive, we need to perform full refresh.
const atxes = await conn.findAll(
core.class.Tx,
{ modifiedOn: { $gt: lastTx }, objectSpace: { $ne: core.space.Model } },
{ sort: { modifiedOn: SortingOrder.Ascending, _id: SortingOrder.Ascending }, limit: transactionThreshold }
const atxes = await ctx.with(
'find-atx',
{},
async () =>
await conn.findAll(
core.class.Tx,
{ modifiedOn: { $gt: lastTx }, objectSpace: { $ne: core.space.Model } },
{ sort: { modifiedOn: SortingOrder.Ascending, _id: SortingOrder.Ascending }, limit: transactionThreshold }
)
)
let needFullRefresh = false
@@ -318,14 +336,23 @@ export async function createClient (
}
async function tryLoadModel (
ctx: MeasureContext,
conn: ClientConnection,
reload: boolean,
persistence?: TxPersistenceStore
): Promise<LoadModelResponse> {
const current = (await persistence?.load()) ?? { full: true, transactions: [], hash: '' }
const current = (await ctx.with('persistence-load', {}, async () => await persistence?.load())) ?? {
full: true,
transactions: [],
hash: ''
}
const lastTxTime = getLastTxTime(current.transactions)
const result = await conn.loadModel(lastTxTime, current.hash)
const result = await ctx.with(
'connection-load-model',
{ hash: current.hash !== '' },
async (ctx) => await conn.loadModel(lastTxTime, current.hash)
)
if (Array.isArray(result)) {
// Fallback to old behavior, only for tests
@@ -337,10 +364,15 @@ async function tryLoadModel (
}
// Save concatenated
await persistence?.store({
...result,
transactions: !result.full ? current.transactions.concat(result.transactions) : result.transactions
})
void (await ctx.with(
'persistence-store',
{},
async (ctx) =>
await persistence?.store({
...result,
transactions: !result.full ? current.transactions.concat(result.transactions) : result.transactions
})
))
if (!result.full && !reload) {
result.transactions = current.transactions.concat(result.transactions)
@@ -361,6 +393,7 @@ function isPersonAccount (tx: Tx): boolean {
}
async function loadModel (
ctx: MeasureContext,
conn: ClientConnection,
allowedPlugins: Plugin[] | undefined,
configs: Map<Ref<PluginConfiguration>, PluginConfiguration>,
@@ -371,7 +404,11 @@ async function loadModel (
): Promise<LoadModelResponse> {
const t = Date.now()
const modelResponse = await tryLoadModel(conn, reload, persistence)
const modelResponse = await ctx.with(
'try-load-model',
{ reload },
async (ctx) => await tryLoadModel(ctx, conn, reload, persistence)
)
if (reload && modelResponse.full) {
return modelResponse
@@ -385,11 +422,14 @@ async function loadModel (
)
}
await buildModel(modelResponse, allowedPlugins, configs, hierarchy, model)
await ctx.with('build-model', {}, async (ctx) => {
await buildModel(ctx, modelResponse, allowedPlugins, configs, hierarchy, model)
})
return modelResponse
}
async function buildModel (
ctx: MeasureContext,
modelResponse: LoadModelResponse,
allowedPlugins: Plugin[] | undefined,
configs: Map<Ref<PluginConfiguration>, PluginConfiguration>,
@@ -400,38 +440,45 @@ async function buildModel (
const userTx: Tx[] = []
const atxes = modelResponse.transactions
atxes.forEach((tx) =>
((tx.modifiedBy === core.account.ConfigUser || tx.modifiedBy === core.account.System) && !isPersonAccount(tx)
? systemTx
: userTx
).push(tx)
)
await ctx.with('split txes', {}, async () => {
atxes.forEach((tx) =>
((tx.modifiedBy === core.account.ConfigUser || tx.modifiedBy === core.account.System) && !isPersonAccount(tx)
? systemTx
: userTx
).push(tx)
)
})
if (allowedPlugins != null) {
fillConfiguration(systemTx, configs)
fillConfiguration(userTx, configs)
await ctx.with('fill config system', {}, async () => {
fillConfiguration(systemTx, configs)
})
await ctx.with('fill config user', {}, async () => {
fillConfiguration(userTx, configs)
})
const excludedPlugins = Array.from(configs.values()).filter(
(it) => !it.enabled || !allowedPlugins.includes(it.pluginId)
)
systemTx = pluginFilterTx(excludedPlugins, configs, systemTx)
await ctx.with('filter txes', {}, async () => {
systemTx = pluginFilterTx(excludedPlugins, configs, systemTx)
})
}
const txes = systemTx.concat(userTx)
for (const tx of txes) {
try {
hierarchy.tx(tx)
} catch (err: any) {
console.error('failed to apply model transaction, skipping', tx._id, tx._class, err?.message)
await ctx.with('build hierarchy', {}, async () => {
for (const tx of txes) {
try {
hierarchy.tx(tx)
} catch (err: any) {
console.error('failed to apply model transaction, skipping', tx._id, tx._class, err?.message)
}
}
}
for (const tx of txes) {
try {
await model.tx(tx)
} catch (err: any) {
console.error('failed to apply model transaction, skipping', tx._id, tx._class, err?.message)
}
}
})
await ctx.with('build model', {}, async (ctx) => {
model.addTxes(ctx, txes, false)
})
}
function getLastTxTime (txes: Tx[]): number {
@@ -468,14 +515,15 @@ function pluginFilterTx (
configs: Map<Ref<PluginConfiguration>, PluginConfiguration>,
systemTx: Tx[]
): Tx[] {
const stx = toIdMap(systemTx)
const totalExcluded = new Set<Ref<Tx>>()
for (const a of excludedPlugins) {
for (const c of configs.values()) {
if (a.pluginId === c.pluginId) {
const excluded = new Set<Ref<Tx>>()
for (const id of c.transactions) {
if (c.classFilter !== undefined) {
const filter = new Set(c.classFilter)
const tx = systemTx.find((it) => it._id === id)
const tx = stx.get(id as Ref<Tx>)
if (
tx?._class === core.class.TxCreateDoc ||
tx?._class === core.class.TxUpdateDoc ||
@@ -483,18 +531,17 @@ function pluginFilterTx (
) {
const cud = tx as TxCUD<Doc>
if (filter.has(cud.objectClass)) {
excluded.add(id as Ref<Tx>)
totalExcluded.add(id as Ref<Tx>)
}
}
} else {
excluded.add(id as Ref<Tx>)
totalExcluded.add(id as Ref<Tx>)
}
}
const exclude = systemTx.filter((t) => excluded.has(t._id))
console.log('exclude plugin', c.pluginId, exclude.length)
systemTx = systemTx.filter((t) => !excluded.has(t._id))
console.log('exclude plugin', c.pluginId, c.transactions.length)
}
}
}
systemTx = systemTx.filter((t) => !totalExcluded.has(t._id))
return systemTx
}
+72
View File
@@ -0,0 +1,72 @@
const se = typeof Symbol !== 'undefined'
const ste = se && typeof Symbol.toStringTag !== 'undefined'
export function getTypeOf (obj: any): string {
const typeofObj = typeof obj
if (typeofObj !== 'object') {
return typeofObj
}
if (obj === null) {
return 'null'
}
if (Array.isArray(obj) && (!ste || !(Symbol.toStringTag in obj))) {
return 'Array'
}
const stringTag = ste && obj[Symbol.toStringTag]
if (typeof stringTag === 'string') {
return stringTag
}
const objPrototype = Object.getPrototypeOf(obj)
if (objPrototype === RegExp.prototype) {
return 'RegExp'
}
if (objPrototype === Date.prototype) {
return 'Date'
}
if (objPrototype === null) {
return 'Object'
}
return {}.toString.call(obj).slice(8, -1)
}
export function clone (obj: any, as?: (doc: any, m: any) => any, needAs?: (value: any) => any | undefined): any {
if (typeof obj === 'undefined') {
return undefined
}
if (typeof obj === 'function') {
return obj
}
const typeOf = getTypeOf(obj)
if (typeOf === 'Date') {
return new Date(obj.getTime())
} else if (typeOf === 'Array' || typeOf === 'Object') {
const isArray = Array.isArray(obj)
const result: any = isArray ? [] : Object.assign({}, obj)
for (const key in obj) {
// include prototype properties
const value = obj[key]
const type = getTypeOf(value)
if (type === 'Array') {
result[key] = clone(value, as, needAs)
} else if (type === 'Object') {
const m = needAs?.(value)
const valClone = clone(value, as, needAs)
result[key] = m !== undefined && as !== undefined ? as(valClone, m) : valClone
} else if (type === 'Date') {
result[key] = new Date(value.getTime())
} else {
if (isArray) {
result[key] = value
}
}
}
return result
} else {
return obj
}
}
+6 -28
View File
@@ -16,11 +16,11 @@
import { FindOptions, Lookup, ToClassRefT, WithLookup } from '.'
import type { AnyAttribute, Class, Classifier, Doc, Domain, Interface, Mixin, Obj, Ref } from './classes'
import { ClassifierKind } from './classes'
import { clone as deepClone } from './clone'
import core from './component'
import { _createMixinProxy, _mixinClass, _toDoc } from './proxy'
import type { Tx, TxCreateDoc, TxMixin, TxRemoveDoc, TxUpdateDoc } from './tx'
import { TxProcessor } from './tx'
import { getTypeOf } from './typeof'
/**
* @public
@@ -587,33 +587,11 @@ export class Hierarchy {
}
clone (obj: any): any {
if (typeof obj === 'undefined') {
return undefined
}
if (typeof obj === 'function') {
return obj
}
const isArray = Array.isArray(obj)
const result: any = isArray ? [] : Object.assign({}, obj)
for (const key in obj) {
// include prototype properties
const value = obj[key]
const type = getTypeOf(value)
if (type === 'Array') {
result[key] = this.clone(value)
} else if (type === 'Object') {
const m = Hierarchy.mixinClass(value)
const valClone = this.clone(value)
result[key] = m !== undefined ? this.as(valClone, m) : valClone
} else if (type === 'Date') {
result[key] = new Date(value.getTime())
} else {
if (isArray) {
result[key] = value
}
}
}
return result
return deepClone(
obj,
(doc, m) => this.as(doc, m),
(value) => Hierarchy.mixinClass(value)
)
}
domains (): Domain[] {
+1 -1
View File
@@ -30,6 +30,6 @@ export * from './tx'
export * from './utils'
export * from './backup'
export * from './status'
export * from './typeof'
export * from './clone'
export * from './common'
export * from './time'
+79 -17
View File
@@ -14,13 +14,13 @@
//
import { PlatformError, Severity, Status } from '@hcengineering/platform'
import { Lookup, ReverseLookups, getObjectValue } from '.'
import type { Class, Doc, Ref } from './classes'
import { Lookup, MeasureContext, ReverseLookups, getObjectValue } from '.'
import type { AttachedDoc, Class, Doc, Ref } from './classes'
import core from './component'
import { Hierarchy } from './hierarchy'
import { checkMixinKey, matchQuery, resultSort } from './query'
import type { DocumentQuery, FindOptions, FindResult, LookupData, Storage, TxResult, WithLookup } from './storage'
import type { Tx, TxCreateDoc, TxMixin, TxRemoveDoc, TxUpdateDoc } from './tx'
import type { Tx, TxCollectionCUD, TxCreateDoc, TxMixin, TxRemoveDoc, TxUpdateDoc } from './tx'
import { TxProcessor } from './tx'
import { toFindResult } from './utils'
@@ -28,17 +28,17 @@ import { toFindResult } from './utils'
* @public
*/
export abstract class MemDb extends TxProcessor implements Storage {
private readonly objectsByClass = new Map<Ref<Class<Doc>>, Doc[]>()
private readonly objectsByClass = new Map<Ref<Class<Doc>>, Map<Ref<Doc>, Doc>>()
private readonly objectById = new Map<Ref<Doc>, Doc>()
constructor (protected readonly hierarchy: Hierarchy) {
super()
}
private getObjectsByClass (_class: Ref<Class<Doc>>): Doc[] {
private getObjectsByClass (_class: Ref<Class<Doc>>): Map<Ref<Doc>, Doc> {
const result = this.objectsByClass.get(_class)
if (result === undefined) {
const result: Doc[] = []
const result = new Map<Ref<Doc>, Doc>()
this.objectsByClass.set(_class, result)
return result
}
@@ -46,10 +46,9 @@ export abstract class MemDb extends TxProcessor implements Storage {
}
private cleanObjectByClass (_class: Ref<Class<Doc>>, _id: Ref<Doc>): void {
let result = this.objectsByClass.get(_class)
const result = this.objectsByClass.get(_class)
if (result !== undefined) {
result = result.filter((cl) => cl._id !== _id)
this.objectsByClass.set(_class, result)
result.delete(_id)
}
}
@@ -152,7 +151,7 @@ export abstract class MemDb extends TxProcessor implements Storage {
) {
result = this.getByIdQuery(query, baseClass)
} else {
result = this.getObjectsByClass(baseClass)
result = Array.from(this.getObjectsByClass(baseClass).values())
}
result = matchQuery(result, query, _class, this.hierarchy, true)
@@ -195,7 +194,7 @@ export abstract class MemDb extends TxProcessor implements Storage {
) {
result = this.getByIdQuery(query, baseClass)
} else {
result = this.getObjectsByClass(baseClass)
result = Array.from(this.getObjectsByClass(baseClass).values())
}
result = matchQuery(result, query, _class, this.hierarchy, true)
@@ -214,12 +213,7 @@ export abstract class MemDb extends TxProcessor implements Storage {
addDoc (doc: Doc): void {
this.hierarchy.getAncestors(doc._class).forEach((_class) => {
const arr = this.getObjectsByClass(_class)
const index = arr.findIndex((p) => p._id === doc._id)
if (index === -1) {
arr.push(doc)
} else {
arr[index] = doc
}
arr.set(doc._id, doc)
})
this.objectById.set(doc._id, doc)
}
@@ -275,6 +269,74 @@ export class ModelDb extends MemDb {
return {}
}
addTxes (ctx: MeasureContext, txes: Tx[], clone: boolean): void {
for (const tx of txes) {
switch (tx._class) {
case core.class.TxCreateDoc:
this.addDoc(TxProcessor.createDoc2Doc(tx as TxCreateDoc<Doc>, clone))
break
case core.class.TxCollectionCUD: {
// We need update only create transactions to contain attached, attachedToClass.
const cud = tx as TxCollectionCUD<Doc, AttachedDoc<Doc>>
if (cud.tx._class === core.class.TxCreateDoc) {
const createTx = cud.tx as TxCreateDoc<AttachedDoc>
const d: TxCreateDoc<AttachedDoc> = {
...createTx,
attributes: {
...createTx.attributes,
attachedTo: cud.objectId,
attachedToClass: cud.objectClass,
collection: cud.collection
}
}
this.addDoc(TxProcessor.createDoc2Doc(d as TxCreateDoc<Doc>, clone))
}
this.addTxes(ctx, [cud.tx], clone)
break
}
case core.class.TxUpdateDoc: {
const cud = tx as TxUpdateDoc<Doc>
const doc = this.findObject(cud.objectId)
if (doc !== undefined) {
TxProcessor.updateDoc2Doc(doc, cud)
} else {
void ctx.error('no document found, failed to apply model transaction, skipping', {
_id: tx._id,
_class: tx._class,
objectId: cud.objectId
})
}
break
}
case core.class.TxRemoveDoc:
try {
this.delDoc((tx as TxRemoveDoc<Doc>).objectId)
} catch (err: any) {
void ctx.error('no document found, failed to apply model transaction, skipping', {
_id: tx._id,
_class: tx._class,
objectId: (tx as TxRemoveDoc<Doc>).objectId
})
}
break
case core.class.TxMixin: {
const mix = tx as TxMixin<Doc, Doc>
const obj = this.findObject(mix.objectId)
if (obj !== undefined) {
TxProcessor.updateMixin4Doc(obj, mix)
} else {
void ctx.error('no document found, failed to apply model transaction, skipping', {
_id: tx._id,
_class: tx._class,
objectId: mix.objectId
})
}
break
}
}
}
}
protected async txUpdateDoc (tx: TxUpdateDoc<Doc>): Promise<TxResult> {
const doc = this.getObject(tx.objectId) as any
TxProcessor.updateDoc2Doc(doc, tx)
+2 -2
View File
@@ -1,7 +1,7 @@
import { PlatformError, Severity, Status } from '@hcengineering/platform'
import { Doc } from './classes'
import { clone } from './clone'
import core from './component'
import justClone from 'just-clone'
/**
* @public
@@ -60,7 +60,7 @@ export function setObjectValue (key: string, doc: Doc, newValue: any): void {
value = lvalue
}
}
value[last] = justClone(newValue)
value[last] = clone(newValue)
return value
}
+3 -3
View File
@@ -13,7 +13,6 @@
// limitations under the License.
//
import justClone from 'just-clone'
import type { KeysByType } from 'simplytyped'
import type {
Account,
@@ -35,6 +34,7 @@ import { _getOperator } from './operator'
import { _toDoc } from './proxy'
import type { DocumentQuery, TxResult } from './storage'
import { generateId } from './utils'
import { clone } from './clone'
/**
* @public
@@ -357,10 +357,10 @@ export abstract class TxProcessor implements WithTx {
return result
}
static createDoc2Doc<T extends Doc>(tx: TxCreateDoc<T>): T {
static createDoc2Doc<T extends Doc>(tx: TxCreateDoc<T>, doClone = true): T {
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
return {
...justClone(tx.attributes),
...(doClone ? clone(tx.attributes) : tx.attributes),
_id: tx.objectId,
_class: tx.objectClass,
space: tx.objectSpace,
-35
View File
@@ -1,35 +0,0 @@
const se = typeof Symbol !== 'undefined'
const ste = se && typeof Symbol.toStringTag !== 'undefined'
export function getTypeOf (obj: any): string {
const typeofObj = typeof obj
if (typeofObj !== 'object') {
return typeofObj
}
if (obj === null) {
return 'null'
}
if (Array.isArray(obj) && (!ste || !(Symbol.toStringTag in obj))) {
return 'Array'
}
const stringTag = ste && obj[Symbol.toStringTag]
if (typeof stringTag === 'string') {
return stringTag
}
const objPrototype = Object.getPrototypeOf(obj)
if (objPrototype === RegExp.prototype) {
return 'RegExp'
}
if (objPrototype === Date.prototype) {
return 'Date'
}
if (objPrototype === null) {
return 'Object'
}
return {}.toString.call(obj).slice(8, -1)
}