UBERF-6161: Storage configuration (#5109)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

server/mongo/src/__tests__/storage.test.ts

@@ -54,6 +54,7 @@ const txes = genMinModel()
 createTaskModel(txes)
 
 async function createNullAdapter (
+  ctx: MeasureContext,
   hierarchy: Hierarchy,
   url: string,
   db: WorkspaceId,
@@ -116,7 +117,13 @@ describe('mongo operations', () => {
     }
 
     const mctx = new MeasureMetricsContext('', {})
-    const txStorage = await createMongoTxAdapter(hierarchy, mongodbUri, getWorkspaceId(dbId, ''), model)
+    const txStorage = await createMongoTxAdapter(
+      new MeasureMetricsContext('', {}),
+      hierarchy,
+      mongodbUri,
+      getWorkspaceId(dbId, ''),
+      model
+    )
 
     // Put all transactions to Tx
     for (const t of txes) {

server/mongo/src/index.ts

@@ -14,5 +14,6 @@
 // limitations under the License.
 //
 
+export * from './rawAdapter'
 export * from './storage'
 export * from './utils'

server/mongo/src/rawAdapter.ts

@@ -0,0 +1,84 @@
+import {
+  SortingOrder,
+  cutObjectArray,
+  toFindResult,
+  type Doc,
+  type DocumentQuery,
+  type Domain,
+  type FindOptions,
+  type FindResult,
+  type WorkspaceId
+} from '@hcengineering/core'
+import type { RawDBAdapter } from '@hcengineering/server-core'
+import { type Document, type Filter, type Sort } from 'mongodb'
+import { toArray, uploadDocuments } from './storage'
+import { getMongoClient, getWorkspaceDB } from './utils'
+
+export function createRawMongoDBAdapter (url: string): RawDBAdapter {
+  const client = getMongoClient(url)
+
+  const collectSort = (options: FindOptions<Doc>): Sort | undefined => {
+    if (options?.sort === undefined) {
+      return undefined
+    }
+    const sort: Sort = {}
+    let count = 0
+    for (const key in options.sort) {
+      const order = options.sort[key] === SortingOrder.Ascending ? 1 : -1
+      sort[key] = order
+      count++
+    }
+    if (count === 0) {
+      return undefined
+    }
+    return sort
+  }
+  return {
+    find: async function <T extends Doc>(
+      workspace: WorkspaceId,
+      domain: Domain,
+      query: DocumentQuery<T>,
+      options?: Omit<FindOptions<T>, 'projection' | 'lookup'>
+    ): Promise<FindResult<T>> {
+      const db = getWorkspaceDB(await client.getClient(), workspace)
+      const coll = db.collection(domain)
+      let cursor = coll.find<T>(query as Filter<Document>, {
+        checkKeys: false,
+        enableUtf8Validation: false
+      })
+
+      let total: number = -1
+      if (options != null) {
+        if (options.sort !== undefined) {
+          const sort = collectSort(options)
+          if (sort !== undefined) {
+            cursor = cursor.sort(sort)
+          }
+        }
+        if (options.limit !== undefined || typeof query._id === 'string') {
+          if (options.total === true) {
+            total = await coll.countDocuments(query)
+          }
+          cursor = cursor.limit(options.limit ?? 1)
+        }
+      }
+
+      // Error in case of timeout
+      try {
+        const res = await toArray<T>(cursor)
+        if (options?.total === true && options?.limit === undefined) {
+          total = res.length
+        }
+        return toFindResult(res, total)
+      } catch (e) {
+        console.error('error during executing cursor in findAll', cutObjectArray(query), options, e)
+        throw e
+      }
+    },
+    upload: async (workspace, domain, docs) => {
+      const db = getWorkspaceDB(await client.getClient(), workspace)
+      const coll = db.collection(domain)
+      await uploadDocuments(docs, coll)
+    }
+  }
+}

server/mongo/src/storage.ts

@@ -100,6 +100,67 @@ interface LookupStep {
   pipeline?: any
 }
 
+export async function toArray<T> (cursor: AbstractCursor<T>): Promise<T[]> {
+  const data = await cursor.toArray()
+  await cursor.close()
+  return data
+}
+
+/**
+ * Return some estimation for object size
+ */
+function calcSize (obj: any): number {
+  if (typeof obj === 'undefined') {
+    return 0
+  }
+  if (typeof obj === 'function') {
+    return 0
+  }
+  let result = 0
+  for (const key in obj) {
+    // include prototype properties
+    const value = obj[key]
+    const type = getTypeOf(value)
+    result += key.length
+
+    switch (type) {
+      case 'Array':
+        result += 4 + calcSize(value)
+        break
+      case 'Object':
+        result += calcSize(value)
+        break
+      case 'Date':
+        result += 24 // Some value
+        break
+      case 'string':
+        result += (value as string).length
+        break
+      case 'number':
+        result += 8
+        break
+      case 'boolean':
+        result += 1
+        break
+      case 'symbol':
+        result += (value as symbol).toString().length
+        break
+      case 'bigint':
+        result += (value as bigint).toString().length
+        break
+      case 'undefined':
+        result += 1
+        break
+      case 'null':
+        result += 1
+        break
+      default:
+        result += value.toString().length
+    }
+  }
+  return result
+}
+
 abstract class MongoAdapterBase implements DbAdapter {
   constructor (
     protected readonly db: Db,
@@ -110,12 +171,6 @@ abstract class MongoAdapterBase implements DbAdapter {
 
   async init (): Promise<void> {}
 
-  async toArray<T>(cursor: AbstractCursor<T>): Promise<T[]> {
-    const data = await cursor.toArray()
-    await cursor.close()
-    return data
-  }
-
   async createIndexes (domain: Domain, config: Pick<IndexingConfiguration<Doc>, 'indexes'>): Promise<void> {
     for (const vv of config.indexes) {
       try {
@@ -469,7 +524,7 @@ abstract class MongoAdapterBase implements DbAdapter {
     let result: WithLookup<T>[] = []
     let total = options?.total === true ? 0 : -1
     try {
-      result = (await ctx.with('toArray', {}, async (ctx) => await this.toArray(cursor), {
+      result = (await ctx.with('toArray', {}, async (ctx) => await toArray(cursor), {
         domain,
         pipeline
       })) as any[]
@@ -489,7 +544,7 @@ abstract class MongoAdapterBase implements DbAdapter {
         checkKeys: false,
         enableUtf8Validation: false
       })
-      const arr = await this.toArray(totalCursor)
+      const arr = await toArray(totalCursor)
       total = arr?.[0]?.total ?? 0
     }
     return toFindResult(this.stripHash(result), total)
@@ -620,7 +675,7 @@ abstract class MongoAdapterBase implements DbAdapter {
 
     // Error in case of timeout
     try {
-      const res: T[] = await ctx.with('toArray', {}, async (ctx) => await this.toArray(cursor), {
+      const res: T[] = await ctx.with('toArray', {}, async (ctx) => await toArray(cursor), {
         mongoQuery,
         options,
         domain
@@ -729,7 +784,7 @@ abstract class MongoAdapterBase implements DbAdapter {
         }
         const pos = (digest ?? '').indexOf('|')
         if (digest == null || digest === '' || pos === -1) {
-          const size = this.calcSize(d)
+          const size = calcSize(d)
           digest = hashID // we just need some random value
 
           bulkUpdate.set(d._id, `${digest}|${size.toString(16)}`)
@@ -755,95 +810,19 @@ abstract class MongoAdapterBase implements DbAdapter {
     }
   }
 
-  /**
-   * Return some estimation for object size
-   */
-  calcSize (obj: any): number {
-    if (typeof obj === 'undefined') {
-      return 0
-    }
-    if (typeof obj === 'function') {
-      return 0
-    }
-    let result = 0
-    for (const key in obj) {
-      // include prototype properties
-      const value = obj[key]
-      const type = getTypeOf(value)
-      result += key.length
-
-      switch (type) {
-        case 'Array':
-          result += 4 + this.calcSize(value)
-          break
-        case 'Object':
-          result += this.calcSize(value)
-          break
-        case 'Date':
-          result += 24 // Some value
-          break
-        case 'string':
-          result += (value as string).length
-          break
-        case 'number':
-          result += 8
-          break
-        case 'boolean':
-          result += 1
-          break
-        case 'symbol':
-          result += (value as symbol).toString().length
-          break
-        case 'bigint':
-          result += (value as bigint).toString().length
-          break
-        case 'undefined':
-          result += 1
-          break
-        case 'null':
-          result += 1
-          break
-        default:
-          result += value.toString().length
-      }
-    }
-    return result
-  }
-
   async load (domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
     if (docs.length === 0) {
       return []
     }
     const cursor = this.db.collection<Doc>(domain).find<Doc>({ _id: { $in: docs } }, { limit: docs.length })
-    const result = await this.toArray(cursor)
+    const result = await toArray(cursor)
     return this.stripHash(this.stripHash(result))
   }
 
   async upload (domain: Domain, docs: Doc[]): Promise<void> {
     const coll = this.db.collection(domain)
 
-    const ops = Array.from(docs)
-
-    while (ops.length > 0) {
-      const part = ops.splice(0, 500)
-      await coll.bulkWrite(
-        part.map((it) => {
-          const digest: string | null = (it as any)['%hash%']
-          if ('%hash%' in it) {
-            delete it['%hash%']
-          }
-          const size = this.calcSize(it)
-
-          return {
-            replaceOne: {
-              filter: { _id: it._id },
-              replacement: { ...it, '%hash%': digest == null ? null : `${digest}|${size.toString(16)}` },
-              upsert: true
-            }
-          }
-        })
-      )
-    }
+    await uploadDocuments(docs, coll)
   }
 
   async update (domain: Domain, operations: Map<Ref<Doc>, DocumentUpdate<Doc>>): Promise<void> {
@@ -1290,7 +1269,7 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter {
       .collection(DOMAIN_TX)
       .find<Tx>({ objectSpace: core.space.Model })
       .sort({ _id: 1, modifiedOn: 1 })
-    const model = await this.toArray(cursor)
+    const model = await toArray(cursor)
     // We need to put all core.account.System transactions first
     const systemTx: Tx[] = []
     const userTx: Tx[] = []
@@ -1311,6 +1290,31 @@ class MongoTxAdapter extends MongoAdapterBase implements TxAdapter {
   }
 }
 
+export async function uploadDocuments (docs: Doc[], coll: Collection<Document>): Promise<void> {
+  const ops = Array.from(docs)
+
+  while (ops.length > 0) {
+    const part = ops.splice(0, 500)
+    await coll.bulkWrite(
+      part.map((it) => {
+        const digest: string | null = (it as any)['%hash%']
+        if ('%hash%' in it) {
+          delete it['%hash%']
+        }
+        const size = calcSize(it)
+
+        return {
+          replaceOne: {
+            filter: { _id: it._id },
+            replacement: { ...it, '%hash%': digest == null ? null : `${digest}|${size.toString(16)}` },
+            upsert: true
+          }
+        }
+      })
+    )
+  }
+}
+
 function fillEnumSort (
   enumOf: Enum,
   key: string,
@@ -1402,6 +1406,7 @@ function translateLikeQuery (pattern: string): { $regex: string, $options: strin
  * @public
  */
 export async function createMongoAdapter (
+  ctx: MeasureContext,
   hierarchy: Hierarchy,
   url: string,
   workspaceId: WorkspaceId,
@@ -1417,6 +1422,7 @@ export async function createMongoAdapter (
  * @public
  */
 export async function createMongoTxAdapter (
+  ctx: MeasureContext,
   hierarchy: Hierarchy,
   url: string,
   workspaceId: WorkspaceId,