Tracebility matrix preparation (#10286)

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

foundations/core/packages/core/src/__tests__/memdb.test.ts

@@ -14,7 +14,7 @@
 //
 
 import { type Client, type DomainParams, type DomainRequestOptions, type DomainResult } from '..'
-import type { Class, Doc, Obj, OperationDomain, Ref } from '../classes'
+import type { Class, Doc, Obj, OperationDomain, Ref, Space } from '../classes'
 import core from '../component'
 import { Hierarchy } from '../hierarchy'
 import { ModelDb, TxDb } from '../memdb'
@@ -293,6 +293,155 @@ describe('memdb', () => {
     expect(result2).toHaveLength(1)
   })
 
+  it('check associations', async () => {
+    const { model } = await createModel()
+    const operations = new TxOperations(model, core.account.System)
+    const association = await operations.findOne(core.class.Association, {})
+    if (association == null) {
+      throw new Error('Association not found')
+    }
+
+    const spaces = await operations.findAll(core.class.Space, {})
+    expect(spaces).toHaveLength(2)
+
+    const first = await operations.addCollection(
+      test.class.TestComment,
+      core.space.Model,
+      spaces[0]._id,
+      spaces[0]._class,
+      'comments',
+      {
+        message: 'msg'
+      }
+    )
+
+    const second = await operations.addCollection(
+      test.class.TestComment,
+      core.space.Model,
+      first,
+      test.class.TestComment,
+      'comments',
+      {
+        message: 'msg2'
+      }
+    )
+
+    await operations.createDoc(core.class.Relation, '' as Ref<Space>, {
+      docA: first,
+      docB: second,
+      association: association._id
+    })
+
+    const r = await operations.findAll(
+      test.class.TestComment,
+      { _id: first },
+      {
+        associations: [[association._id, 1]]
+      }
+    )
+    expect(r.length).toEqual(1)
+    expect((r[0].$associations?.[association._id + '_b'][0] as any)?._id).toEqual(second)
+  })
+
+  it('check deep associations', async () => {
+    const { model } = await createModel()
+    const operations = new TxOperations(model, core.account.System)
+    const association = await operations.findOne(core.class.Association, {})
+    if (association == null) {
+      throw new Error('Association not found')
+    }
+
+    const spaces = await operations.findAll(core.class.Space, {})
+    expect(spaces).toHaveLength(2)
+
+    const zero = await operations.addCollection(
+      test.class.TestComment,
+      core.space.Model,
+      spaces[0]._id,
+      spaces[0]._class,
+      'comments',
+      {
+        message: 'msg'
+      }
+    )
+
+    const first = await operations.addCollection(
+      test.class.TestComment,
+      core.space.Model,
+      spaces[0]._id,
+      spaces[0]._class,
+      'comments',
+      {
+        message: 'msg'
+      }
+    )
+
+    const second = await operations.addCollection(
+      test.class.TestComment,
+      core.space.Model,
+      first,
+      test.class.TestComment,
+      'comments',
+      {
+        message: 'msg2'
+      }
+    )
+
+    const second2 = await operations.addCollection(
+      test.class.TestComment,
+      core.space.Model,
+      first,
+      test.class.TestComment,
+      'comments',
+      {
+        message: 'msg2'
+      }
+    )
+
+    const third = await operations.addCollection(
+      test.class.TestComment,
+      core.space.Model,
+      spaces[0]._id,
+      spaces[0]._class,
+      'comments',
+      {
+        message: 'msg3'
+      }
+    )
+    await operations.createDoc(core.class.Relation, '' as Ref<Space>, {
+      docA: first,
+      docB: second,
+      association: association._id
+    })
+
+    await operations.createDoc(core.class.Relation, '' as Ref<Space>, {
+      docA: first,
+      docB: second2,
+      association: association._id
+    })
+
+    await operations.createDoc(core.class.Relation, '' as Ref<Space>, {
+      docA: second,
+      docB: third,
+      association: association._id
+    })
+
+    const r = await operations.findAll(
+      test.class.TestComment,
+      { _id: { $in: [zero, first] } },
+      {
+        associations: [[association._id, 1, [[association._id, 1]]]]
+      }
+    )
+    expect(r.length).toEqual(2)
+    expect(r[1].$associations?.[`${association._id}_b`]).toHaveLength(2)
+    expect((r[1].$associations?.[`${association._id}_b`][0] as any)?._id).toEqual(second)
+    expect(r[1].$associations?.[`${association._id}_b`][1]?.$associations?.[`${association._id}_b`]).toHaveLength(0)
+    expect(
+      (r[1].$associations?.[`${association._id}_b`][0]?.$associations?.[`${association._id}_b`][0] as any)?._id
+    ).toEqual(third)
+  })
+
   it('lookups', async () => {
     const { model } = await createModel()
 

foundations/core/packages/core/src/__tests__/minmodel.ts

@@ -16,7 +16,7 @@
 import type { IntlString, Plugin } from '@hcengineering/platform'
 import { plugin } from '@hcengineering/platform'
 import type { Arr, AttachedDoc, Class, Data, Doc, Interface, Mixin, Obj, Ref, Space } from '../classes'
-import { ClassifierKind, DOMAIN_MODEL } from '../classes'
+import { ClassifierKind, DOMAIN_MODEL, DOMAIN_RELATION } from '../classes'
 import core from '../component'
 import type { DocumentUpdate, TxCUD, TxCreateDoc, TxRemoveDoc, TxUpdateDoc } from '../tx'
 import { DOMAIN_TX, TxFactory } from '../tx'
@@ -252,6 +252,34 @@ export function genMinModel (): TxCUD<Doc>[] {
     })
   )
 
+  txes.push(
+    createClass(core.class.Relation, {
+      label: 'Relation' as IntlString,
+      extends: core.class.Doc,
+      kind: ClassifierKind.CLASS,
+      domain: DOMAIN_RELATION
+    })
+  )
+
+  txes.push(
+    createClass(core.class.Association, {
+      label: 'Association' as IntlString,
+      extends: core.class.Doc,
+      kind: ClassifierKind.CLASS,
+      domain: DOMAIN_MODEL
+    })
+  )
+
+  txes.push(
+    createDoc(core.class.Association, {
+      nameA: 'my-assoc',
+      nameB: 'my-assoc',
+      classA: test.class.TestComment,
+      classB: test.class.TestComment,
+      type: '1:1'
+    })
+  )
+
   txes.push(
     createDoc(core.class.Space, {
       name: 'Sp1',

foundations/core/packages/core/src/memdb.ts

@@ -159,7 +159,7 @@ export abstract class MemDb extends TxProcessor implements Storage {
   private async getAssociationValue<T extends Doc>(
     doc: T,
     associations: AssociationQuery[]
-  ): Promise<Record<string, Doc[]>> {
+  ): Promise<Record<string, WithLookup<Doc>[]>> {
     const result: Record<string, Doc[]> = {}
     for (const association of associations) {
       const _id = association[0]
@@ -170,8 +170,11 @@ export abstract class MemDb extends TxProcessor implements Storage {
       const key2 = !isReverse ? 'docB' : 'docA'
       const _class = !isReverse ? assoc.classB : assoc.classA
       const relations = await this.findAll(core.class.Relation, { association: _id, [key]: doc._id })
-      const objects = await this.findAll(_class, { _id: { $in: relations.map((r) => r[key2]) } })
-      result[_id] = objects
+      let objects = await this.findAll(_class, { _id: { $in: relations.map((r) => r[key2]) } })
+      if (association[2] !== undefined) {
+        objects = toFindResult(await this.fillAssociations(objects, association[2]), objects.length)
+      }
+      result[`${_id}_${!isReverse ? 'b' : 'a'}`] = objects
     }
     return result
   }

foundations/core/packages/core/src/storage.ts

@@ -128,7 +128,7 @@ export type Projection<T extends Doc> = {
   [P in keyof T]?: 0 | 1
 }
 
-export type AssociationQuery = [Ref<Association>, 1 | -1]
+export type AssociationQuery = [Ref<Association>, 1 | -1] | [Ref<Association>, 1 | -1, AssociationQuery[]]
 
 /**
  * @public
@@ -205,7 +205,7 @@ export type LookupData<T extends Doc> = Partial<RefsAsDocs<T>>
  */
 export type WithLookup<T extends Doc> = T & {
   $lookup?: LookupData<T>
-  $associations?: Record<string, Doc[]>
+  $associations?: Record<string, WithLookup<Doc>[]>
   $source?: {
     $score: number // Score for document result
     [key: string]: any

foundations/core/packages/query/src/index.ts

@@ -780,7 +780,9 @@ export class LiveQuery implements WithTx, Client {
         }
         const docs = q.result.getDocs()
         for (const doc of docs) {
-          const docToUpdate = doc.$associations?.[association._id]?.find((it) => it._id === tx.objectId)
+          const docToUpdate =
+            doc.$associations?.[`${association._id}_a`]?.find((it) => it._id === tx.objectId) ??
+            doc.$associations?.[`${association._id}_b`]?.find((it) => it._id === tx.objectId)
           if (docToUpdate !== undefined) {
             if (tx._class === core.class.TxMixin) {
               TxProcessor.updateMixin4Doc(docToUpdate, tx as TxMixin<Doc, Doc>)
@@ -1128,12 +1130,13 @@ export class LiveQuery implements WithTx, Client {
         _id: direct ? relation.docB : relation.docA
       })
       if (docToPush === undefined) return
-      const arr = res?.$associations?.[relation.association] ?? []
-      arr.push(docToPush)
       if (res?.$associations === undefined) {
         res.$associations = {}
       }
-      res.$associations[relation.association] = arr
+      const key = direct ? 'b' : 'a'
+      const arr = res?.$associations?.[`${relation.association}_${key}`] ?? []
+      arr.push(docToPush)
+      res.$associations[`${relation.association}_${key}`] = arr
       q.result.updateDoc(res, false)
       this.queriesToUpdate.set(q.id, q)
     }

foundations/server/packages/core/src/__tests__/shared-integration.ts

@@ -448,7 +448,7 @@ export function runSharedIntegrationTests (adapterName: string, getContext: () =
           }
         )
         expect(r.length).toEqual(1)
-        expect(r[0].$associations?.[association._id][0]?._id).toEqual(secondTask)
+        expect(r[0].$associations?.[`${association._id}_b`][0]?._id).toEqual(secondTask)
       })
     })
 

foundations/server/packages/mongo/src/__tests__/storage.test.ts

@@ -323,7 +323,78 @@ describe('mongo operations', () => {
       }
     )
     expect(r.length).toEqual(1)
-    expect((r[0].$associations?.[association._id][0] as unknown as Task)?._id).toEqual(secondTask)
+    expect((r[0].$associations?.[association._id + '_b'][0] as unknown as Task)?._id).toEqual(secondTask)
+  })
+
+  it('check deep associations', async () => {
+    const association = await operations.findOne(core.class.Association, {})
+    if (association == null) {
+      throw new Error('Association not found')
+    }
+
+    const zeroTask = await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
+      name: 'my-task',
+      description: 'Descr',
+      rate: 20
+    })
+
+    const firstTask = await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
+      name: 'my-task',
+      description: 'Descr',
+      rate: 20
+    })
+
+    const secondTask = await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
+      name: 'my-task2',
+      description: 'Descr',
+      rate: 20
+    })
+
+    const secondATask = await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
+      name: 'my-task22',
+      description: 'Descr',
+      rate: 20
+    })
+
+    const thirdTask = await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
+      name: 'my-task3',
+      description: 'Descr',
+      rate: 20
+    })
+
+    await operations.createDoc(core.class.Relation, '' as Ref<Space>, {
+      docA: firstTask,
+      docB: secondTask,
+      association: association._id
+    })
+
+    await operations.createDoc(core.class.Relation, '' as Ref<Space>, {
+      docA: firstTask,
+      docB: secondATask,
+      association: association._id
+    })
+
+    await operations.createDoc(core.class.Relation, '' as Ref<Space>, {
+      docA: secondTask,
+      docB: thirdTask,
+      association: association._id
+    })
+
+    const r = await client.findAll(
+      taskPlugin.class.Task,
+      { _id: { $in: [zeroTask, firstTask] } },
+      {
+        associations: [[association._id, 1, [[association._id, 1]]]]
+      }
+    )
+    expect(r.length).toEqual(2)
+    expect(r[1].$associations?.[`${association._id}_b`]).toHaveLength(2)
+    expect((r[1].$associations?.[`${association._id}_b`][0] as unknown as Task)?._id).toEqual(secondTask)
+    expect(r[1].$associations?.[`${association._id}_b`][1]?.$associations?.[`${association._id}_b`]).toHaveLength(0)
+    expect(
+      (r[1].$associations?.[`${association._id}_b`][0]?.$associations?.[`${association._id}_b`][0] as unknown as Task)
+        ?._id
+    ).toEqual(thirdTask)
   })
 
   // Run shared integration tests

foundations/server/packages/mongo/src/storage.ts

@@ -467,19 +467,21 @@ abstract class MongoAdapterBase implements DbAdapter {
     return result
   }
 
-  private getAssociations (associations: AssociationQuery[]): LookupStep[] {
+  private getAssociations (associations: AssociationQuery[], parentId: string = ''): LookupStep[] {
     const res: LookupStep[] = []
     for (const association of associations) {
-      const assoc = this.modelDb.findObject(association[0])
+      const _id = association[0]
+      const assoc = this.modelDb.findObject(_id)
       if (assoc === undefined) continue
       const isReverse = association[1] === -1
       const _class = !isReverse ? assoc.classB : assoc.classA
+      const fullId = _id + (isReverse ? '_a' : '_b')
       const targetDomain = this.hierarchy.getDomain(_class)
       if (targetDomain === DOMAIN_MODEL) continue
-      const as = association[0] + '_hidden_association'
+      const as = parentId + fullId + '_hidden_association'
       res.push({
         from: DOMAIN_RELATION,
-        localField: '_id',
+        localField: `${parentId !== '' ? parentId + '.' : ''}_id`,
         foreignField: isReverse ? 'docB' : 'docA',
         as
       })
@@ -487,8 +489,11 @@ abstract class MongoAdapterBase implements DbAdapter {
         from: targetDomain,
         localField: as + '.' + (isReverse ? 'docA' : 'docB'),
         foreignField: '_id',
-        as: association[0] + '_association'
+        as: parentId + fullId + '_association'
       })
+      if (association[2] !== undefined) {
+        res.push(...this.getAssociations(association[2], parentId + fullId + '_association'))
+      }
     }
     return res
   }
@@ -518,19 +523,36 @@ abstract class MongoAdapterBase implements DbAdapter {
     }
   }
 
-  private fillAssociationsValue (associations: AssociationQuery[], object: any): Record<string, Doc[]> {
+  private fillAssociationsValue (
+    source: any,
+    associations: AssociationQuery[],
+    parentKey: string,
+    targetId: Ref<Doc>
+  ): Record<string, Doc[]> {
     const res: Record<string, Doc[]> = {}
     for (const association of associations) {
-      const assocKey = association[0] + '_hidden_association'
-      const data = object[assocKey]
+      const _id = association[0]
+      const isReverse = association[1] === -1
+      // const key = _id + (isReverse ? '.a' : '.b')
+      const fullId = _id + (isReverse ? '_a' : '_b')
+      const assocKey = parentKey + fullId + '_hidden_association'
+      const data = source[assocKey]
       if (data !== undefined && Array.isArray(data)) {
         const filtered = new Set(
-          data.filter((it) => it.association === association[0]).map((it) => (association[1] === 1 ? it.docB : it.docA))
+          data
+            .filter((it) => it.association === _id && (isReverse ? it.docB : it.docA) === targetId)
+            .map((it) => (!isReverse ? it.docB : it.docA))
         )
-        const fullKey = association[0] + '_association'
-        const arr = object[fullKey]
+        const fullKey = parentKey + fullId + '_association'
+        const arr = source[fullKey] as WithLookup<Doc>[]
         if (arr !== undefined && Array.isArray(arr)) {
-          res[association[0]] = arr.filter((it) => filtered.has(it._id))
+          const objects = arr.filter((it) => filtered.has(it._id))
+          if (association[2] !== undefined) {
+            for (const obj of objects) {
+              this.fillAssociations(obj, association[2], fullKey, source)
+            }
+          }
+          res[fullId] = objects
         }
       }
     }
@@ -718,10 +740,7 @@ abstract class MongoAdapterBase implements DbAdapter {
         }
       }
       if (options.associations !== undefined && options.associations.length > 0) {
-        row.$associations = this.fillAssociationsValue(options.associations, row)
-        for (const [, v] of Object.entries(row.$associations)) {
-          this.stripHash(v)
-        }
+        this.fillAssociations(row, options.associations)
       }
       this.clearExtraLookups(row)
     }
@@ -756,6 +775,19 @@ abstract class MongoAdapterBase implements DbAdapter {
     return toFindResult(this.stripHash(result) as T[], total)
   }
 
+  private fillAssociations<T extends Doc>(
+    obj: WithLookup<T>,
+    associations: AssociationQuery[],
+    parentId: string = '',
+    source?: Doc
+  ): WithLookup<T> {
+    obj.$associations = this.fillAssociationsValue(source ?? obj, associations, parentId, obj._id)
+    for (const [, v] of Object.entries(obj.$associations)) {
+      this.stripHash(v)
+    }
+    return obj
+  }
+
   private translateKey<T extends Doc>(
     key: string,
     clazz: Ref<Class<T>>,

foundations/server/packages/postgres/src/__tests__/storage.test.ts

@@ -385,6 +385,57 @@ describe('postgres operations', () => {
       }
     )
     expect(r.length).toEqual(1)
-    expect((r[0].$associations?.[association._id][0] as unknown as Task)?._id).toEqual(secondTask)
+    expect((r[0].$associations?.[association._id + '_b'][0] as unknown as Task)?._id).toEqual(secondTask)
+  })
+
+  it('check deep associations', async () => {
+    const association = await operations.findOne(core.class.Association, {})
+    if (association == null) {
+      throw new Error('Association not found')
+    }
+
+    const firstTask = await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
+      name: 'my-task',
+      description: 'Descr',
+      rate: 20
+    })
+
+    const secondTask = await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
+      name: 'my-task2',
+      description: 'Descr',
+      rate: 20
+    })
+
+    const thirdTask = await operations.createDoc(taskPlugin.class.Task, '' as Ref<Space>, {
+      name: 'my-task2',
+      description: 'Descr',
+      rate: 20
+    })
+
+    await operations.createDoc(core.class.Relation, '' as Ref<Space>, {
+      docA: firstTask,
+      docB: secondTask,
+      association: association._id
+    })
+
+    await operations.createDoc(core.class.Relation, '' as Ref<Space>, {
+      docA: secondTask,
+      docB: thirdTask,
+      association: association._id
+    })
+
+    const r = await client.findAll(
+      taskPlugin.class.Task,
+      { _id: firstTask },
+      {
+        associations: [[association._id, 1, [[association._id, 1]]]]
+      }
+    )
+    expect(r.length).toEqual(1)
+    expect((r[0].$associations?.[`${association._id}_b`][0] as unknown as Task)?._id).toEqual(secondTask)
+    expect(
+      (r[0].$associations?.[`${association._id}_b`][0]?.$associations?.[`${association._id}_b`][0] as unknown as Task)
+        ?._id
+    ).toEqual(thirdTask)
   })
 })

foundations/server/packages/postgres/src/storage.ts

@@ -100,7 +100,6 @@ import {
   DBCollectionHelper,
   type DBDoc,
   escape,
-  filterProjection,
   inferType,
   isDataField,
   isOwner,
@@ -109,7 +108,8 @@ import {
   parseDoc,
   parseDocWithProjection,
   parseUpdate,
-  simpleEscape
+  simpleEscape,
+  toWithLookup
 } from './utils'
 async function * createCursorGenerator (
   client: postgres.Sql,
@@ -554,7 +554,7 @@ abstract class PostgresAdapterBase implements DbAdapter {
                 total
               )
             } else {
-              const res = this.parseLookup<T>(result, joins, projection, domain)
+              const res = await this.parseLookup<T>(ctx, result, joins, projection, options.associations, domain)
               return toFindResult(res, total)
             }
           })) as FindResult<T>
@@ -647,144 +647,191 @@ abstract class PostgresAdapterBase implements DbAdapter {
     }
   }
 
-  private parseLookup<T extends Doc>(
+  private async parseLookup<T extends Doc>(
+    ctx: MeasureContext<SessionData>,
     rows: any[],
     joins: JoinProps[],
     projection: Projection<T> | undefined,
+    associations: AssociationQuery[] | undefined,
     domain: string
-  ): WithLookup<T>[] {
+  ): Promise<WithLookup<T>[]> {
     const map = new Map<Ref<T>, WithLookup<T>>()
+
     const modelJoins: JoinProps[] = []
     const reverseJoins: JoinProps[] = []
     const simpleJoins: JoinProps[] = []
+
     for (const join of joins) {
-      if (join.table === DOMAIN_MODEL) {
-        modelJoins.push(join)
-      } else if (join.isReverse) {
-        reverseJoins.push(join)
-      } else if (join.path !== '') {
-        simpleJoins.push(join)
-      }
+      if (join.table === DOMAIN_MODEL) modelJoins.push(join)
+      else if (join.isReverse) reverseJoins.push(join)
+      else if (join.path !== '') simpleJoins.push(join)
     }
+
     for (const row of rows) {
-      /* eslint-disable @typescript-eslint/consistent-type-assertions */
-      let doc: WithLookup<T> = map.get(row._id) ?? ({ _id: row._id, $lookup: {}, $associations: {} } as WithLookup<T>)
-      const associations: Record<string, any> = doc.$associations as Record<string, any>
-      const lookup: Record<string, any> = doc.$lookup as Record<string, any>
-      let joinIndex: number | undefined
-      let skip = false
-      try {
-        const schema = getSchema(domain)
-        for (const column in row) {
-          if (column.startsWith('reverse_lookup_')) {
-            if (row[column] != null) {
-              const join = reverseJoins.find((j) => j.toAlias.toLowerCase() === column)
-              if (join === undefined) {
-                continue
-              }
-              const res = this.getLookupValue(join.path, lookup, false)
-              if (res === undefined) continue
-              const { obj, key } = res
+      const doc = toWithLookup<T>(parseDoc(row, getSchema(row._class)))
 
-              const parsed = row[column].map((p: any) => parseDoc(p, schema))
-              obj[key] = parsed
-            }
-          } else if (column.startsWith('lookup_')) {
-            const keys = column.split('_')
-            let key = keys[keys.length - 1]
-            if (keys[keys.length - 2] === '') {
-              key = '_' + key
-            }
+      const lookup = doc.$lookup as Record<string, any>
 
-            if (key === 'workspaceId') {
-              continue
-            }
+      this.parseLookupColumns(row, simpleJoins, reverseJoins, lookup, domain)
 
-            if (key === '_id') {
-              joinIndex = joinIndex === undefined ? 0 : ++joinIndex
-              if (row[column] === null) {
-                skip = true
-                continue
-              }
-              skip = false
-            }
-
-            if (skip) {
-              continue
-            }
-
-            const join = simpleJoins[joinIndex ?? 0]
-            if (join === undefined) {
-              continue
-            }
-            const res = this.getLookupValue(join.path, lookup)
-            if (res === undefined) continue
-            const { obj, key: p } = res
-
-            if (key === 'data') {
-              obj[p] = { ...obj[p], ...row[column] }
-            } else {
-              if (key === 'createdOn' || key === 'modifiedOn') {
-                const val = Number.parseInt(row[column])
-                obj[p][key] = Number.isNaN(val) ? null : val
-              } else if (key === '%hash%') {
-                continue
-              } else if (key === 'attachedTo' && row[column] === 'NULL') {
-                continue
-              } else {
-                obj[p][key] = row[column] === 'NULL' ? null : row[column]
-              }
-            }
-          } else if (column.startsWith('assoc_')) {
-            if (row[column] == null) continue
-            const keys = column.split('_')
-            const key = keys[keys.length - 1]
-            const associationDomain = keys[1]
-            const associationSchema = getSchema(associationDomain)
-            const parsed = row[column].map((p: any) => parseDoc(p, associationSchema))
-            associations[key] = parsed
-          } else {
-            joinIndex = undefined
-            if (!map.has(row._id)) {
-              if (column === 'workspaceId') {
-                continue
-              }
-              if (column === 'data') {
-                let data = row[column]
-                data = filterProjection(data, projection)
-                doc = { ...doc, ...data }
-              } else {
-                if (column === 'createdOn' || column === 'modifiedOn') {
-                  const val = Number.parseInt(row[column])
-                  ;(doc as any)[column] = Number.isNaN(val) ? null : val
-                } else if (column === '%hash%') {
-                  // Ignore
-                } else {
-                  ;(doc as any)[column] = row[column] === 'NULL' ? null : row[column]
-                }
-              }
-            }
-          }
-        }
-      } catch (err) {
-        console.log(err)
-        throw err
-      }
       for (const modelJoin of modelJoins) {
         const res = this.getLookupValue(modelJoin.path, lookup)
         if (res === undefined) continue
+
         const { obj, key } = res
         const val = this.getModelLookupValue<T>(doc, modelJoin, [...simpleJoins, ...modelJoins])
+
         if (val !== undefined && modelJoin.toClass !== undefined) {
-          const res = this.modelDb.findAllSync(modelJoin.toClass, {
+          const res2 = this.modelDb.findAllSync(modelJoin.toClass, {
             [modelJoin.toField]: val
           })
-          obj[key] = modelJoin.isReverse ? res : res[0]
+          obj[key] = modelJoin.isReverse ? res2 : res2[0]
         }
       }
+
       map.set(row._id, doc)
     }
-    return Array.from(map.values())
+
+    if (associations !== undefined && map.size > 0) {
+      await this.fetchAssociations(ctx, map, associations)
+    }
+
+    return [...map.values()]
+  }
+
+  async fetchAssociations (
+    ctx: MeasureContext<SessionData>,
+    parentMap: Map<string, WithLookup<Doc>>,
+    associations: AssociationQuery[]
+  ): Promise<void> {
+    for (const association of associations) {
+      const [assocId, dir, nested] = association
+      const isReverse = dir === -1
+      const keyA = isReverse ? 'docB' : 'docA'
+      const keyB = isReverse ? 'docA' : 'docB'
+
+      const assoc = this.modelDb.findObject(assocId)
+      if (assoc === undefined) {
+        continue
+      }
+      const _class = isReverse ? assoc.classA : assoc.classB
+      const domain = this.hierarchy.findDomain(_class)
+      if (domain === undefined) continue
+      const tagetDomain = translateDomain(domain)
+
+      const vars = new ValuesVariables()
+
+      const wsId = vars.add(this.workspaceId, '::uuid')
+      const parentIds = vars.add(Array.from(parentMap.keys()), '::text[]')
+      const assocIdVar = vars.add(assocId)
+
+      const rows = await this.mgr.retry(ctx.id, this.mgrId, async (connection) => {
+        return await connection.execute(
+          `
+          SELECT assoc.*, r."${keyA}" as parent_id
+          FROM ${tagetDomain} AS assoc
+          JOIN ${translateDomain(DOMAIN_RELATION)} AS r
+            ON r."${keyB}" = assoc."_id"
+          WHERE r."${keyA}" = ANY(${parentIds})
+            AND r.association = ${assocIdVar}
+            AND assoc."workspaceId" = ${wsId}
+          `,
+          vars.getValues()
+        )
+      })
+
+      const key = `${assocId}_${!isReverse ? 'b' : 'a'}`
+
+      const nextParentMap = new Map<string, WithLookup<Doc>>()
+      for (const row of rows) {
+        const parentId = row.parent_id
+        const parsed = parseDoc(row, getSchema(row._class))
+
+        const parent = parentMap.get(parentId)
+        if (parent === undefined) continue
+
+        if (parent.$associations === undefined) {
+          parent.$associations = {}
+        }
+
+        if (parent.$associations[key] === undefined) parent.$associations[key] = []
+        parent.$associations[key].push(parsed)
+        nextParentMap.set(parsed._id, parsed)
+      }
+
+      if (nested !== undefined && nested.length > 0 && nextParentMap.size > 0) {
+        await this.fetchAssociations(ctx, nextParentMap, nested)
+      }
+    }
+  }
+
+  private parseLookupColumns (
+    row: any,
+    simpleJoins: JoinProps[],
+    reverseJoins: JoinProps[],
+    lookup: Record<string, any>,
+    domain: string
+  ): void {
+    const schema = getSchema(domain)
+    let joinIndex: number | undefined
+    let skip = false
+
+    for (const column in row) {
+      if (column.startsWith('reverse_lookup_')) {
+        const join = reverseJoins.find((j) => j.toAlias.toLowerCase() === column)
+        if (join === undefined || row[column] == null) continue
+
+        const parsed = row[column].map((p: any) => parseDoc(p, schema))
+
+        const res = this.getLookupValue(join.path, lookup, false)
+        if (res === undefined) continue
+        const { obj, key } = res
+        obj[key] = parsed
+
+        continue
+      }
+
+      if (column.startsWith('lookup_')) {
+        const keys = column.split('_')
+        let key = keys[keys.length - 1]
+        if (keys[keys.length - 2] === '') {
+          key = '_' + key
+        }
+
+        if (key === 'workspaceId') continue
+
+        if (key === '_id') {
+          joinIndex = joinIndex === undefined ? 0 : joinIndex + 1
+          if (row[column] == null) {
+            skip = true
+            continue
+          }
+          skip = false
+        }
+
+        if (skip) continue
+
+        const join = simpleJoins[joinIndex ?? 0]
+        if (join === undefined) continue
+
+        const res = this.getLookupValue(join.path, lookup)
+        if (res === undefined) continue
+        const { obj, key: p } = res
+
+        if (key === 'data') {
+          obj[p] = { ...obj[p], ...row[column] }
+        } else if (key === 'createdOn' || key === 'modifiedOn') {
+          const val = parseInt(row[column])
+          obj[p][key] = Number.isNaN(val) ? null : val
+        } else if (key === '%hash%') {
+          // ignore
+        } else if (key === 'attachedTo' && row[column] === 'NULL') {
+          // ignore
+        } else {
+          obj[p][key] = row[column] === 'NULL' ? null : row[column]
+        }
+      }
+    }
   }
 
   private getLookupValue (
@@ -1399,36 +1446,6 @@ abstract class PostgresAdapterBase implements DbAdapter {
     return res
   }
 
-  getAssociationsProjections (vars: ValuesVariables, baseDomain: string, associations: AssociationQuery[]): string[] {
-    const res: string[] = []
-    for (const association of associations) {
-      const _id = escape(association[0])
-      const assoc = this.modelDb.findObject(_id)
-      if (assoc === undefined) {
-        continue
-      }
-      const isReverse = association[1] === -1
-      const _class = isReverse ? assoc.classA : assoc.classB
-      const domain = this.hierarchy.findDomain(_class)
-      if (domain === undefined) continue
-      const tagetDomain = translateDomain(domain)
-      const keyA = isReverse ? 'docB' : 'docA'
-      const keyB = isReverse ? 'docA' : 'docB'
-      const wsId = vars.add(this.workspaceId, '::uuid')
-      res.push(
-        `(SELECT jsonb_agg(assoc.*) 
-          FROM ${tagetDomain} AS assoc 
-          JOIN ${translateDomain(DOMAIN_RELATION)} as relation 
-          ON relation."${keyB}" = assoc."_id" 
-          AND relation."workspaceId" = ${wsId}
-          WHERE relation."${keyA}" = ${translateDomain(baseDomain)}."_id" 
-          AND relation.association = '${_id}'
-          AND assoc."workspaceId" = ${wsId}) AS assoc_${tagetDomain}_${_id}`
-      )
-    }
-    return res
-  }
-
   @withContext('get-domain-hash')
   async getDomainHash (ctx: MeasureContext, domain: Domain): Promise<string> {
     return await calcHashHash(ctx, domain, this)
@@ -1472,9 +1489,6 @@ abstract class PostgresAdapterBase implements DbAdapter {
     for (const join of joins) {
       res.push(...this.getProjectionsAliases(vars, join))
     }
-    if (associations !== undefined) {
-      res.push(...this.getAssociationsProjections(vars, baseDomain, associations))
-    }
     return res.join(', ')
   }
 

foundations/server/packages/postgres/src/utils.ts

@@ -28,6 +28,7 @@ import core, {
   type Projection,
   type Ref,
   systemAccountUuid,
+  type WithLookup,
   type WorkspaceUuid
 } from '@hcengineering/core'
 import { type DomainHelperOperations } from '@hcengineering/server-core'
@@ -473,6 +474,17 @@ export function parseDocWithProjection<T extends Doc> (
   return res
 }
 
+export function toWithLookup<T extends Doc> (doc: T): WithLookup<T> {
+  const res = doc as WithLookup<T>
+  if (res.$associations === undefined) {
+    res.$associations = {}
+  }
+  if (res.$lookup === undefined) {
+    res.$lookup = {}
+  }
+  return res
+}
+
 export function parseDoc<T extends Doc> (doc: DBDoc, schema: Schema): T {
   const { workspaceId, data, ...rest } = doc
   for (const key in rest) {

plugins/view-resources/src/components/RelationsEditor.svelte

@@ -13,9 +13,9 @@
 // limitations under the License.
 -->
 <script lang="ts">
-  import { createQuery, getClient } from '@hcengineering/presentation'
-  import core, { Association, Doc, Ref } from '@hcengineering/core'
+  import core, { Association, AssociationQuery, Doc } from '@hcengineering/core'
   import { getEmbeddedLabel } from '@hcengineering/platform'
+  import { createQuery, getClient } from '@hcengineering/presentation'
   import { createEventDispatcher } from 'svelte'
 
   import RelationEditor from './RelationEditor.svelte'
@@ -51,54 +51,45 @@
     getAssociations(object)
   })
 
-  let relationsA: Record<Ref<Association>, Doc[]> = {}
-  let relationsB: Record<Ref<Association>, Doc[]> = {}
-  let relationsALoaded = false
-  let relationsBLoaded = false
+  let relations: Record<string, Doc[]> = {}
+  let relationsLoaded = false
+  $: associations = [
+    ...associationsA.map((a) => [a._id, -1] as AssociationQuery),
+    ...associationsB.map((a) => [a._id, 1] as AssociationQuery)
+  ]
 
   const queryA = createQuery()
   $: queryA.query(
     object._class,
     { _id: object._id },
     (res) => {
-      relationsA = res?.[0]?.$associations ?? {}
-      relationsALoaded = true
+      relations = res?.[0]?.$associations ?? {}
+      relationsLoaded = true
     },
-    { associations: associationsA.map((a) => [a._id, -1]) }
+    { associations }
   )
 
-  const queryB = createQuery()
-  $: queryB.query(
-    object._class,
-    { _id: object._id },
-    (res) => {
-      relationsB = res?.[0]?.$associations ?? {}
-      relationsBLoaded = true
-    },
-    { associations: associationsB.map((a) => [a._id, 1]) }
-  )
-
-  $: if (relationsALoaded && relationsBLoaded) {
+  $: if (relationsLoaded) {
     dispatch('loaded')
   }
 </script>
 
-{#each associationsB as association (association._id)}
+{#each associationsB as association (`${association._id}_b`)}
   <RelationEditor
     {association}
     {object}
-    docs={relationsB[association._id] ?? []}
+    docs={relations[`${association._id}_b`] ?? []}
     {readonly}
     label={getEmbeddedLabel(association.nameB)}
     direction="B"
     {emptyKind}
   />
 {/each}
-{#each associationsA as association (association._id)}
+{#each associationsA as association (`${association._id}_a`)}
   <RelationEditor
     {association}
     {object}
-    docs={relationsA[association._id] ?? []}
+    docs={relations[`${association._id}_a`] ?? []}
     {readonly}
     label={getEmbeddedLabel(association.nameA)}
     direction="A"

plugins/view-resources/src/components/ViewletSetting.svelte

@@ -13,7 +13,7 @@
 // limitations under the License.
 -->
 <script lang="ts">
-  import core, { AnyAttribute, Association, Class, Doc, Ref, Type } from '@hcengineering/core'
+  import core, { AnyAttribute, Association, AssociationQuery, Class, Doc, Ref, Type } from '@hcengineering/core'
   import { Asset, getEmbeddedLabel, IntlString } from '@hcengineering/platform'
   import { createQuery, getAttributePresenterClass, getClient, hasResource } from '@hcengineering/presentation'
   import { DropdownLabelsIntl, Loading, resizeObserver } from '@hcengineering/ui'
@@ -234,23 +234,41 @@
     return false
   }
 
-  function processAssociation (association: Association, direction: 'A' | 'B', result: Config[]): void {
-    const associationValue = `$associations.${association._id}.${direction}`
-    const name = direction === 'A' ? association.nameA : association.nameB
-    const targetClass = direction === 'A' ? association.classA : association.classB
+  function getParentsString (parents: AssociationQuery[]): string {
+    return parents.map(([assocId, direction]) => `$associations.${assocId}_${direction === 1 ? 'a' : 'b'}`).join('.')
+  }
+
+  function processAssociation (
+    association: Association,
+    direction: 'a' | 'b',
+    result: Config[],
+    preference: ViewletPreference | undefined,
+    parents: AssociationQuery[]
+  ): void {
+    const associationName = `$associations.${association._id}_${direction}`
+    const resultName = parents.length > 0 ? `${getParentsString(parents)}.${associationName}` : associationName
+
+    const name = direction === 'a' ? association.nameA : association.nameB
+    const targetClass = direction === 'a' ? association.classA : association.classB
 
     if (name.trim().length === 0) return
+    const model = client.getModel()
 
-    for (const res of result) {
-      const key = typeof res.value === 'string' ? res.value : res.value?.key
-      if (key === associationValue) return
-    }
+    const resultLabels = parents
+      .map((r) => {
+        const assoc = model.findObject(r[0])
+        if (assoc === undefined) return ''
+        return r[1] === 1 ? assoc.nameA : assoc.nameB
+      })
+      .filter((it) => it.length > 0)
+    resultLabels.push(name)
+    const fullLabel = resultLabels.join(' › ')
 
     const clazz = hierarchy.getClass(targetClass)
     const newValue: AttributeConfig = {
       type: 'attribute',
-      value: associationValue,
-      label: getEmbeddedLabel(name),
+      value: resultName,
+      label: getEmbeddedLabel(fullLabel),
       enabled: false,
       _class: targetClass,
       icon: clazz.icon
@@ -259,6 +277,15 @@
     if (!isExist(result, newValue)) {
       result.push(newValue)
     }
+
+    if (preference === undefined) return
+    const exists = preference.config.find((p) => {
+      const key = typeof p === 'string' ? p : p.key
+      return key === resultName
+    })
+    if (exists) {
+      addAssociations(result, targetClass, preference, [...parents, [association._id, direction === 'a' ? 1 : -1]])
+    }
   }
 
   function getConfig (viewlet: Viewlet, preference: ViewletPreference | undefined): Config[] {
@@ -278,37 +305,44 @@
         })
       }
 
-      const ancestors = new Set(hierarchy.getAncestors(viewlet.attachTo))
-      const parent = hierarchy.getParentClass(viewlet.attachTo)
-      const parentMixins = hierarchy
-        .getDescendants(parent)
-        .map((p) => hierarchy.getClass(p))
-        .filter((p) => hierarchy.isMixin(p._id) && p.extends && ancestors.has(p.extends))
-
-      parentMixins.forEach((it) => {
-        hierarchy.getOwnAttributes(it._id).forEach((attr) => {
-          processAttribute(attr, result, true)
-        })
-      })
-
-      // Process associations
-
-      const allClasses = [...ancestors, ...parentMixins.map((it) => it._id)]
-
-      const associationsB = client.getModel().findAllSync(core.class.Association, { classA: { $in: allClasses } })
-      const associationsA = client.getModel().findAllSync(core.class.Association, { classB: { $in: allClasses } })
-
-      associationsB.forEach((a) => {
-        processAssociation(a, 'B', result)
-      })
-      associationsA.forEach((a) => {
-        processAssociation(a, 'A', result)
-      })
+      addAssociations(result, viewlet.attachTo, preference)
     }
 
     return preference === undefined ? result : setStatus(result, preference)
   }
 
+  function addAssociations (
+    result: Config[],
+    _class: Ref<Class<Doc>>,
+    preference: ViewletPreference | undefined,
+    parents: AssociationQuery[] = []
+  ): void {
+    const ancestors = new Set(hierarchy.getAncestors(_class))
+    const parent = hierarchy.getParentClass(_class)
+    const parentMixins = hierarchy
+      .getDescendants(parent)
+      .map((p) => hierarchy.getClass(p))
+      .filter((p) => hierarchy.isMixin(p._id) && p.extends && ancestors.has(p.extends))
+
+    parentMixins.forEach((it) => {
+      hierarchy.getOwnAttributes(it._id).forEach((attr) => {
+        processAttribute(attr, result, true)
+      })
+    })
+
+    const allClasses = [...ancestors, ...parentMixins.map((it) => it._id)]
+
+    const associationsB = client.getModel().findAllSync(core.class.Association, { classA: { $in: allClasses } })
+    const associationsA = client.getModel().findAllSync(core.class.Association, { classB: { $in: allClasses } })
+
+    associationsB.forEach((a) => {
+      processAssociation(a, 'b', result, preference, parents)
+    })
+    associationsA.forEach((a) => {
+      processAssociation(a, 'a', result, preference, parents)
+    })
+  }
+
   async function save (viewletId: Ref<Viewlet>, items: Array<Config | AttributeConfig>): Promise<void> {
     const configValues = items.filter(
       (p) =>

plugins/view-resources/src/utils.ts

@@ -508,17 +508,46 @@ function getKeyLookup<T extends Doc> (
 }
 
 export function buildConfigAssociation (config: Array<BuildModelKey | string>): AssociationQuery[] | undefined {
-  const res: AssociationQuery[] = []
+  const record: Record<string, any> = {}
   for (const key of config) {
     const k = typeof key === 'string' ? key : key.key
     if (k.startsWith('$associations')) {
-      const parts = k.split('.')
-      if (parts.length === 3) {
-        res.push([parts[1] as Ref<Association>, parts[2] === 'A' ? -1 : 1])
+      buildaAssociation(k, record)
+    }
+  }
+  return convertAssociationsRecord(record)
+}
+
+function buildaAssociation (stringKey: string, record: Record<string, any>): void {
+  const parts = stringKey.split('$associations.').filter((it) => it.length > 0)
+  let curr = record
+  for (let part of parts) {
+    if (part.endsWith('.')) {
+      part = part.slice(0, -1)
+    }
+    if (curr[part] === undefined) {
+      curr[part] = {}
+    }
+    curr = record[part]
+  }
+}
+
+function convertAssociationsRecord (record: Record<string, any>): AssociationQuery[] {
+  const res: AssociationQuery[] = []
+  for (const key of Object.keys(record)) {
+    const segments = key.split('_')
+    if (segments?.length >= 2) {
+      const nested = Object.keys(record[key]).length > 0 ? convertAssociationsRecord(record[key]) : undefined
+      if (nested !== undefined) {
+        const r: AssociationQuery = [segments[0] as Ref<Association>, segments[1] === 'a' ? -1 : 1, nested]
+        res.push(r)
+      } else {
+        const base: AssociationQuery = [segments[0] as Ref<Association>, segments[1] === 'a' ? -1 : 1]
+        res.push(base)
       }
     }
   }
-  return res.length > 0 ? res : undefined
+  return res
 }
 
 export function buildConfigLookup<T extends Doc> (
@@ -556,11 +585,11 @@ export async function buildModel (options: BuildModelOptions): Promise<Attribute
     .map(async (key) => {
       try {
         // Check if it is a mixin attribute configuration
+        if (key.key.startsWith('$associations')) {
+          return await getRelationPresenter(options.client, key)
+        }
         const pos = key.key.lastIndexOf('.')
         if (pos !== -1) {
-          if (key.key.startsWith('$associations')) {
-            return await getRelationPresenter(options.client, key)
-          }
           const mixinName = key.key.substring(0, pos) as Ref<Class<Doc>>
           if (!mixinName.includes('$lookup')) {
             const realKey = key.key.substring(pos + 1)
@@ -599,16 +628,17 @@ export async function buildModel (options: BuildModelOptions): Promise<Attribute
 
 async function getRelationPresenter (client: Client, key: BuildModelKey): Promise<AttributeModel> {
   const parts = key.key.split('.')
-  if (parts.length !== 3) {
+  if (parts.length < 2) {
     throw new Error('invalid relation key ' + key.key)
   }
-  const assocId = parts[1] as Ref<Association>
+  const fragments = parts[parts.length - 1].split('_')
+  const assocId = fragments[0] as Ref<Association>
   const assoc = client.getModel().findObject(assocId)
   if (assoc === undefined) {
     throw new Error('association not found for ' + assocId)
   }
-  const _class = parts[2] === 'A' ? assoc.classA : assoc.classB
-  const name = parts[2] === 'A' ? assoc.nameA : assoc.nameB
+  const _class = fragments[1] === 'a' ? assoc.classA : assoc.classB
+  const name = fragments[1] === 'a' ? assoc.nameA : assoc.nameB
 
   const hierarchy = client.getHierarchy()
 
@@ -622,7 +652,7 @@ async function getRelationPresenter (client: Client, key: BuildModelKey): Promis
   const presenter = await getResource(presenterMixin.presenter)
 
   return {
-    key: `${parts[0]}.${parts[1]}`,
+    key: key.key,
     sortingKey: '',
     _class,
     label: getEmbeddedLabel(name),