// // Copyright © 2020, 2021 Anticrm Platform Contributors. // Copyright © 2021 Hardcore Engineering Inc. // // Licensed under the Eclipse Public License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. You may // obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // // See the License for the specific language governing permissions and // limitations under the License. // import core, { AnyAttribute, AttachedDoc, Class, ClassifierKind, Collection, Doc, DocumentQuery, FindOptions, FindResult, Hierarchy, IndexKind, MeasureContext, Obj, ObjQueryType, Ref, toFindResult, Tx, TxCollectionCUD, TxCreateDoc, TxMixin, TxProcessor, TxRemoveDoc, TxResult, TxUpdateDoc } from '@hcengineering/core' import type { FullTextAdapter, IndexedDoc, WithFind } from './types' /** * @public */ export class FullTextIndex implements WithFind { constructor ( private readonly hierarchy: Hierarchy, private readonly adapter: FullTextAdapter, private readonly dbStorage: WithFind, private readonly skipUpdateAttached: boolean ) {} protected async txRemoveDoc (ctx: MeasureContext, tx: TxRemoveDoc): Promise { // console.log('FullTextIndex.txRemoveDoc: Method not implemented.') await this.adapter.remove(tx.objectId) return {} } protected async txMixin (ctx: MeasureContext, tx: TxMixin): Promise { const attributes = this.getFullTextAttributes(tx.mixin) let result = {} if (attributes === undefined) return result const ops: any = tx.attributes const update: any = {} let shouldUpdate = false for (const attr of attributes) { if (ops[attr.name] !== undefined) { update[(tx.mixin as string) + '.' + attr.name] = ops[attr.name] shouldUpdate = true } } if (shouldUpdate) { result = await this.adapter.update(tx.objectId, update) if (!this.skipUpdateAttached) { await this.updateAttachedDocs(ctx, tx, update) } } return result } async tx (ctx: MeasureContext, tx: Tx): Promise { switch (tx._class) { case core.class.TxCreateDoc: return await this.txCreateDoc(ctx, tx as TxCreateDoc) case core.class.TxCollectionCUD: return await this.txCollectionCUD(ctx, tx as TxCollectionCUD) case core.class.TxUpdateDoc: return await this.txUpdateDoc(ctx, tx as TxUpdateDoc) case core.class.TxRemoveDoc: return await this.txRemoveDoc(ctx, tx as TxRemoveDoc) case core.class.TxMixin: return await this.txMixin(ctx, tx as TxMixin) case core.class.TxApplyIf: // Apply if processed on server return await Promise.resolve({}) } throw new Error('TxProcessor: unhandled transaction class: ' + tx._class) } protected txCollectionCUD (ctx: MeasureContext, tx: TxCollectionCUD): Promise { // We need update only create transactions to contain attached, attachedToClass. if (tx.tx._class === core.class.TxCreateDoc) { const createTx = tx.tx as TxCreateDoc const d: TxCreateDoc = { ...createTx, attributes: { ...createTx.attributes, attachedTo: tx.objectId, attachedToClass: tx.objectClass, collection: tx.collection } } return this.txCreateDoc(ctx, d) } return this.tx(ctx, tx.tx) } async findAll( ctx: MeasureContext, _class: Ref>, query: DocumentQuery, options?: FindOptions ): Promise> { console.log('search', query) const { _id, $search, ...mainQuery } = query if ($search === undefined) return toFindResult([]) const ids: Set> = new Set>() const baseClass = this.hierarchy.getBaseClass(_class) const classes = this.hierarchy.getDescendants(baseClass) const fullTextLimit = 10000 const docs = await this.adapter.search(classes, query, fullTextLimit) for (const doc of docs) { if (this.hierarchy.isDerived(doc._class, baseClass)) { ids.add(doc.id) } if (doc.attachedTo !== undefined) { if (doc.attachedToClass !== undefined && this.hierarchy.isDerived(doc.attachedToClass, baseClass)) { if (this.hierarchy.isDerived(doc.attachedToClass, baseClass)) { ids.add(doc.attachedTo) } } else { ids.add(doc.attachedTo) } } } const resultIds = Array.from(getResultIds(ids, _id)) return await this.dbStorage.findAll(ctx, _class, { _id: { $in: resultIds }, ...mainQuery }, options) } private getFullTextAttributes (clazz: Ref>, parentDoc?: Doc): AnyAttribute[] { const allAttributes = this.hierarchy.getAllAttributes(clazz) const result: AnyAttribute[] = [] for (const [, attr] of allAttributes) { if (isFullTextAttribute(attr)) { result.push(attr) } } // We also need to add all mixin attribues if parent is specified. if (parentDoc !== undefined) { this.hierarchy .getDescendants(clazz) .filter((m) => this.hierarchy.getClass(m).kind === ClassifierKind.MIXIN) .forEach((m) => { for (const [, v] of this.hierarchy.getAllAttributes(m, clazz)) { if (isFullTextAttribute(v) && this.hierarchy.hasMixin(parentDoc, m)) { result.push(v) } } }) } return result } protected async txCreateDoc (ctx: MeasureContext, tx: TxCreateDoc): Promise { const attributes = this.getFullTextAttributes(tx.objectClass) const doc = TxProcessor.createDoc2Doc(tx) let parentContent: Record = {} if (this.hierarchy.isDerived(doc._class, core.class.AttachedDoc)) { const attachedDoc = doc as AttachedDoc if (attachedDoc.attachedToClass !== undefined && attachedDoc.attachedTo !== undefined) { const parentDoc = ( await this.dbStorage.findAll(ctx, attachedDoc.attachedToClass, { _id: attachedDoc.attachedTo }, { limit: 1 }) )[0] if (parentDoc !== undefined) { const parentAttributes = this.getFullTextAttributes(parentDoc._class, parentDoc) if (parentAttributes.length > 0) { parentContent = this.getContent(parentAttributes, parentDoc) } } } } if (attributes.length === 0 && Object.keys(parentContent).length === 0) return {} let content = this.getContent(attributes, doc) content = { ...parentContent, ...content } const indexedDoc: IndexedDoc = { id: doc._id, _class: doc._class, modifiedBy: doc.modifiedBy, modifiedOn: doc.modifiedOn, space: doc.space, attachedTo: (doc as AttachedDoc).attachedTo, attachedToClass: (doc as AttachedDoc).attachedToClass, ...content } return await this.adapter.index(indexedDoc) } protected async txUpdateDoc (ctx: MeasureContext, tx: TxUpdateDoc): Promise { const attributes = this.getFullTextAttributes(tx.objectClass) let result = {} if (attributes.length === 0) return result const ops: any = tx.operations const update: any = {} let shouldUpdate = false for (const attr of attributes) { if (ops[attr.name] !== undefined) { update[attr.name] = ops[attr.name] shouldUpdate = true } } if (tx.operations.space !== undefined) { update.space = tx.operations.space shouldUpdate = true } if (shouldUpdate) { result = await this.adapter.update(tx.objectId, update) if (!this.skipUpdateAttached) { await this.updateAttachedDocs(ctx, tx, update) } } return result } private getContent (attributes: AnyAttribute[], doc: Doc): Record { const attrs: Record = {} for (const attr of attributes) { const isMixinAttr = this.hierarchy.isMixin(attr.attributeOf) if (isMixinAttr) { attrs[(attr.attributeOf as string) + '.' + attr.name] = (doc as any)[attr.attributeOf]?.[attr.name]?.toString() ?? '' } else { attrs[attr.name] = (doc as any)[attr.name]?.toString() ?? '' } } return attrs } private async updateAttachedDocs ( ctx: MeasureContext, tx: { objectId: Ref, objectClass: Ref> }, update: any ): Promise { const doc = (await this.dbStorage.findAll(ctx, tx.objectClass, { _id: tx.objectId }, { limit: 1 }))[0] if (doc === undefined) return const attributes = this.hierarchy.getAllAttributes(doc._class) // Find all mixin atttibutes for document. this.hierarchy .getDescendants(doc._class) .filter((m) => this.hierarchy.getClass(m).kind === ClassifierKind.MIXIN && this.hierarchy.hasMixin(doc, m)) .forEach((m) => { for (const [k, v] of this.hierarchy.getAllAttributes(m, doc._class)) { attributes.set(k, v) } }) for (const attribute of attributes.values()) { if (this.hierarchy.isDerived(attribute.type._class, core.class.Collection)) { const collection = attribute.type as Collection const allAttached = await this.dbStorage.findAll(ctx, collection.of, { attachedTo: tx.objectId }) if (allAttached.length === 0) continue const docUpdate: any = {} for (const key in update) { docUpdate[key] = update[key] } for (const attached of allAttached) { try { await this.adapter.update(attached._id, docUpdate) } catch (err: any) { if (((err.message as string) ?? '').includes('document_missing_exception:')) { console.error( 'missing document in elastic for', tx.objectId, 'attached', attached._id, 'collection', attached.collection ) // We have no document for attached object, so ignore for now. it is probable rebuild of elastic DB. continue } throw err } } } } } } function isFullTextAttribute (attr: AnyAttribute): boolean { return ( attr.index === IndexKind.FullText && (attr.type._class === core.class.TypeNumber || attr.type._class === core.class.TypeString || attr.type._class === core.class.TypeMarkup) ) } function getResultIds (ids: Set>, _id: ObjQueryType> | undefined): Set> { const result = new Set>() if (_id !== undefined) { if (typeof _id === 'string') { if (!ids.has(_id)) { return new Set() } else { result.add(_id) } } else if (_id.$in !== undefined) { for (const id of _id.$in) { if (ids.has(id)) { result.add(id) } } } else if (_id.$nin !== undefined) { for (const id of ids) { if (!_id.$nin.includes(id)) { result.add(id) } } } } else { return ids } return result }