UBERF-6540: Fix isIndexable and clean wrong indexed documents (#5347)

server/core/src/fulltext.ts

@@ -39,10 +39,11 @@ import core, {
   docKey,
   isFullTextAttribute,
   isIndexedAttribute,
-  toFindResult
+  toFindResult,
+  isClassIndexable
 } from '@hcengineering/core'
 import { type FullTextIndexPipeline } from './indexer'
-import { createStateDoc, isClassIndexable } from './indexer/utils'
+import { createStateDoc } from './indexer/utils'
 import { getScoringConfig, mapSearchResultDoc } from './mapper'
 import { type StorageAdapter } from './storage'
 import type { FullTextAdapter, IndexedDoc, WithFind } from './types'

server/core/src/indexer/content.ts

@@ -14,6 +14,7 @@
 //
 
 import core, {
+  getFullTextIndexableAttributes,
   type Blob,
   type Class,
   type Doc,
@@ -28,13 +29,13 @@ import { type DbAdapter } from '../adapter'
 import { type StorageAdapter } from '../storage'
 import { type ContentTextAdapter, type IndexedDoc } from '../types'
 import {
+  contentStageId,
+  fieldStateId,
   type DocUpdateHandler,
   type FullTextPipeline,
-  type FullTextPipelineStage,
-  contentStageId,
-  fieldStateId
+  type FullTextPipelineStage
 } from './types'
-import { docKey, docUpdKey, getFullTextIndexableAttributes } from './utils'
+import { docKey, docUpdKey } from './utils'
 
 /**
  * @public

server/core/src/indexer/field.ts

@@ -23,7 +23,9 @@ import core, {
   type IndexStageState,
   type MeasureContext,
   type Ref,
-  type ServerStorage
+  type ServerStorage,
+  getFullTextIndexableAttributes,
+  getFullTextContext
 } from '@hcengineering/core'
 import { deepEqual } from 'fast-equals'
 import { type DbAdapter } from '../adapter'
@@ -41,8 +43,6 @@ import {
   docUpdKey,
   getContent,
   getCustomAttrKeys,
-  getFullTextContext,
-  getFullTextIndexableAttributes,
   isFullTextAttribute,
   loadIndexStageStage
 } from './utils'
@@ -250,6 +250,7 @@ export class IndexedFieldStage implements FullTextPipelineStage {
     }
   }
 
+  // Remove should be safe to missing class
   async remove (docs: DocIndexState[], pipeline: FullTextPipeline): Promise<void> {
     for (const doc of docs) {
       if (doc.attachedTo !== undefined) {
@@ -260,8 +261,8 @@ export class IndexedFieldStage implements FullTextPipelineStage {
           const { _class, attr, extra, docId } = extractDocKey(k)
 
           if (_class !== undefined && docId === undefined) {
-            const keyAttr = pipeline.hierarchy.getAttribute(_class, attr)
-            if (isFullTextAttribute(keyAttr)) {
+            const keyAttr = pipeline.hierarchy.findAttribute(_class, attr)
+            if (keyAttr !== undefined && isFullTextAttribute(keyAttr)) {
               ;(parentDocUpdate as any)[docUpdKey(attr, { _class, docId: doc._id, extra })] = null
             }
           }

server/core/src/indexer/fulltextPush.ts

@@ -26,7 +26,8 @@ import core, {
   type MeasureContext,
   type Ref,
   type ServerStorage,
-  type WorkspaceId
+  type WorkspaceId,
+  getFullTextContext
 } from '@hcengineering/core'
 import { jsonToText, markupToJSON } from '@hcengineering/text'
 import { type DbAdapter } from '../adapter'
@@ -41,7 +42,7 @@ import {
   type FullTextPipelineStage,
   fullTextPushStageId
 } from './types'
-import { collectPropagate, collectPropagateClasses, docKey, getFullTextContext, isCustomAttr } from './utils'
+import { collectPropagate, collectPropagateClasses, docKey, isCustomAttr } from './utils'
 
 /**
  * @public

server/core/src/indexer/indexer.ts

@@ -29,6 +29,7 @@ import core, {
   type WorkspaceId,
   _getOperator,
   docKey,
+  groupByArray,
   setObjectValue,
   toFindResult
 } from '@hcengineering/core'
@@ -539,6 +540,7 @@ export class FullTextIndexPipeline implements FullTextPipeline {
   }
 
   private async processRemove (): Promise<void> {
+    let total = 0
     while (true) {
       const result = await this.storage.findAll(
         this.metrics,
@@ -547,9 +549,7 @@ export class FullTextIndexPipeline implements FullTextPipeline {
           removed: true
         },
         {
-          sort: {
-            modifiedOn: 1
-          },
+          limit: 1000,
           projection: {
             _id: 1,
             stages: 1,
@@ -584,6 +584,12 @@ export class FullTextIndexPipeline implements FullTextPipeline {
       await this.flush(true)
       if (toRemoveIds.length > 0) {
         await this.storage.clean(this.metrics, DOMAIN_DOC_INDEX_STATE, toRemoveIds)
+        total += toRemoveIds.length
+        await this.metrics.info('indexer', {
+          _classes: Array.from(groupByArray(toIndex, (it) => it.objectClass).keys()),
+          total,
+          count: toRemoveIds.length
+        })
       } else {
         break
       }

server/core/src/indexer/summary.ts

@@ -26,7 +26,8 @@ import core, {
   isFullTextAttribute,
   type MeasureContext,
   type Ref,
-  type ServerStorage
+  type ServerStorage,
+  getFullTextContext
 } from '@hcengineering/core'
 import { translate } from '@hcengineering/platform'
 import { jsonToText, markupToJSON } from '@hcengineering/text'
@@ -39,13 +40,7 @@ import {
   type FullTextPipeline,
   type FullTextPipelineStage
 } from './types'
-import {
-  collectPropagate,
-  collectPropagateClasses,
-  getFullTextContext,
-  isCustomAttr,
-  loadIndexStageStage
-} from './utils'
+import { collectPropagate, collectPropagateClasses, isCustomAttr, loadIndexStageStage } from './utils'
 
 /**
  * @public

server/core/src/indexer/utils.ts

@@ -15,25 +15,15 @@
 
 import core, {
   type AnyAttribute,
-  type AttachedDoc,
   type Class,
-  ClassifierKind,
-  type Collection,
   type Data,
   type Doc,
   type DocIndexState,
-  DOMAIN_BLOB,
-  DOMAIN_DOC_INDEX_STATE,
-  DOMAIN_FULLTEXT_BLOB,
-  DOMAIN_MODEL,
-  DOMAIN_TRANSIENT,
-  DOMAIN_TX,
   type FullTextSearchContext,
   generateId,
+  getFullTextContext,
   type Hierarchy,
   type IndexStageState,
-  isFullTextAttribute,
-  isIndexedAttribute,
   type MeasureContext,
   type Obj,
   type Ref,
@@ -44,30 +34,6 @@ import { deepEqual } from 'fast-equals'
 import { type DbAdapter } from '../adapter'
 import plugin from '../plugin'
 import { type FullTextPipeline } from './types'
-/**
- * @public
- */
-export function getFullTextIndexableAttributes (hierarchy: Hierarchy, clazz: Ref<Class<Obj>>): AnyAttribute[] {
-  const allAttributes = hierarchy.getAllAttributes(clazz)
-  const result: AnyAttribute[] = []
-  for (const [, attr] of allAttributes) {
-    if (isFullTextAttribute(attr) || isIndexedAttribute(attr)) {
-      result.push(attr)
-    }
-  }
-
-  hierarchy
-    .getDescendants(clazz)
-    .filter((m) => hierarchy.getClass(m).kind === ClassifierKind.MIXIN)
-    .forEach((m) => {
-      for (const [, v] of hierarchy.getAllAttributes(m, clazz)) {
-        if (isFullTextAttribute(v) || isIndexedAttribute(v)) {
-          result.push(v)
-        }
-      }
-    })
-  return result
-}
 
 export { docKey, docUpdKey, extractDocKey, isFullTextAttribute } from '@hcengineering/core'
 export type { IndexKeyOptions } from '@hcengineering/core'
@@ -96,63 +62,6 @@ export function getContent (
   return attrs
 }
 
-/**
- * @public
- */
-export function isClassIndexable (hierarchy: Hierarchy, c: Ref<Class<Doc>>): boolean {
-  const indexed = hierarchy.getClassifierProp(c, 'class_indexed')
-  if (indexed !== undefined) {
-    return indexed as boolean
-  }
-  const domain = hierarchy.findDomain(c)
-  if (domain === undefined) {
-    hierarchy.setClassifierProp(c, 'class_indexed', false)
-    return false
-  }
-
-  if (
-    domain === DOMAIN_DOC_INDEX_STATE ||
-    domain === DOMAIN_TX ||
-    domain === DOMAIN_MODEL ||
-    domain === DOMAIN_BLOB ||
-    domain === DOMAIN_FULLTEXT_BLOB ||
-    domain === DOMAIN_TRANSIENT
-  ) {
-    hierarchy.setClassifierProp(c, 'class_indexed', false)
-    return false
-  }
-
-  const indexMixin = hierarchy.classHierarchyMixin(c, core.mixin.IndexConfiguration)
-  if (indexMixin?.searchDisabled !== undefined && indexMixin?.searchDisabled) {
-    hierarchy.setClassifierProp(c, 'class_indexed', false)
-    return false
-  }
-
-  const attrs = getFullTextIndexableAttributes(hierarchy, c)
-  for (const d of hierarchy.getDescendants(c)) {
-    if (hierarchy.isMixin(d)) {
-      attrs.push(...getFullTextIndexableAttributes(hierarchy, d))
-    }
-  }
-
-  let result = true
-
-  if (attrs.length === 0 && !(getFullTextContext(hierarchy, c)?.forceIndex ?? false)) {
-    result = false
-    // We need check if document has collections with indexable fields.
-    const attrs = hierarchy.getAllAttributes(c).values()
-    for (const attr of attrs) {
-      if (attr.type._class === core.class.Collection) {
-        if (isClassIndexable(hierarchy, (attr.type as Collection<AttachedDoc>).of)) {
-          result = true
-          break
-        }
-      }
-    }
-  }
-  hierarchy.setClassifierProp(c, 'class_indexed', result)
-  return result
-}
 /**
  * @public
  */
@@ -225,35 +134,6 @@ export async function loadIndexStageStage (
   return [result, state]
 }
 
-/**
- * @public
- */
-export function getFullTextContext (
-  hierarchy: Hierarchy,
-  objectClass: Ref<Class<Doc>>
-): Omit<FullTextSearchContext, keyof Class<Doc>> {
-  let objClass = hierarchy.getClass(objectClass)
-
-  while (true) {
-    if (hierarchy.hasMixin(objClass, core.mixin.FullTextSearchContext)) {
-      const ctx = hierarchy.as<Class<Doc>, FullTextSearchContext>(objClass, core.mixin.FullTextSearchContext)
-      if (ctx !== undefined) {
-        return ctx
-      }
-    }
-    if (objClass.extends === undefined) {
-      break
-    }
-    objClass = hierarchy.getClass(objClass.extends)
-  }
-  return {
-    fullTextSummary: false,
-    forceIndex: false,
-    propagate: [],
-    childProcessingAllowed: true
-  }
-}
-
 /**
  * @public
  */