Merge remote-tracking branch 'origin/develop' into staging

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

.vscode/launch.json

@@ -34,7 +34,7 @@
       "request": "launch",
       "args": ["src/__start.ts"],
       "env": {
-        "ELASTIC_URL": "http://localhost:9200",
+        "FULLTEXT_URL": "http://localhost:4700",
         "MONGO_URL": "mongodb://localhost:27017",
         "DB_URL": "mongodb://localhost:27017",
         // "DB_URL": "postgresql://postgres:example@localhost:5432",
@@ -65,6 +65,34 @@
       "cwd": "${workspaceRoot}/pods/server",
       "protocol": "inspector"
     },
+    {
+      "name": "Debug Fulltext",
+      "type": "node",
+      "request": "launch",
+      "args": ["src/index.ts"],
+      "env": {
+        "PORT": "4700",// For mongo
+        // "PORT": "4701", // for pg
+        "FULLTEXT_DB_URL": "http://localhost:9200",
+        "DB_URL": "mongodb://localhost:27017",
+        // "DB_URL": "postgresql://postgres:example@localhost:5432",
+        "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
+        "SERVER_SECRET": "secret",
+        "REKONI_URL": "http://localhost:4004",
+        "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
+        "REGION":"pg",
+        "ELASTIC_INDEX_NAME": "local_storage_index",
+        "STATS_URL":"http://host.docker.internal:4900",
+        "ACCOUNTS_URL": "http://localhost:3000",
+      },
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
+      "runtimeVersion": "20",
+      "showAsyncStacks": true,
+      "outputCapture": "std",
+      "sourceMaps": true,
+      "cwd": "${workspaceRoot}/pods/fulltext",
+      "protocol": "inspector"
+    },
     {
       "name": "Debug Account",
       "type": "node",

common/config/rush/command-line.json

@@ -238,7 +238,7 @@
       "summary": "Build docker with platform",
       "description": "use to build all docker containers required for platform",
       "safeForSimultaneousRushProcesses": true,
-      "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats"
+      "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext"
     },        
     {
       "commandKind": "global",

common/config/rush/pnpm-lock.yaml

@@ -21,8 +21,8 @@ dependencies:
     specifier: ^4.20240729.0
     version: 4.20241004.0
   '@elastic/elasticsearch':
-    specifier: ^7.14.0
-    version: 7.17.13
+    specifier: ^7.17.14
+    version: 7.17.14
   '@electron/notarize':
     specifier: ^2.3.2
     version: 2.3.2
@@ -467,6 +467,9 @@ dependencies:
   '@rush-temp/model-server-drive':
     specifier: file:./projects/model-server-drive.tgz
     version: file:projects/model-server-drive.tgz
+  '@rush-temp/model-server-fulltext':
+    specifier: file:./projects/model-server-fulltext.tgz
+    version: file:projects/model-server-fulltext.tgz
   '@rush-temp/model-server-gmail':
     specifier: file:./projects/model-server-gmail.tgz
     version: file:projects/model-server-gmail.tgz
@@ -617,6 +620,9 @@ dependencies:
   '@rush-temp/pod-front':
     specifier: file:./projects/pod-front.tgz
     version: file:projects/pod-front.tgz
+  '@rush-temp/pod-fulltext':
+    specifier: file:./projects/pod-fulltext.tgz
+    version: file:projects/pod-fulltext.tgz
   '@rush-temp/pod-github':
     specifier: file:./projects/pod-github.tgz
     version: file:projects/pod-github.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-prosemirror@1.2.12)
@@ -812,6 +818,12 @@ dependencies:
   '@rush-temp/server-drive-resources':
     specifier: file:./projects/server-drive-resources.tgz
     version: file:projects/server-drive-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2)
+  '@rush-temp/server-fulltext':
+    specifier: file:./projects/server-fulltext.tgz
+    version: file:projects/server-fulltext.tgz(esbuild@0.20.1)(ts-node@10.9.2)
+  '@rush-temp/server-fulltext-resources':
+    specifier: file:./projects/server-fulltext-resources.tgz
+    version: file:projects/server-fulltext-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2)
   '@rush-temp/server-github':
     specifier: file:./projects/server-github.tgz
     version: file:projects/server-github.tgz(esbuild@0.20.1)(ts-node@10.9.2)
@@ -3071,11 +3083,11 @@ packages:
       '@elastic/ecs-helpers': 2.1.1
     dev: false
 
-  /@elastic/elasticsearch@7.17.13:
-    resolution: {integrity: sha512-GMXtFVqd3FgUlTtPL/GDc+3GhwvfZ0kSuegCvVVqb58kd+0I6U6u7PL8QFRLHtwzqLEBmYLdwr4PRkBAWKGlzA==}
+  /@elastic/elasticsearch@7.17.14:
+    resolution: {integrity: sha512-6uQ1pVXutwz1Krwooo67W+3K8BwH1ASMh1WoHTpomUzw8EXecXN5lHIJ9EPqTHuv1WqR2LKkSJyagcq0HYUJpg==}
     engines: {node: '>=12'}
     dependencies:
-      debug: 4.3.4
+      debug: 4.3.5
       hpagent: 0.1.2
       ms: 2.1.3
       secure-json-parse: 2.7.0
@@ -23193,12 +23205,12 @@ packages:
     dev: false
 
   file:projects/elastic.tgz(@types/node@20.11.19)(esbuild@0.20.1):
-    resolution: {integrity: sha512-pNrJ3byQAXmu2LIoiS7ph3F/dImZtZBe/tSV3ATcOLuPiPxtd3f+3rMK6Ajszw1mxvlVdXBiDtHcaL3iwdx5ZQ==, tarball: file:projects/elastic.tgz}
+    resolution: {integrity: sha512-WGiDJdr4on2TaKPfaKntlenxzUJ5uuxtibsd/ZkppUh0BghpFSH37C8WGps7kQRS4oqpb+mB3QhxVhxIz2IvGg==, tarball: file:projects/elastic.tgz}
     id: file:projects/elastic.tgz
     name: '@rush-temp/elastic'
     version: 0.0.0
     dependencies:
-      '@elastic/elasticsearch': 7.17.13
+      '@elastic/elasticsearch': 7.17.14
       '@types/jest': 29.5.12
       '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
       '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
@@ -23226,7 +23238,7 @@ packages:
     dev: false
 
   file:projects/front.tgz(esbuild@0.20.1):
-    resolution: {integrity: sha512-kbnGRdB+2sGaVlIVQlK7RJo8OwjxEgTaNzWOg+3CSzpLuJFGE29mFog3D7BB2Q2g7hkEip1B6SiXHR/leCVvVQ==, tarball: file:projects/front.tgz}
+    resolution: {integrity: sha512-Y14f9YJTEcruCyze5lu63lxO9ODS2tUcO4WvCVNnRlro0vT13XvmOw+eDiTvohHI2aHROX+NuJuIEtGlzyqLBw==, tarball: file:projects/front.tgz}
     id: file:projects/front.tgz
     name: '@rush-temp/front'
     version: 0.0.0
@@ -24355,7 +24367,7 @@ packages:
     dev: false
 
   file:projects/middleware.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2):
-    resolution: {integrity: sha512-+LHNVG0cNOCtW49PLe0rmRphnhY2+TLFzsqVmzNahdBTv3PHdSQ0YtW94U8eomL9gd6h1hWnMjNlfbWlFKGEwA==, tarball: file:projects/middleware.tgz}
+    resolution: {integrity: sha512-XcnDd4BMYa4b8U2G4S6Hr2OAIMGQFNwzs8/EYYaEp+dmEx9lSibJByGSfaeZaMkrbVxO6auouuC1rWuZnmiieg==, tarball: file:projects/middleware.tgz}
     id: file:projects/middleware.tgz
     name: '@rush-temp/middleware'
     version: 0.0.0
@@ -24456,7 +24468,7 @@ packages:
     dev: false
 
   file:projects/model-all.tgz:
-    resolution: {integrity: sha512-FHdenDxGsdlbQ6948ecdTtmUq5jHHwbY/ZXllcSYgV5WoGxFG/kJvPH6mqrh2y5MMNDIkcQTkf2J65mAxxdL8w==, tarball: file:projects/model-all.tgz}
+    resolution: {integrity: sha512-lZSE4CGfPJX/cTsbqnSoz/ge496Cwl3zeRFHM2wrpgfCFyeCNp00xpRv62LPGe38dvHo/IIzy+Uu0RSVPHUOPw==, tarball: file:projects/model-all.tgz}
     name: '@rush-temp/model-all'
     version: 0.0.0
     dependencies:
@@ -25212,6 +25224,24 @@ packages:
       - supports-color
     dev: false
 
+  file:projects/model-server-fulltext.tgz:
+    resolution: {integrity: sha512-qOs1G3E2P64j6xsyD1w2+ZPLclBeAyx1xmGEI19kneyA8IB8NNRcpI5GIB9vIrYJ2lJUi0jyI+sHkLoxmgtBaA==, tarball: file:projects/model-server-fulltext.tgz}
+    name: '@rush-temp/model-server-fulltext'
+    version: 0.0.0
+    dependencies:
+      '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2)
+      '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
+      eslint: 8.56.0
+      eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2)
+      eslint-plugin-import: 2.29.1(eslint@8.56.0)
+      eslint-plugin-n: 15.7.0(eslint@8.56.0)
+      eslint-plugin-promise: 6.1.1(eslint@8.56.0)
+      prettier: 3.2.5
+      typescript: 5.6.2
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   file:projects/model-server-gmail.tgz:
     resolution: {integrity: sha512-vdwW7KKXUWLji9gEb2uEGwbjyJfeQr2g4G/vhZH15sy87rI5f7+M7x5p7zZq5IOF9dk6YArAbILD/yl7u+E3Xg==, tarball: file:projects/model-server-gmail.tgz}
     name: '@rush-temp/model-server-gmail'
@@ -26539,7 +26569,7 @@ packages:
     dev: false
 
   file:projects/pod-front.tgz:
-    resolution: {integrity: sha512-YkIJf+SF6JuDvwC0ANFW/F/DLjC7M/5vm+aXzIK/CJUnojLex6nCeyDNzLqfl0E3oecLOKz13pRHsDzgNHYAGQ==, tarball: file:projects/pod-front.tgz}
+    resolution: {integrity: sha512-717gPfBibQBUuH6G8gN+SGt3xRVSHqMAc4pq2R+xmc4y0bb8pmROILpwF5pdLWzvIwtc0VeLKbk0fNxD+jeQ7A==, tarball: file:projects/pod-front.tgz}
     name: '@rush-temp/pod-front'
     version: 0.0.0
     dependencies:
@@ -26585,6 +26615,47 @@ packages:
       - supports-color
     dev: false
 
+  file:projects/pod-fulltext.tgz:
+    resolution: {integrity: sha512-TNpmIc39Ot7eD4avFPov9XJHNfcOjSAVFKtWlkkeWSBLte8BdQ2zYC2JZ3+Tb8Dzu1nX01UQm5WvRVWJY78R3A==, tarball: file:projects/pod-fulltext.tgz}
+    name: '@rush-temp/pod-fulltext'
+    version: 0.0.0
+    dependencies:
+      '@koa/cors': 5.0.0
+      '@types/jest': 29.5.12
+      '@types/koa': 2.15.0
+      '@types/koa-bodyparser': 4.3.12
+      '@types/koa-router': 7.4.8
+      '@types/koa__cors': 5.0.0
+      '@types/node': 20.11.19
+      '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2)
+      '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
+      cross-env: 7.0.3
+      dotenv: 16.0.3
+      esbuild: 0.20.1
+      eslint: 8.56.0
+      eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2)
+      eslint-plugin-import: 2.29.1(eslint@8.56.0)
+      eslint-plugin-n: 15.7.0(eslint@8.56.0)
+      eslint-plugin-promise: 6.1.1(eslint@8.56.0)
+      jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
+      koa: 2.15.3
+      koa-bodyparser: 4.4.1
+      koa-router: 12.0.1
+      prettier: 3.2.5
+      ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.6.2)
+      ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.6.2)
+      typescript: 5.6.2
+    transitivePeerDependencies:
+      - '@babel/core'
+      - '@jest/types'
+      - '@swc/core'
+      - '@swc/wasm'
+      - babel-jest
+      - babel-plugin-macros
+      - node-notifier
+      - supports-color
+    dev: false
+
   file:projects/pod-github.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-prosemirror@1.2.12):
     resolution: {integrity: sha512-jSnbzFjyWZA1Hnjtyotd09jvLvQ+oGX0V4UOaXqZfyxM65JFeoit/DSrKlQTWJjSCbw9y6S90CYZN+pK3h8InA==, tarball: file:projects/pod-github.tgz}
     id: file:projects/pod-github.tgz
@@ -26825,7 +26896,7 @@ packages:
     dev: false
 
   file:projects/pod-server.tgz(utf-8-validate@6.0.4):
-    resolution: {integrity: sha512-boGyT8HkEJkWqcXXAAZ869HB68/GpHcMTUBZlhkUS9oztHADMfpAbGQ+yVVvypye4e9nsYTF1LhpsDrToOtPYw==, tarball: file:projects/pod-server.tgz}
+    resolution: {integrity: sha512-BV8NwL5NYGZtthvNwCy8vAMyZNBjbT8FmzRMaKq40wTer+rQ7T0xDrl96WBsIVsyv60knHY31oXjYkjE4P3M6A==, tarball: file:projects/pod-server.tgz}
     id: file:projects/pod-server.tgz
     name: '@rush-temp/pod-server'
     version: 0.0.0
@@ -28443,7 +28514,7 @@ packages:
     dev: false
 
   file:projects/server-backup.tgz(esbuild@0.20.1)(ts-node@10.9.2):
-    resolution: {integrity: sha512-oDyT5+30r1kpzb+Cx/HTsY1f4fRjjtSy7n13JjzYPmkn37nVcHYb1X1RImHxhoiPehYWdftTsE5v5l2on0mOPg==, tarball: file:projects/server-backup.tgz}
+    resolution: {integrity: sha512-e0MNgA1LeSOikxK3b6X/HMuzpyQ4MjAoJtHi84X28nxD+49sEy1NctCTptyqz2qWWicAZAghKV46Qi9N9+RnEw==, tarball: file:projects/server-backup.tgz}
     id: file:projects/server-backup.tgz
     name: '@rush-temp/server-backup'
     version: 0.0.0
@@ -28977,6 +29048,66 @@ packages:
       - ts-node
     dev: false
 
+  file:projects/server-fulltext-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2):
+    resolution: {integrity: sha512-NebNMPNgIeG8N0qA5Q4ROcs5Jn/dmE0WX92uxw/qVUO/k8DLMxRsrFvyKrrMGtYgeivbxJPe6PEd3FijUAMfiw==, tarball: file:projects/server-fulltext-resources.tgz}
+    id: file:projects/server-fulltext-resources.tgz
+    name: '@rush-temp/server-fulltext-resources'
+    version: 0.0.0
+    dependencies:
+      '@types/jest': 29.5.12
+      '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2)
+      '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
+      eslint: 8.56.0
+      eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2)
+      eslint-plugin-import: 2.29.1(eslint@8.56.0)
+      eslint-plugin-n: 15.7.0(eslint@8.56.0)
+      eslint-plugin-promise: 6.1.1(eslint@8.56.0)
+      jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
+      prettier: 3.2.5
+      ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.6.2)
+      typescript: 5.6.2
+    transitivePeerDependencies:
+      - '@babel/core'
+      - '@jest/types'
+      - '@types/node'
+      - babel-jest
+      - babel-plugin-macros
+      - esbuild
+      - node-notifier
+      - supports-color
+      - ts-node
+    dev: false
+
+  file:projects/server-fulltext.tgz(esbuild@0.20.1)(ts-node@10.9.2):
+    resolution: {integrity: sha512-QYDVer6XKHqKih+CXofNkysDl9TafFIlN5sw4GdJjLT3/AJt7L2FeCKMuPgMtYQhihj2kodtBfNUbnSeLA8qJw==, tarball: file:projects/server-fulltext.tgz}
+    id: file:projects/server-fulltext.tgz
+    name: '@rush-temp/server-fulltext'
+    version: 0.0.0
+    dependencies:
+      '@types/jest': 29.5.12
+      '@types/node': 20.11.19
+      '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2)
+      '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
+      eslint: 8.56.0
+      eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2)
+      eslint-plugin-import: 2.29.1(eslint@8.56.0)
+      eslint-plugin-n: 15.7.0(eslint@8.56.0)
+      eslint-plugin-promise: 6.1.1(eslint@8.56.0)
+      jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
+      prettier: 3.2.5
+      ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.6.2)
+      typescript: 5.6.2
+    transitivePeerDependencies:
+      - '@babel/core'
+      - '@jest/types'
+      - babel-jest
+      - babel-plugin-macros
+      - esbuild
+      - node-notifier
+      - supports-color
+      - ts-node
+    dev: false
+
   file:projects/server-github-model.tgz:
     resolution: {integrity: sha512-1qQ8IXhyHSiV5P+M7sGq0QHkkAG2n2DK3knDQJv/xzOQamJ6kcr2nlaqtAD48LfO1wVTtUkjwYwE9RbAEgpl5w==, tarball: file:projects/server-github-model.tgz}
     name: '@rush-temp/server-github-model'
@@ -29239,7 +29370,7 @@ packages:
     dev: false
 
   file:projects/server-indexer.tgz(esbuild@0.20.1)(ts-node@10.9.2):
-    resolution: {integrity: sha512-rMMlimV5wzW1ZPKeGdkdpeODEn0cyYDQJWti1GmagrteQed+ZH0DL51uPP0z7aioLG/ODxgFZKj8GWD+tMSrnA==, tarball: file:projects/server-indexer.tgz}
+    resolution: {integrity: sha512-H/iX6BGnUy04J35NoefNnWyoCqEd5RFve5SMXioIuA4tWqglHcAz8JqyPPLlfN+onrsh4Pz0tTVRV6lOZXSXWw==, tarball: file:projects/server-indexer.tgz}
     id: file:projects/server-indexer.tgz
     name: '@rush-temp/server-indexer'
     version: 0.0.0
@@ -29520,7 +29651,7 @@ packages:
     dev: false
 
   file:projects/server-pipeline.tgz:
-    resolution: {integrity: sha512-gaF/29tMBmt1rcHqrdJ87g7xHkb39eb8oZfDEVzyeK7qwf1vRjOYE0el4Pp/5OvkBrZSQGEwKgfUK3lHI3iLJQ==, tarball: file:projects/server-pipeline.tgz}
+    resolution: {integrity: sha512-VCAeDWNsMsyk2flDq7dhTEuS204IxDIMmB8t44NfPWwciiA/9dwla/bq1Fewnmb/28XS1xdw0UuE+X+qX0AHGA==, tarball: file:projects/server-pipeline.tgz}
     name: '@rush-temp/server-pipeline'
     version: 0.0.0
     dependencies:
@@ -31515,12 +31646,12 @@ packages:
     dev: false
 
   file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4):
-    resolution: {integrity: sha512-Xiug9crD6dZ0BSoLCOQVEQE05/LitZ2QEPOeqZP0k9K31mffLqyYbCZfbRMvWMzBoFolNR8yUC38wIPvkRal1g==, tarball: file:projects/tool.tgz}
+    resolution: {integrity: sha512-VMtn5vIgitHobfk3a7XXFR5cr1nCbF/KasUqkn+o+EBDB3H+1n9krwH41lyS9KRHDe6HelYQVcBAPbCn87oPcA==, tarball: file:projects/tool.tgz}
     id: file:projects/tool.tgz
     name: '@rush-temp/tool'
     version: 0.0.0
     dependencies:
-      '@elastic/elasticsearch': 7.17.13
+      '@elastic/elasticsearch': 7.17.14
       '@types/jest': 29.5.12
       '@types/mime-types': 2.1.4
       '@types/minio': 7.0.18

dev/docker-compose.yaml

@@ -99,8 +99,6 @@ services:
       - "host.docker.internal:host-gateway"
     ports:
       - 4900:4900
-    volumes:
-      - ./branding.json:/var/cfg/branding.json
     environment:
       - PORT=4900
       - SERVER_SECRET=secret
@@ -222,7 +220,6 @@ services:
       - "host.docker.internal:host-gateway"
     links:
       - mongodb
-      - elastic
       - minio
       - rekoni
       - account
@@ -240,21 +237,19 @@ services:
       - SERVER_SECRET=secret
       - ENABLE_COMPRESSION=true
       - STATS_URL=http://host.docker.internal:4900
-      - ELASTIC_URL=http://host.docker.internal:9200
+      - FULLTEXT_URL=http://host.docker.internal:4700
       # - DB_URL=postgresql://postgres:example@postgres:5432
       - DB_URL=${MONGO_URL}
       - MONGO_URL=${MONGO_URL}
       - 'MONGO_OPTIONS={"appName": "transactor", "maxPoolSize": 10}'
       - METRICS_CONSOLE=false
       - METRICS_FILE=metrics.txt
-      - STORAGE_CONFIG=${STORAGE_CONFIG}
-      - REKONI_URL=http://host.docker.internal:4004
+      - STORAGE_CONFIG=${STORAGE_CONFIG}      
       - FRONT_URL=http://host.docker.internal:8087
       # - APM_SERVER_URL=http://apm-server:8200
       - SES_URL=''
       - ACCOUNTS_URL=http://host.docker.internal:3000
-      - LAST_NAME_FIRST=true
-      - ELASTIC_INDEX_NAME=local_storage_index
+      - LAST_NAME_FIRST=true      
       - BRANDING_PATH=/var/cfg/branding.json
       - SUPPORT_WORKSPACE=support
       - AI_BOT_URL=http://host.docker.internal:4010
@@ -265,7 +260,6 @@ services:
       - "host.docker.internal:host-gateway"
     links:
       - postgres
-      - elastic
       - minio
       - rekoni
       - account
@@ -282,7 +276,7 @@ services:
       - SERVER_PORT=3331
       - SERVER_SECRET=secret
       - ENABLE_COMPRESSION=true
-      - ELASTIC_URL=http://host.docker.internal:9200
+      - FULLTEXT_URL=http://host.docker.internal:4701
       - STATS_URL=http://host.docker.internal:4900
       - DB_URL=postgresql://postgres:example@postgres:5432
       - MONGO_URL=${MONGO_URL}
@@ -290,13 +284,11 @@ services:
       - METRICS_CONSOLE=false
       - METRICS_FILE=metrics.txt
       - STORAGE_CONFIG=${STORAGE_CONFIG}
-      - REKONI_URL=http://host.docker.internal:4004
       - FRONT_URL=http://host.docker.internal:8087
       # - APM_SERVER_URL=http://apm-server:8200
       - SES_URL=''
       - ACCOUNTS_URL=http://host.docker.internal:3000
-      - LAST_NAME_FIRST=true
-      - ELASTIC_INDEX_NAME=local_storage_index
+      - LAST_NAME_FIRST=true      
       - BRANDING_PATH=/var/cfg/branding.json      
     restart: unless-stopped    
   rekoni:
@@ -308,6 +300,53 @@ services:
       resources:
         limits:
           memory: 1024M
+  fulltext:
+    image: hardcoreeng/fulltext
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    restart: unless-stopped
+    links:
+      - elastic
+      - mongodb
+    ports:
+      - 4700:4700
+    environment:
+      - SERVER_SECRET=secret
+      - DB_URL=${MONGO_URL}
+      - FULLTEXT_DB_URL=http://host.docker.internal:9200
+      - ELASTIC_INDEX_NAME=local_storage_index
+      - STORAGE_CONFIG=${STORAGE_CONFIG}
+      - STATS_URL=http://host.docker.internal:4900
+      - REKONI_URL=http://host.docker.internal:4004
+      - ACCOUNTS_URL=http://host.docker.internal:3000
+    deploy:
+      resources:
+        limits:
+          memory: 300M
+  fulltext_pg:
+    image: hardcoreeng/fulltext
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    restart: unless-stopped
+    links:
+      - elastic
+      - postgres    
+    ports:
+      - 4701:4701
+    environment:
+      - PORT=4701
+      - SERVER_SECRET=secret
+      - DB_URL=postgresql://postgres:example@postgres:5432
+      - FULLTEXT_DB_URL=http://host.docker.internal:9200
+      - ELASTIC_INDEX_NAME=local_storage_index
+      - STORAGE_CONFIG=${STORAGE_CONFIG}
+      - STATS_URL=http://host.docker.internal:4900
+      - REKONI_URL=http://host.docker.internal:4004
+      - ACCOUNTS_URL=http://host.docker.internal:3000
+    deploy:
+      resources:
+        limits:
+          memory: 300M          
   print:
     image: hardcoreeng/print
     extra_hosts:

dev/tool/package.json

@@ -55,7 +55,7 @@
     "@types/jest": "^29.5.5"
   },
   "dependencies": {
-    "@elastic/elasticsearch": "^7.14.0",
+    "@elastic/elasticsearch": "^7.17.14",
     "@hcengineering/account": "^0.6.0",
     "@hcengineering/workspace-service": "^0.6.0",
     "@hcengineering/attachment": "^0.6.14",

dev/tool/src/clean.ts

@@ -1240,7 +1240,6 @@ async function updateId (
       const { _id, space, modifiedBy, modifiedOn, createdBy, createdOn, _class, ...data } = docIndexState
       await txop.createDoc(docIndexState._class, docIndexState.space, {
         ...data,
-        stages: {},
         removed: false
       })
       await txop.update(docIndexState, { removed: true, needIndex: true })

dev/tool/src/index.ts

@@ -115,7 +115,7 @@ import {
 } from './clean'
 import { changeConfiguration } from './configuration'
 import { moveAccountDbFromMongoToPG, moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db'
-import { fixJsonMarkup, migrateMarkup, restoreLostMarkup } from './markup'
+import { fixJsonMarkup, migrateMarkup } from './markup'
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
 import { fixAccountEmails, renameAccount } from './renameAccount'
 import { moveFiles, showLostFiles } from './storage'
@@ -1282,38 +1282,6 @@ export function devTool (
       })
     })
 
-  program.command('show-lost-markup <workspace>').action(async (workspace: string, cmd: any) => {
-    const { dbUrl } = prepareTools()
-    await withDatabase(dbUrl, async (db) => {
-      await withStorage(async (adapter) => {
-        try {
-          const workspaceId = getWorkspaceId(workspace)
-          const token = generateToken(systemAccountEmail, workspaceId)
-          const endpoint = await getTransactorEndpoint(token)
-          await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'show' })
-        } catch (err: any) {
-          console.error(err)
-        }
-      })
-    })
-  })
-
-  program.command('restore-lost-markup <workspace>').action(async (workspace: string, cmd: any) => {
-    const { dbUrl } = prepareTools()
-    await withDatabase(dbUrl, async (db) => {
-      await withStorage(async (adapter) => {
-        try {
-          const workspaceId = getWorkspaceId(workspace)
-          const token = generateToken(systemAccountEmail, workspaceId)
-          const endpoint = await getTransactorEndpoint(token)
-          await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'restore' })
-        } catch (err: any) {
-          console.error(err)
-        }
-      })
-    })
-  })
-
   program.command('fix-bw-workspace <workspace>').action(async (workspace: string) => {
     await withStorage(async (adapter) => {
       await fixMinioBW(toolCtx, getWorkspaceId(workspace), adapter)

dev/tool/src/markup.ts

@@ -3,28 +3,20 @@ import core, {
   type AnyAttribute,
   type Class,
   type Client as CoreClient,
-  type CollaborativeDoc,
   type Doc,
-  type DocIndexState,
   type Domain,
   type Hierarchy,
-  type Markup,
   type MeasureContext,
   type Ref,
-  type TxMixin,
-  type TxCreateDoc,
-  type TxUpdateDoc,
   type WorkspaceId,
-  RateLimiter,
   collaborativeDocParse,
   makeCollaborativeDoc,
-  SortingOrder,
-  TxProcessor
+  RateLimiter
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo'
 import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core'
 import { connect } from '@hcengineering/server-tool'
-import { isEmptyMarkup, jsonToText, markupToYDoc } from '@hcengineering/text'
+import { jsonToText, markupToYDoc } from '@hcengineering/text'
 import { type Db, type FindCursor, type MongoClient } from 'mongodb'
 
 export async function fixJsonMarkup (
@@ -213,124 +205,3 @@ async function processMigrateMarkupFor (
 
   console.log('processed', processed)
 }
-
-export async function restoreLostMarkup (
-  ctx: MeasureContext,
-  workspaceId: WorkspaceId,
-  transactorUrl: string,
-  storageAdapter: StorageAdapter,
-  { command }: { command: 'show' | 'restore' }
-): Promise<void> {
-  const connection = (await connect(transactorUrl, workspaceId, undefined, {
-    mode: 'backup'
-  })) as unknown as CoreClient
-
-  try {
-    const hierarchy = connection.getHierarchy()
-    const classes = hierarchy.getDescendants(core.class.Doc)
-
-    for (const _class of classes) {
-      const isAttachedDoc = hierarchy.isDerived(_class, core.class.AttachedDoc)
-
-      const attributes = hierarchy.getAllAttributes(_class)
-      const attrs = Array.from(attributes.values()).filter((p) => p.type._class === core.class.TypeCollaborativeDoc)
-
-      // ignore classes with no collaborative attributes
-      if (attrs.length === 0) continue
-
-      const docs = await connection.findAll(_class, { _class })
-      for (const doc of docs) {
-        for (const attr of attrs) {
-          const value = hierarchy.isMixin(attr.attributeOf)
-            ? ((doc as any)[attr.attributeOf]?.[attr.name] as CollaborativeDoc)
-            : ((doc as any)[attr.name] as CollaborativeDoc)
-
-          if (value == null || value === '') continue
-
-          const { documentId } = collaborativeDocParse(value)
-          const stat = await storageAdapter.stat(ctx, workspaceId, documentId)
-          if (stat !== undefined) continue
-
-          const query = isAttachedDoc
-            ? {
-                'tx.objectId': doc._id,
-                'tx._class': { $in: [core.class.TxCreateDoc, core.class.TxUpdateDoc] }
-              }
-            : {
-                objectId: doc._id
-              }
-
-          let restored = false
-
-          // try to restore by txes
-          // we need last tx that modified the attribute
-
-          const txes = await connection.findAll(isAttachedDoc ? core.class.TxCollectionCUD : core.class.TxCUD, query, {
-            sort: { modifiedOn: SortingOrder.Descending }
-          })
-          for (const tx of txes) {
-            const innerTx = TxProcessor.extractTx(tx)
-
-            let markup: string | undefined
-            if (innerTx._class === core.class.TxMixin) {
-              const mixinTx = innerTx as TxMixin<Doc, Doc>
-              markup = (mixinTx.attributes as any)[attr.name]
-            } else if (innerTx._class === core.class.TxCreateDoc) {
-              const createTx = innerTx as TxCreateDoc<Doc>
-              markup = (createTx.attributes as any)[attr.name]
-            } else if (innerTx._class === core.class.TxUpdateDoc) {
-              const updateTex = innerTx as TxUpdateDoc<Doc>
-              markup = (updateTex.operations as any)[attr.name]
-            } else {
-              continue
-            }
-
-            if (markup === undefined || !markup.startsWith('{')) continue
-            if (isEmptyMarkup(markup)) continue
-
-            console.log(doc._class, doc._id, attr.name, markup)
-            if (command === 'restore') {
-              const ydoc = markupToYDoc(markup, attr.name)
-              await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, value, ydoc)
-            }
-            restored = true
-            break
-          }
-
-          if (restored) continue
-
-          // try to restore by doc index state
-          const docIndexState = await connection.findOne(core.class.DocIndexState, {
-            _id: doc._id as Ref<DocIndexState>
-          })
-          if (docIndexState !== undefined) {
-            // document:class:Document%content#content#base64
-            const attrName = `${doc._class}%${attr.name}#content#base64`
-            const base64: string | undefined = docIndexState.attributes[attrName]
-            if (base64 !== undefined) {
-              const text = Buffer.from(base64, 'base64').toString()
-              if (text !== '') {
-                const markup: Markup = JSON.stringify({
-                  type: 'doc',
-                  content: [
-                    {
-                      type: 'paragraph',
-                      content: [{ type: 'text', text, marks: [] }]
-                    }
-                  ]
-                })
-                console.log(doc._class, doc._id, attr.name, markup)
-                if (command === 'restore') {
-                  const ydoc = markupToYDoc(markup, attr.name)
-                  await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, value, ydoc)
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  } finally {
-    await connection.close()
-  }
-}

dev/tool/src/workspace.ts

@@ -104,7 +104,7 @@ export async function recreateElastic (mongoUrl: string, workspaceId: WorkspaceI
     const db = getWorkspaceMongoDB(_client, workspaceId)
     await db
       .collection(DOMAIN_DOC_INDEX_STATE)
-      .updateMany({ _class: core.class.DocIndexState }, { $set: { stages: {}, needIndex: true } })
+      .updateMany({ _class: core.class.DocIndexState }, { $set: { needIndex: true } })
   } finally {
     client.close()
   }

models/activity/src/index.ts

@@ -31,8 +31,8 @@ import {
   type DocUpdateMessageViewletAttributesConfig,
   type IgnoreActivity,
   type Reaction,
-  type SavedMessage,
-  type ReplyProvider
+  type ReplyProvider,
+  type SavedMessage
 } from '@hcengineering/activity'
 import contact, { type Person } from '@hcengineering/contact'
 import core, {
@@ -67,19 +67,21 @@ import {
 } from '@hcengineering/model'
 import { TAttachedDoc, TClass, TDoc } from '@hcengineering/model-core'
 import preference, { TPreference } from '@hcengineering/model-preference'
+import presentation from '@hcengineering/model-presentation'
 import view from '@hcengineering/model-view'
 import type { Asset, IntlString, Resource } from '@hcengineering/platform'
 import { type AnyComponent } from '@hcengineering/ui/src/types'
-import presentation from '@hcengineering/model-presentation'
 
-import activity from './plugin'
 import { buildActions } from './actions'
 import { buildNotifications } from './notification'
+import activity from './plugin'
 
 export { activityId } from '@hcengineering/activity'
 export { activityOperation, migrateMessagesSpace } from './migration'
 
 export const DOMAIN_ACTIVITY = 'activity' as Domain
+export const DOMAIN_USER_MENTION = 'user_mention' as Domain
+export const DOMAIN_REACTION = 'reaction' as Domain
 
 @Mixin(activity.mixin.ActivityDoc, core.class.Class)
 export class TActivityDoc extends TClass implements ActivityDoc {
@@ -213,7 +215,7 @@ export class TActivityMessagesFilter extends TDoc implements ActivityMessagesFil
   filter!: Resource<(message: ActivityMessage, _class?: Ref<Doc>) => boolean>
 }
 
-@Model(activity.class.Reaction, core.class.AttachedDoc, DOMAIN_ACTIVITY)
+@Model(activity.class.Reaction, core.class.AttachedDoc, DOMAIN_REACTION)
 @UX(activity.string.Reactions)
 export class TReaction extends TAttachedDoc implements Reaction {
   @Prop(TypeRef(activity.class.ActivityMessage), core.string.AttachedTo)
@@ -248,7 +250,7 @@ export class TReplyProvider extends TDoc implements ReplyProvider {
   function!: Resource<(message: ActivityMessage) => Promise<void>>
 }
 
-@Model(activity.class.UserMentionInfo, core.class.AttachedDoc, DOMAIN_ACTIVITY)
+@Model(activity.class.UserMentionInfo, core.class.AttachedDoc, DOMAIN_USER_MENTION)
 export class TUserMentionInfo extends TAttachedDoc {
   user!: Ref<Person>
   content!: string

models/activity/src/migration.ts

@@ -14,6 +14,7 @@
 //
 
 import { type ActivityMessage, type DocUpdateMessage, type Reaction } from '@hcengineering/activity'
+import contact from '@hcengineering/contact'
 import core, { type Class, type Doc, type Domain, groupByArray, type Ref, type Space } from '@hcengineering/core'
 import {
   type MigrateOperation,
@@ -25,10 +26,8 @@ import {
   tryMigrate
 } from '@hcengineering/model'
 import { htmlToMarkup } from '@hcengineering/text'
-import contact from '@hcengineering/contact'
-
+import { activityId, DOMAIN_ACTIVITY, DOMAIN_REACTION, DOMAIN_USER_MENTION } from './index'
 import activity from './plugin'
-import { activityId, DOMAIN_ACTIVITY } from './index'
 
 const DOMAIN_CHUNTER = 'chunter' as Domain
 
@@ -223,6 +222,13 @@ export const activityOperation: MigrateOperation = {
         func: async (client: MigrationClient): Promise<void> => {
           await client.update(DOMAIN_ACTIVITY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
         }
+      },
+      {
+        state: 'move-reactions',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.move(DOMAIN_ACTIVITY, { _class: activity.class.Reaction }, DOMAIN_REACTION)
+          await client.move(DOMAIN_ACTIVITY, { _class: activity.class.UserMentionInfo }, DOMAIN_USER_MENTION)
+        }
       }
     ])
   },

models/all/package.json

@@ -109,6 +109,7 @@
     "@hcengineering/model-server-controlled-documents": "^0.1.0",
     "@hcengineering/model-analytics-collector": "^0.6.0",
     "@hcengineering/model-server-ai-bot": "^0.6.0",
-    "@hcengineering/model-ai-bot": "^0.6.0"
+    "@hcengineering/model-ai-bot": "^0.6.0",
+    "@hcengineering/model-server-fulltext": "^0.6.0"
   }
 }

models/all/src/index.ts

@@ -99,6 +99,7 @@ import {
   serverDocumentsId,
   createModel as serverDocumentsModel
 } from '@hcengineering/model-server-controlled-documents'
+import { serverFulltextId, createModel as serverFulltextModel } from '@hcengineering/model-server-fulltext'
 
 import { type Plugin } from '@hcengineering/platform'
 
@@ -434,7 +435,8 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[
     [serverProductsModel, serverProductsId],
     [serverTrainingModel, serverTrainingId],
     [serverDocumentsModel, serverDocumentsId],
-    [serverAiBotModel, serverAiBotId]
+    [serverAiBotModel, serverAiBotId],
+    [serverFulltextModel, serverFulltextId]
   ]
 
   for (const [b, id, config] of builders) {

models/attachment/src/migration.ts

@@ -19,7 +19,7 @@ import {
   type MigrationClient,
   type MigrationUpgradeClient
 } from '@hcengineering/model'
-import { attachmentId, DOMAIN_ATTACHMENT } from '.'
+import attachment, { attachmentId, DOMAIN_ATTACHMENT } from '.'
 
 export const attachmentOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {
@@ -29,6 +29,20 @@ export const attachmentOperation: MigrateOperation = {
         func: async (client: MigrationClient): Promise<void> => {
           await client.update(DOMAIN_ATTACHMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
         }
+      },
+      {
+        state: 'fix-attachedTo',
+        func: async (client: MigrationClient): Promise<void> => {
+          await client.update(
+            DOMAIN_ATTACHMENT,
+            { _class: attachment.class.Attachment, attachedToClass: 'chunter:class:Comment' },
+            {
+              $set: {
+                attachedToClass: 'chunter:class:ChatMessage'
+              }
+            }
+          )
+        }
       }
     ])
   },

models/bitrix/src/index.ts

@@ -145,4 +145,12 @@ export function createModel (builder: Builder): void {
       { modifiedOn: 1 }
     ]
   })
+  builder.mixin(bitrix.class.FieldMapping, core.class.Class, core.mixin.IndexConfiguration, {
+    indexes: [],
+    searchDisabled: true
+  })
+  builder.mixin(bitrix.class.EntityMapping, core.class.Class, core.mixin.IndexConfiguration, {
+    indexes: [],
+    searchDisabled: true
+  })
 }

models/chunter/src/index.ts

@@ -314,6 +314,16 @@ export function createModel (builder: Builder): void {
 
   defineActions(builder)
   defineNotifications(builder)
+
+  builder.mixin(chunter.class.InlineButton, core.class.Class, core.mixin.IndexConfiguration, {
+    indexes: [],
+    searchDisabled: true
+  })
+
+  builder.mixin(chunter.class.ChatSyncInfo, core.class.Class, core.mixin.IndexConfiguration, {
+    indexes: [],
+    searchDisabled: true
+  })
 }
 
 export default chunter

models/controlled-documents/src/index.ts

@@ -888,8 +888,7 @@ export function defineNotifications (builder: Builder): void {
 
 export function defineSearch (builder: Builder): void {
   builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
-    toClass: documents.class.Document,
-    parentPropagate: true
+    toClass: documents.class.Document
   })
 
   builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {

models/core/src/core.ts

@@ -66,7 +66,6 @@ import {
   TypeCollaborativeDoc,
   TypeFileSize,
   TypeIntlString,
-  TypeRecord,
   TypeRef,
   TypeString,
   TypeTimestamp,
@@ -330,19 +329,6 @@ export class TDocIndexState extends TDoc implements DocIndexState {
   @Hidden()
     objectClass!: Ref<Class<Doc>>
 
-  @Prop(TypeRef(core.class.Doc), core.string.AttachedTo)
-  @Index(IndexKind.Indexed)
-  @Hidden()
-    attachedTo?: Ref<Doc>
-
-  @Prop(TypeRef(core.class.Class), core.string.AttachedToClass)
-  @Index(IndexKind.Indexed)
-  @Hidden()
-    attachedToClass?: Ref<Class<Doc>>
-
-  // Indexable attributes of document.
-  attributes!: Record<string, any>
-
   @Prop(TypeBoolean(), getEmbeddedLabel('Removed'))
   @Hidden()
     removed!: boolean
@@ -350,16 +336,6 @@ export class TDocIndexState extends TDoc implements DocIndexState {
   @Prop(TypeBoolean(), getEmbeddedLabel('NeedIndexing'))
   @Hidden()
     needIndex!: boolean
-
-  // States for different stages
-  @Prop(TypeRecord(), getEmbeddedLabel('Stages'))
-  // @Index(IndexKind.Indexed)
-  @Hidden()
-    stages!: Record<string, boolean>
-
-  @Prop(TypeString(), getEmbeddedLabel('Generation'))
-  @Hidden()
-    generationId?: string
 }
 
 @Model(core.class.FullTextSearchContext, core.class.Doc, DOMAIN_MODEL)

models/core/src/index.ts

@@ -295,16 +295,20 @@ export function createModel (builder: Builder): void {
     domain: DOMAIN_DOC_INDEX_STATE,
     indexes: [
       {
-        keys: { needIndex: 1 }
+        keys: { needIndex: 1, objectClass: 1 }
       }
     ],
     disabled: [
       { attachedToClass: 1 },
       { stages: 1 },
-      { generationId: 1 },
       { space: 1 },
       { _class: 1 },
+      { needIndex: 1 },
+      { objectClass: 1 },
+      { _class: 1 },
+      { attachedTo: 1 },
       { modifiedBy: 1 },
+      { modifiedOn: 1 },
       { createdBy: 1 },
       { createdBy: -1 },
       { createdOn: -1 }
@@ -318,4 +322,17 @@ export function createModel (builder: Builder): void {
 
   definePermissions(builder)
   defineSpaceType(builder)
+
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: core.class.MigrationState,
+    forceIndex: false
+  })
+  builder.mixin(core.class.Configuration, core.class.Class, core.mixin.IndexConfiguration, {
+    indexes: [],
+    searchDisabled: true
+  })
+  builder.mixin(core.class.MigrationState, core.class.Class, core.mixin.IndexConfiguration, {
+    indexes: [],
+    searchDisabled: true
+  })
 }

models/core/src/migration.ts

@@ -15,7 +15,6 @@
 
 import { saveCollaborativeDoc } from '@hcengineering/collaboration'
 import core, {
-  DOMAIN_DOC_INDEX_STATE,
   DOMAIN_SPACE,
   DOMAIN_STATUS,
   DOMAIN_TX,
@@ -24,7 +23,6 @@ import core, {
   collaborativeDocParse,
   coreId,
   generateId,
-  isClassIndexable,
   makeCollaborativeDoc,
   type AnyAttribute,
   type Doc,
@@ -260,25 +258,6 @@ async function processMigrateContentFor (
 
 export const coreOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {
-    // We need to delete all documents in doc index state for missing classes
-    const allClasses = client.hierarchy.getDescendants(core.class.Doc)
-    const contexts = new Map(
-      client.model.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it])
-    )
-
-    const allIndexed = allClasses.filter((it) => isClassIndexable(client.hierarchy, it, contexts))
-
-    // Next remove all non indexed classes and missing classes as well.
-    await client.update(
-      DOMAIN_DOC_INDEX_STATE,
-      { objectClass: { $nin: allIndexed } },
-      {
-        $set: {
-          removed: true,
-          needIndex: true
-        }
-      }
-    )
     await tryMigrate(client, coreId, [
       {
         state: 'statuses-to-model',
@@ -296,12 +275,6 @@ export const coreOperation: MigrateOperation = {
         state: 'old-statuses-transactions',
         func: migrateStatusTransactions
       },
-      {
-        state: 'add-need-index',
-        func: async (client: MigrationClient) => {
-          await client.update(DOMAIN_DOC_INDEX_STATE, {}, { $set: { needIndex: true } })
-        }
-      },
       {
         state: 'collaborative-content-to-storage',
         func: migrateCollaborativeContentToStorage

models/gmail/src/index.ts

@@ -215,8 +215,7 @@ export function createModel (builder: Builder): void {
   )
 
   builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
-    toClass: gmail.class.Message,
-    parentPropagate: false
+    toClass: gmail.class.Message
   })
 
   builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {

models/love/src/index.ts

@@ -391,4 +391,19 @@ export function createModel (builder: Builder): void {
   builder.mixin(love.class.MeetingMinutes, core.class.Class, view.mixin.ObjectPresenter, {
     presenter: love.component.MeetingMinutesPresenter
   })
+
+  builder.mixin(love.class.Room, core.class.Class, core.mixin.IndexConfiguration, {
+    indexes: [],
+    searchDisabled: true
+  })
+
+  builder.mixin(love.class.Office, core.class.Class, core.mixin.IndexConfiguration, {
+    indexes: [],
+    searchDisabled: true
+  })
+
+  builder.mixin(love.class.Floor, core.class.Class, core.mixin.IndexConfiguration, {
+    indexes: [],
+    searchDisabled: true
+  })
 }

models/recruit/src/index.ts

@@ -1469,26 +1469,6 @@ export function createModel (builder: Builder): void {
     propagate: []
   })
 
-  createAction(builder, {
-    label: recruit.string.MatchVacancy,
-    icon: recruit.icon.Vacancy,
-    action: view.actionImpl.ShowPopup,
-    actionProps: {
-      component: recruit.component.MatchVacancy,
-      element: 'top',
-      fillProps: {
-        _objects: 'objects'
-      }
-    },
-    input: 'any',
-    category: recruit.category.Recruit,
-    target: recruit.mixin.Candidate,
-    context: {
-      mode: ['context', 'browser'],
-      group: 'create'
-    }
-  })
-
   builder.mixin(recruit.mixin.Candidate, core.class.Class, view.mixin.ObjectEditorFooter, {
     editor: tracker.component.RelatedIssuesSection,
     props: {

models/recruit/src/plugin.ts

@@ -126,9 +126,9 @@ export default mergeIds(recruitId, recruit, {
     VacancyTemplateEditor: '' as AnyComponent,
     ApplicationMatchPresenter: '' as AnyComponent,
 
-    MatchVacancy: '' as AnyComponent,
     NotificationApplicantPresenter: '' as AnyComponent,
-    VacancyEditor: '' as AnyComponent
+    VacancyEditor: '' as AnyComponent,
+    ApplicantNamePresenter: '' as AnyComponent
   },
   template: {
     DefaultVacancy: '' as Ref<ProjectType>

models/server-contact/src/index.ts

@@ -43,15 +43,15 @@ export function createModel (builder: Builder): void {
   })
 
   builder.mixin(contact.class.Contact, core.class.Class, serverCore.mixin.SearchPresenter, {
-    searchConfig: {
-      iconConfig: {
-        component: contact.component.AvatarRef,
-        props: ['_id']
-      },
-      title: { props: ['name'] }
+    iconConfig: {
+      component: contact.component.AvatarRef,
+      fields: [['_id']]
     },
-    getSearchTitle: {
-      name: serverContact.function.ContactNameProvider
+    title: {
+      component: contact.component.ContactNamePresenter, // Will present on UI.
+      fields: [['name']],
+      template: [['func', serverContact.function.ContactNameProvider, 'true']],
+      extraFields: [[['func', serverContact.function.ContactNameProvider, 'false']]]
     }
   })
 

models/server-controlled-documents/src/index.ts

@@ -58,13 +58,10 @@ export function createModel (builder: Builder): void {
   })
 
   builder.mixin(documents.class.DocumentMeta, core.class.Class, serverCore.mixin.SearchPresenter, {
-    searchConfig: {
-      iconConfig: {
-        component: documents.component.DocumentIcon,
-        props: []
-      },
-      title: 'title'
-    }
+    iconConfig: {
+      component: documents.component.DocumentIcon
+    },
+    title: [['title']]
   })
 
   builder.mixin(documents.class.ControlledDocument, core.class.Class, serverNotification.mixin.TextPresenter, {

models/server-core/src/index.ts

@@ -20,11 +20,10 @@ import type { Resource } from '@hcengineering/platform'
 
 import core, { DOMAIN_MODEL } from '@hcengineering/core'
 import type {
-  ClassSearchConfig,
+  FieldTemplateComponent,
   ObjectDDParticipant,
   ObjectDDParticipantFunc,
   SearchPresenter,
-  SearchPresenterFunc,
   Trigger,
   TriggerFunc
 } from '@hcengineering/server-core'
@@ -44,9 +43,7 @@ export class TObjectDDParticipant extends TClass implements ObjectDDParticipant
 
 @Mixin(serverCore.mixin.SearchPresenter, core.class.Class)
 export class TSearchPresenter extends TClass implements SearchPresenter {
-  searchConfig!: ClassSearchConfig
-  getSearchShortTitle!: SearchPresenterFunc
-  getSearchTitle!: SearchPresenterFunc
+  title!: FieldTemplateComponent
 }
 
 export function createModel (builder: Builder): void {

models/server-document/src/index.ts

@@ -28,13 +28,11 @@ export function createModel (builder: Builder): void {
   })
 
   builder.mixin(document.class.Document, core.class.Class, serverCore.mixin.SearchPresenter, {
-    searchConfig: {
-      iconConfig: {
-        component: document.component.DocumentSearchIcon,
-        props: ['icon', 'color']
-      },
-      title: 'title'
-    }
+    iconConfig: {
+      component: document.component.DocumentSearchIcon,
+      fields: [['icon'], ['color']]
+    },
+    title: [['title']]
   })
 
   builder.mixin<Class<Doc>, ObjectDDParticipant>(

models/server-drive/src/index.ts

@@ -41,16 +41,12 @@ export function createModel (builder: Builder): void {
   )
 
   builder.mixin(drive.class.File, core.class.Class, serverCore.mixin.SearchPresenter, {
-    searchConfig: {
-      icon: drive.icon.File,
-      title: 'title'
-    }
+    searchIcon: drive.icon.File,
+    title: [['title']]
   })
 
   builder.mixin(drive.class.Folder, core.class.Class, serverCore.mixin.SearchPresenter, {
-    searchConfig: {
-      icon: drive.icon.Folder,
-      title: 'title'
-    }
+    searchIcon: drive.icon.Folder,
+    title: [['title']]
   })
 }

models/server-fulltext/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+  extends: ['./node_modules/@hcengineering/platform-rig/profiles/model/eslint.config.json'],
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: './tsconfig.json'
+  }
+}

models/server-fulltext/.npmignore

@@ -0,0 +1,4 @@
+*
+!/lib/**
+!CHANGELOG.md
+/lib/**/__tests__/

models/server-fulltext/config/rig.json

@@ -0,0 +1,5 @@
+{  
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+  "rigPackageName": "@hcengineering/platform-rig",
+  "rigProfile": "model"
+}

models/server-fulltext/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@hcengineering/model-server-fulltext",
+  "version": "0.6.0",
+  "main": "lib/index.js",
+  "svelte": "src/index.ts",
+  "types": "types/index.d.ts",
+  "author": "Anticrm Platform Contributors",
+  "template": "@hcengineering/model-package",
+  "license": "EPL-2.0",
+  "scripts": {
+    "build": "compile",
+    "build:watch": "compile",
+    "format": "format src",
+    "_phase:build": "compile transpile src",
+    "_phase:format": "format src",
+    "_phase:validate": "compile validate"
+  },
+  "devDependencies": {
+    "@hcengineering/platform-rig": "^0.6.0",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint": "^8.54.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "prettier": "^3.1.0",
+    "typescript": "^5.3.3"
+  },
+  "dependencies": {
+    "@hcengineering/core": "^0.6.32",
+    "@hcengineering/model": "^0.6.11",
+    "@hcengineering/platform": "^0.6.11",
+    "@hcengineering/server-fulltext": "^0.6.0",
+    "@hcengineering/server-core": "^0.6.1"
+  }
+}

models/server-fulltext/src/index.ts

@@ -0,0 +1,27 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { type Builder } from '@hcengineering/model'
+
+import core from '@hcengineering/core'
+import serverCore from '@hcengineering/server-core'
+import serverFulltext from '@hcengineering/server-fulltext'
+export { serverFulltextId } from '@hcengineering/server-fulltext'
+export function createModel (builder: Builder): void {
+  builder.createDoc(serverCore.class.Trigger, core.space.Model, {
+    trigger: serverFulltext.trigger.OnChange,
+    arrays: true
+  })
+}

models/server-fulltext/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "./node_modules/@hcengineering/platform-rig/profiles/model/tsconfig.json",
+
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./lib",
+    "declarationDir": "./types",
+    "tsBuildInfoFile": ".build/build.tsbuildinfo"
+  }
+}

models/server-products/src/index.ts

@@ -14,12 +14,10 @@ export const serverProductsId = 'server-products' as Plugin
 
 export function createModel (builder: Builder): void {
   builder.mixin(products.class.Product, core.class.Class, serverCore.mixin.SearchPresenter, {
-    searchConfig: {
-      iconConfig: {
-        component: products.component.ProductSearchIcon,
-        props: ['icon', 'color']
-      },
-      title: 'name'
-    }
+    iconConfig: {
+      component: products.component.ProductSearchIcon,
+      fields: [['icon'], ['color']]
+    },
+    title: [['name']]
   })
 }

models/server-recruit/src/index.ts

@@ -15,15 +15,15 @@
 
 import { type Builder } from '@hcengineering/model'
 
+import contact from '@hcengineering/contact'
 import core from '@hcengineering/core'
 import recruit from '@hcengineering/model-recruit'
 import notification from '@hcengineering/notification'
 import serverCore from '@hcengineering/server-core'
 import serverNotification from '@hcengineering/server-notification'
 import serverRecruit from '@hcengineering/server-recruit'
-import serverContact from '@hcengineering/server-contact'
-import contact from '@hcengineering/contact'
 import serverView from '@hcengineering/server-view'
+import serverContact from '@hcengineering/server-contact'
 
 export { serverRecruitId } from '@hcengineering/server-recruit'
 
@@ -68,26 +68,32 @@ export function createModel (builder: Builder): void {
   })
 
   builder.mixin(recruit.class.Vacancy, core.class.Class, serverCore.mixin.SearchPresenter, {
-    searchConfig: {
-      icon: recruit.icon.Vacancy,
-      title: 'name'
-    }
+    searchIcon: recruit.icon.Vacancy,
+    title: [['name']]
   })
 
   builder.mixin(recruit.class.Applicant, core.class.Class, serverCore.mixin.SearchPresenter, {
-    searchConfig: {
-      iconConfig: {
-        component: contact.component.AvatarRef,
-        props: [{ _id: ['attachedTo'] }]
-      },
-      shortTitle: 'identifier',
-      title: {
-        tmpl: '{name} - {vacName}',
-        props: [{ _class: ['attachedTo', '_class'] }, { name: ['attachedTo', 'name'] }, { vacName: ['space', 'name'] }]
-      }
+    iconConfig: {
+      component: contact.component.AvatarRef,
+      fields: [['attachedTo']]
     },
-    getSearchTitle: {
-      name: serverContact.function.ContactNameProvider
+    shortTitle: [['identifier']],
+    title: {
+      component: recruit.component.ApplicantNamePresenter, // Will present on UI.
+      fields: [
+        ['parent', 'name'],
+        ['space', 'name']
+      ],
+      template: [
+        ['func', serverContact.function.ContactNameProvider, 'true'],
+        ['space', 'name']
+      ],
+      extraFields: [
+        [
+          ['func', serverContact.function.ContactNameProvider, 'false'],
+          ['space', 'name']
+        ]
+      ]
     }
   })
 

models/server-tracker/src/index.ts

@@ -13,6 +13,7 @@
 // limitations under the License.
 //
 
+import contact from '@hcengineering/contact'
 import core from '@hcengineering/core'
 import { type Builder } from '@hcengineering/model'
 import tracker from '@hcengineering/model-tracker'
@@ -20,7 +21,6 @@ import notification from '@hcengineering/notification'
 import serverCore from '@hcengineering/server-core'
 import serverNotification from '@hcengineering/server-notification'
 import serverTracker from '@hcengineering/server-tracker'
-import contact from '@hcengineering/contact'
 import serverView from '@hcengineering/server-view'
 
 export { serverTrackerId } from '@hcengineering/server-tracker'
@@ -43,14 +43,12 @@ export function createModel (builder: Builder): void {
   })
 
   builder.mixin(tracker.class.Issue, core.class.Class, serverCore.mixin.SearchPresenter, {
-    searchConfig: {
-      iconConfig: {
-        component: tracker.component.IssueSearchIcon,
-        props: ['status', 'space']
-      },
-      shortTitle: 'identifier',
-      title: 'title'
-    }
+    iconConfig: {
+      component: tracker.component.IssueSearchIcon,
+      fields: [['status'], ['space']]
+    },
+    shortTitle: [['identifier']],
+    title: [['title']]
   })
 
   builder.createDoc(serverCore.class.Trigger, core.space.Model, {

models/setting/src/index.ts

@@ -16,7 +16,7 @@
 import activity from '@hcengineering/activity'
 import contact from '@hcengineering/contact'
 import { AccountRole, DOMAIN_MODEL, type Account, type Blob, type Domain, type Ref } from '@hcengineering/core'
-import { Mixin, Model, type Builder, UX } from '@hcengineering/model'
+import { Mixin, Model, UX, type Builder } from '@hcengineering/model'
 import core, { TClass, TConfiguration, TDoc } from '@hcengineering/model-core'
 import view, { createAction } from '@hcengineering/model-view'
 import notification from '@hcengineering/notification'
@@ -28,12 +28,12 @@ import {
   type Integration,
   type IntegrationType,
   type InviteSettings,
-  type WorkspaceSetting,
   type SettingsCategory,
-  type UserMixin,
+  type SpaceTypeCreator,
   type SpaceTypeEditor,
   type SpaceTypeEditorSection,
-  type SpaceTypeCreator
+  type UserMixin,
+  type WorkspaceSetting
 } from '@hcengineering/setting'
 import templates from '@hcengineering/templates'
 import setting from './plugin'

models/telegram/src/index.ts

@@ -182,7 +182,6 @@ export function createModel (builder: Builder): void {
 
   builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
     toClass: telegram.class.Message,
-    parentPropagate: false,
     childProcessingAllowed: true
   })
 

models/view/src/index.ts

@@ -1209,6 +1209,10 @@ export function createModel (builder: Builder): void {
   builder.mixin(core.class.Space, core.class.Class, view.mixin.IgnoreActions, {
     actions: [view.action.Open, view.action.OpenInNewTab, view.action.Delete]
   })
+  builder.mixin(view.class.FilteredView, core.class.Class, core.mixin.IndexConfiguration, {
+    indexes: [],
+    searchDisabled: true
+  })
 }
 
 export default view

models/view/src/plugin.ts

@@ -78,7 +78,6 @@ export default mergeIds(viewId, view, {
     MarkupEditor: '' as AnyComponent,
     MarkupEditorPopup: '' as AnyComponent,
     ListView: '' as AnyComponent,
-    IndexedDocumentPreview: '' as AnyComponent,
     SpaceRefPresenter: '' as AnyComponent,
     EnumPresenter: '' as AnyComponent,
     StatusPresenter: '' as AnyComponent,

packages/core/src/classes.ts

@@ -360,16 +360,6 @@ export interface TransientConfiguration extends Class<Doc> {
  */
 export const DOMAIN_BLOB = 'blob' as Domain
 
-/**
- * Special domain to access s3 blob data.
- * @public
- */
-export const DOMAIN_FULLTEXT_BLOB = 'fulltext-blob' as Domain
-
-/**
- * Special domain to access s3 blob data.
- * @public
- */
 export const DOMAIN_DOC_INDEX_STATE = 'doc-index-state' as Domain
 
 // S P A C E
@@ -526,25 +516,8 @@ export function versionToString (version: Version | Data<Version>): string {
  */
 export interface DocIndexState extends Doc {
   objectClass: Ref<Class<Doc>>
-
-  attachedTo?: Ref<Doc>
-  attachedToClass?: Ref<Class<Doc>>
-
-  generationId?: string
-
-  // States for stages
-  stages: Record<string, boolean>
-
   needIndex: boolean
-
   removed: boolean
-
-  // Indexable attributes, including child ones.
-  attributes: Record<string, any>
-  mixins?: Ref<Class<Doc>>[]
-  // Full Summary
-  fullSummary?: string | null
-  shortSummary?: string | null
 }
 
 /**
@@ -594,9 +567,6 @@ export interface FullTextSearchContext extends Doc {
   // If defined, will propagate all document from child's based on class
   propagateClasses?: Ref<Class<Doc>>[]
 
-  // Do we need to propagate child value to parent one. Default(true)
-  parentPropagate?: boolean
-
   childProcessingAllowed?: boolean
 }
 

packages/core/src/server.ts

@@ -57,6 +57,8 @@ export interface SessionData {
 
   workspace: WorkspaceIdWithUrl
   branding: Branding | null
+
+  needWarmupFulltext?: boolean
 }
 
 /**
@@ -78,7 +80,12 @@ export interface LowLevelStorage {
   clean: (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]) => Promise<void>
 
   // Low level direct group API
-  groupBy: <T>(ctx: MeasureContext, domain: Domain, field: string) => Promise<Set<T>>
+  groupBy: <T, P extends Doc>(
+    ctx: MeasureContext,
+    domain: Domain,
+    field: string,
+    query?: DocumentQuery<P>
+  ) => Promise<Map<T, number>>
 
   // migrations
   rawFindAll: <T extends Doc>(domain: Domain, query: DocumentQuery<T>, options?: FindOptions<T>) => Promise<T[]>

packages/core/src/storage.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 
-import type { Asset } from '@hcengineering/platform'
+import type { Asset, Resource } from '@hcengineering/platform'
 
 import type { KeysByType } from 'simplytyped'
 import type { AttachedDoc, Class, Doc, Ref, Space } from './classes'
@@ -221,16 +221,23 @@ export interface SearchOptions {
   limit?: number
 }
 
+export interface SearchComponentWithProps {
+  component?: Resource<any>
+  props?: Record<string, string>
+}
+
 /**
  * @public
  */
 export interface SearchResultDoc {
   id: Ref<Doc>
+
   icon?: Asset
-  iconComponent?: string
-  iconProps?: Record<string, string>
+  iconComponent?: SearchComponentWithProps
   shortTitle?: string
+  shortTitleComponent?: SearchComponentWithProps
   title?: string
+  titleComponent?: SearchComponentWithProps
   score?: number
   doc: Pick<Doc, '_id' | '_class'>
 }

packages/core/src/utils.ts

@@ -26,7 +26,6 @@ import {
   Collection,
   Doc,
   DocData,
-  DocIndexState,
   DOMAIN_BLOB,
   DOMAIN_DOC_INDEX_STATE,
   DOMAIN_MODEL,
@@ -41,6 +40,7 @@ import {
   Space,
   TypedSpace,
   WorkspaceMode,
+  type Domain,
   type PluginConfiguration
 } from './classes'
 import core from './component'
@@ -165,72 +165,11 @@ export function isWorkspaceCreating (mode?: WorkspaceMode): boolean {
   return ['pending-creation', 'creating'].includes(mode)
 }
 
-const attributesPrefix = 'attributes.'
-
 /**
  * @public
  */
-export interface IndexKeyOptions {
-  _class?: Ref<Class<Obj>>
-  docId?: Ref<DocIndexState>
-  extra?: string[]
-  digest?: boolean
-}
-/**
- * @public
- */
-
-export function docUpdKey (name: string, opt?: IndexKeyOptions): string {
-  return attributesPrefix + docKey(name, opt)
-}
-/**
- * @public
- */
-export function docKey (name: string, opt?: IndexKeyOptions): string {
-  const extra = opt?.extra !== undefined && opt?.extra?.length > 0 ? `#${opt.extra?.join('#') ?? ''}` : ''
-  const digestName = opt?.digest === true ? name + '^digest' : name
-  return opt?._class === undefined ? digestName : `${opt?._class}%${digestName}${extra}`
-}
-
-/**
- * @public
- */
-export function extractDocKey (key: string): {
-  _class?: Ref<Class<Doc>>
-  attr: string
-  docId?: Ref<DocIndexState>
-  extra: string[]
-  digest: boolean
-} {
-  let k = key
-  if (k.startsWith(attributesPrefix)) {
-    k = k.slice(attributesPrefix.length)
-  }
-  let docId: Ref<DocIndexState> | undefined
-  let _class: Ref<Class<Doc>> | undefined
-  let attr = ''
-  const docSepPos = k.indexOf('|')
-  if (docSepPos !== -1) {
-    docId = k.substring(0, docSepPos).replace('_', '.') as Ref<DocIndexState>
-    k = k.substring(docSepPos + 1)
-  }
-  const clPos = k.indexOf('%')
-  if (clPos !== -1) {
-    _class = k.substring(0, clPos) as Ref<Class<Doc>>
-    attr = k.substring(clPos + 1)
-  } else {
-    attr = k
-  }
-  const extra = attr.split('#')
-  attr = extra.splice(0, 1)[0]
-  const digestPos = attr.indexOf('^digest')
-  let digest = false
-  if (digestPos !== -1) {
-    attr = attr.substring(0, digestPos)
-    digest = true
-  }
-
-  return { docId, attr, _class, extra, digest }
+export function docKey (name: string, _class?: Ref<Class<Doc>>): string {
+  return _class === undefined || _class !== core.class.Doc ? name : `${_class}%${name}`
 }
 
 /**
@@ -744,7 +683,9 @@ export function isClassIndexable (
     domain === DOMAIN_TX ||
     domain === DOMAIN_MODEL ||
     domain === DOMAIN_BLOB ||
+    domain === ('preference' as Domain) ||
     domain === DOMAIN_TRANSIENT ||
+    domain === ('settings' as Domain) ||
     domain === DOMAIN_BENCHMARK
   ) {
     hierarchy.setClassifierProp(c, 'class_indexed', false)

packages/platform/src/resource.ts

@@ -137,6 +137,30 @@ export function getResourceP<T> (resource: Resource<T>): T | Promise<T> {
   return cachedResource.get(resource) ?? getResource(resource)
 }
 
+/**
+ * @public
+ * @param resource -
+ * @returns
+ */
+export function getResourceC<T> (resource: Resource<T> | undefined, callback: (resource: T | undefined) => void): void {
+  if (resource === undefined) {
+    callback(undefined)
+    return
+  }
+  const cached = cachedResource.get(resource)
+  if (cached !== undefined) {
+    callback(cached)
+  } else {
+    void getResource(resource)
+      .then((r) => {
+        callback(r)
+      })
+      .catch(() => {
+        callback(undefined)
+      })
+  }
+}
+
 /**
  * @public
  */

packages/presentation/src/components/IndexedDocumentCompare.svelte

@@ -1,33 +0,0 @@
-<script lang="ts">
-  import { DocIndexState } from '@hcengineering/core'
-  import { EditBox, Dialog } from '@hcengineering/ui'
-  import IndexedDocumentContent from './IndexedDocumentContent.svelte'
-
-  export let left: DocIndexState
-  export let right: DocIndexState | undefined
-
-  let search: string = ''
-</script>
-
-<Dialog isFullSize on:changeContent on:close on:fullsize>
-  <EditBox autoFocus bind:value={search} kind={'default-large'} fullSize />
-  <div class="flex-row-top px-3 mt-4 text-base">
-    <div class="indexed-doc-part flex-col">
-      <IndexedDocumentContent indexDoc={left} {search} />
-    </div>
-    {#if right !== undefined}
-      <div class="indexed-doc-part flex-col ml-4">
-        <IndexedDocumentContent indexDoc={right} {search} />
-      </div>
-    {/if}
-  </div>
-</Dialog>
-
-<style lang="scss">
-  .indexed-doc-part {
-    overflow: auto;
-    flex-basis: 100%;
-    height: fit-content;
-    max-height: 100%;
-  }
-</style>

packages/presentation/src/components/IndexedDocumentContent.svelte

@@ -1,97 +0,0 @@
-<script lang="ts">
-  import { AnyAttribute, DocIndexState, extractDocKey, isFullTextAttribute } from '@hcengineering/core'
-
-  import { Label, Icon } from '@hcengineering/ui'
-  import { getClient } from '../utils'
-
-  export let indexDoc: DocIndexState
-  export let search: string = ''
-
-  const client = getClient()
-
-  function getContent (extra: string[], value: string): string[] {
-    if (value == null || value === '') {
-      return []
-    }
-    const result = extra.includes('base64') ? decodeURIComponent(escape(atob(value))) : value
-
-    return `${result}`.split('\n')
-  }
-
-  $: summary = indexDoc?.fullSummary ?? undefined
-
-  $: attributes =
-    indexDoc !== undefined
-      ? Object.entries(indexDoc.attributes).reduce<[AnyAttribute, string[][]][]>((a, b) => {
-        const bb = extractDocKey(b[0])
-        if (bb._class === undefined) {
-          return a
-        }
-        const attr = client.getHierarchy().findAttribute(bb._class, bb.attr)
-        if (attr === undefined || !isFullTextAttribute(attr)) {
-          return a
-        }
-        const pos = a.findIndex((it) => it[0] === attr)
-        if (pos !== -1) {
-          a[pos][1].push(getContent(bb.extra, b[1]))
-        } else {
-          a.push([attr, [getContent(bb.extra, b[1])]])
-        }
-        return a
-      }, [])
-      : []
-</script>
-
-{#if summary}
-  {#if search.length > 0}
-    <span class="font-medium">Result:</span>
-    {#each summary.split('\n').filter((line, idx, arr) => {
-      return line.toLowerCase().includes(search.toLowerCase()) || arr[idx - 1]
-        ?.toLowerCase()
-        .includes(search.toLowerCase())
-    }) as line}
-      <span class:highlight={line.toLowerCase().includes(search.toLowerCase())}>{line}</span>
-    {/each}
-    <br />
-  {/if}
-  <span class="font-medium">Summary:</span>
-  {#each summary.split('\n') as line}
-    {@const hl = search.length > 0 && line.toLowerCase().includes(search.toLowerCase())}
-    <span class="select-text" class:text-md={!hl} class:highlight={hl}>{line}</span>
-  {/each}
-{:else if indexDoc}
-  {#each attributes as attr}
-    {@const clOf = client.getHierarchy().getClass(attr[0].attributeOf)}
-    <div class="flex-row-center">
-      {#if clOf.icon}
-        <div class="mr-1">
-          <Icon size={'medium'} icon={clOf.icon} />
-        </div>
-      {/if}
-      <Label label={clOf.label} />.<Label label={attr[0].label} />
-    </div>
-    <div class="p-1 flex-row flex-wrap">
-      {#each attr[1] as doc}
-        <div class="p-1" class:flex-col={doc.length > 1}>
-          {#if search.length > 0}
-            <span class="font-medium">Result:</span>
-            {#each doc.filter((line) => line.toLowerCase().includes(search.toLowerCase())) as line}
-              <span class="select-text" class:highlight={true}>{line}</span>
-            {/each}
-            <br />
-          {/if}
-          {#each doc as line}
-            {@const hl = search.length > 0 && line.toLowerCase().includes(search.toLowerCase())}
-            <span class="select-text" class:text-md={!hl} class:highlight={hl}>{line}</span>
-          {/each}
-        </div>
-      {/each}
-    </div>
-  {/each}
-{/if}
-
-<style lang="scss">
-  .highlight {
-    color: var(--theme-link-color);
-  }
-</style>

packages/presentation/src/components/IndexedDocumentPreview.svelte

@@ -1,52 +0,0 @@
-<script lang="ts">
-  import core, { Doc, DocIndexState, Ref } from '@hcengineering/core'
-
-  import { EditBox, Label, Dialog } from '@hcengineering/ui'
-  import { createQuery } from '../utils'
-  import IndexedDocumentContent from './IndexedDocumentContent.svelte'
-  import presentation from '../plugin'
-
-  export let objectId: Ref<Doc> | undefined
-  export let indexDoc: DocIndexState | undefined = undefined
-  export let search: string = ''
-  export let noPanel = false
-
-  const indexDocQuery = createQuery()
-  $: if (objectId !== undefined) {
-    indexDocQuery.query(core.class.DocIndexState, { _id: objectId as Ref<DocIndexState> }, (res) => {
-      indexDoc = res.shift()
-    })
-  } else {
-    indexDocQuery.unsubscribe()
-  }
-</script>
-
-{#if noPanel}
-  <div class="p-1 flex-col">
-    <Label label={presentation.string.DocumentPreview} />
-    <EditBox autoFocus bind:value={search} kind={'default-large'} fullSize />
-  </div>
-  <div class="indexed-doc flex-col px-3 mt-4 text-base">
-    {#if indexDoc}
-      <IndexedDocumentContent {indexDoc} {search} />
-    {/if}
-  </div>
-{:else}
-  <Dialog isFullSize on:changeContent on:close on:fullsize>
-    <EditBox autoFocus bind:value={search} kind={'default-large'} fullSize />
-    <div class="indexed-doc flex-col px-3 mt-4 text-base">
-      {#if indexDoc}
-        <IndexedDocumentContent {indexDoc} {search} />
-      {/if}
-    </div>
-  </Dialog>
-{/if}
-
-<style lang="scss">
-  .indexed-doc {
-    overflow: auto;
-    flex-basis: 100%;
-    height: fit-content;
-    max-height: 100%;
-  }
-</style>

packages/presentation/src/components/SearchResult.svelte

@@ -15,33 +15,51 @@
 -->
 <script lang="ts">
   import type { SearchResultDoc } from '@hcengineering/core'
-  import { getResource } from '@hcengineering/platform'
-  import { AnyComponent, Icon } from '@hcengineering/ui'
+  import { getResourceC } from '@hcengineering/platform'
+  import { Icon, type AnySvelteComponent } from '@hcengineering/ui'
 
   export let value: SearchResultDoc
 
-  $: iconComponent = value.iconComponent ? (value.iconComponent as AnyComponent) : undefined
-  $: icon = value.icon !== undefined ? value.icon : undefined
+  $: icon = value.icon
+
+  let iconComponent: AnySvelteComponent | undefined
+  let shortTitleComponent: AnySvelteComponent | undefined
+  let titleComponent: AnySvelteComponent | undefined
+
+  $: getResourceC(value.iconComponent?.component, (r) => {
+    iconComponent = r
+  })
+
+  $: getResourceC(value.shortTitleComponent?.component, (r) => {
+    shortTitleComponent = r
+  })
+
+  $: getResourceC(value.titleComponent?.component, (r) => {
+    titleComponent = r
+  })
 </script>
 
 <div class="flex-row-center">
   <div class="flex-center p-1 content-dark-color flex-no-shrink">
-    {#if icon !== undefined}
-      <Icon {icon} size={'medium'} />
-    {/if}
     {#if iconComponent}
       <div class="icon-place">
-        {#await getResource(iconComponent) then component}
-          <svelte:component this={component} size={'smaller'} {...value.iconProps} />
-        {/await}
+        <svelte:component this={iconComponent} size={'smaller'} {...value.iconComponent?.props} />
       </div>
+    {:else if icon !== undefined}
+      <Icon {icon} size={'small'} />
     {/if}
   </div>
-  <span class="ml-1 max-w-120 overflow-label searchResult">
-    {#if value.shortTitle !== undefined}
+  <span class="ml-1 max-w-120 overflow-label searchResult flex-row-center">
+    {#if shortTitleComponent}
+      <svelte:component this={shortTitleComponent} {...value.shortTitleComponent?.props} />
+    {:else if value.shortTitle !== undefined}
       <span class="shortTitle">{value.shortTitle}</span>
     {/if}
-    <span class="name">{value.title}</span>
+    {#if titleComponent}
+      <svelte:component this={titleComponent} {...value.titleComponent?.props} />
+    {:else}
+      <span class="name">{value.title}</span>
+    {/if}
   </span>
 </div>
 

packages/presentation/src/index.ts

@@ -35,8 +35,6 @@ export { default as SpaceSelect } from './components/SpaceSelect.svelte'
 export { default as SpaceSelector } from './components/SpaceSelector.svelte'
 export { default as SpacesMultiPopup } from './components/SpacesMultiPopup.svelte'
 export { default as ObjectSearchPopup } from './components/ObjectSearchPopup.svelte'
-export { default as IndexedDocumentPreview } from './components/IndexedDocumentPreview.svelte'
-export { default as IndexedDocumentCompare } from './components/IndexedDocumentCompare.svelte'
 export { default as NavLink } from './components/NavLink.svelte'
 export { default as IconDownload } from './components/icons/Download.svelte'
 export { default as FilePreview } from './components/FilePreview.svelte'

plugins/ai-bot/src/rest.ts

@@ -74,4 +74,10 @@ export interface PostTranscriptRequest {
   transcript: string
   participant: Ref<Person>
   roomName: string
+  final: boolean
+}
+
+export interface IdentityResponse {
+  identity: Ref<Person>
+  name: string
 }

plugins/contact-resources/src/components/ContactNamePresenter.svelte

@@ -0,0 +1,37 @@
+<!--
+// Copyright © 2024 Anticrm Platform Contributors.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+-->
+
+<script lang="ts">
+  import contactPlugin, { getFirstName, getLastName } from '@hcengineering/contact'
+  import { getMetadata } from '@hcengineering/platform'
+
+  export let name: string | null | undefined = undefined
+
+  function getDisplayName (name: string | null | undefined): string {
+    if (name == null) {
+      return ''
+    }
+
+    const lastFirst = getMetadata(contactPlugin.metadata.LastNameFirst) === true
+    const fname = getFirstName(name ?? '') ?? ''
+    const lname = getLastName(name ?? '') ?? ''
+
+    return lastFirst ? lname + ' ' + fname : fname + ' ' + lname
+  }
+
+  $: displayName = getDisplayName(name)
+</script>
+
+{displayName}

plugins/contact-resources/src/index.ts

@@ -123,6 +123,7 @@ import NameChangedActivityMessage from './components/activity/NameChangedActivit
 import IconAddMember from './components/icons/AddMember.svelte'
 import ExpandRightDouble from './components/icons/ExpandRightDouble.svelte'
 import IconMembers from './components/icons/Members.svelte'
+import ContactNamePresenter from './components/ContactNamePresenter.svelte'
 
 import { get, writable } from 'svelte/store'
 import contact from './plugin'
@@ -380,7 +381,8 @@ export default async (): Promise<Resources> => ({
     PersonIcon,
     EditOrganizationPanel,
     ChannelIcon,
-    SpaceMembersEditor
+    SpaceMembersEditor,
+    ContactNamePresenter
   },
   completion: {
     EmployeeQuery: async (

plugins/contact/src/index.ts

@@ -221,7 +221,8 @@ export const contactPlugin = plugin(contactId, {
     EditOrganizationPanel: '' as AnyComponent,
     CollaborationUserAvatar: '' as AnyComponent,
     CreateGuest: '' as AnyComponent,
-    SpaceMembersEditor: '' as AnyComponent
+    SpaceMembersEditor: '' as AnyComponent,
+    ContactNamePresenter: '' as AnyComponent
   },
   channelProvider: {
     Email: '' as Ref<ChannelProvider>,

plugins/love-resources/src/components/Room.svelte

@@ -58,6 +58,7 @@
     connecting: boolean
     muted: boolean
     mirror: boolean
+    isAgent: boolean
   }
 
   let participants: ParticipantData[] = []
@@ -80,7 +81,7 @@
         const element = track.attach()
         attachTrack(element, participant)
       }
-      updateStyle(participants.length, $screenSharing)
+      updateStyle(getActiveParticipants(participants).length, $screenSharing)
     } else {
       const part = participants.find((p) => p._id === participant.identity)
       if (part !== undefined) {
@@ -105,7 +106,7 @@
       } else {
         track.detach(screen)
       }
-      updateStyle(participants.length, $screenSharing)
+      updateStyle(getActiveParticipants(participants).length, $screenSharing)
     }
   }
 
@@ -117,7 +118,7 @@
         const element = publication.track.attach()
         void attachTrack(element, participant)
       }
-      updateStyle(participants.length, $screenSharing)
+      updateStyle(getActiveParticipants(participants).length, $screenSharing)
     } else {
       const part = participants.find((p) => p._id === participant.identity)
       if (part !== undefined) {
@@ -135,7 +136,8 @@
         name: participant.name ?? '',
         muted: !participant.isMicrophoneEnabled,
         mirror: participant.isLocal,
-        connecting: false
+        connecting: false,
+        isAgent: participant.isAgent
       })
     }
     participants = participants
@@ -166,11 +168,12 @@
       name: participant.name ?? '',
       muted: !participant.isMicrophoneEnabled,
       mirror: participant.isLocal,
-      connecting: false
+      connecting: false,
+      isAgent: participant.isAgent
     }
     participants.push(value)
     participants = participants
-    updateStyle(participants.length, $screenSharing)
+    updateStyle(getActiveParticipants(participants).length, $screenSharing)
   }
 
   function handleParticipantDisconnected (participant: RemoteParticipant): void {
@@ -179,7 +182,7 @@
       participants.splice(index, 1)
       participants = participants
     }
-    updateStyle(participants.length, $screenSharing)
+    updateStyle(getActiveParticipants(participants).length, $screenSharing)
   }
 
   function muteHandler (publication: TrackPublication, participant: Participant): void {
@@ -207,7 +210,7 @@
     if (publication?.track?.kind === Track.Kind.Video) {
       if (publication.track.source === Track.Source.ScreenShare) {
         publication.track.detach(screen)
-        updateStyle(participants.length, $screenSharing)
+        updateStyle(getActiveParticipants(participants).length, $screenSharing)
       } else {
         const index = participants.findIndex((p) => p._id === participant.identity)
         if (index !== -1) {
@@ -285,10 +288,6 @@
 
   onDestroy(
     infos.subscribe((data) => {
-      const aiParticipant = aiPersonId !== undefined ? participants.find(({ _id }) => _id === aiPersonId) : undefined
-      if (aiParticipant && !data.some((it) => it.room === room._id && it.person === aiParticipant._id)) {
-        participants = participants.filter(({ _id }) => _id !== aiPersonId)
-      }
       for (const info of data) {
         if (info.room !== room._id) continue
         const current = participants.find((p) => p._id === info.person)
@@ -298,12 +297,13 @@
           name: info.name,
           muted: true,
           mirror: false,
-          connecting: true
+          connecting: true,
+          isAgent: aiPersonId === info.person
         }
         participants.push(value)
       }
       participants = participants
-      updateStyle(participants.length, $screenSharing)
+      updateStyle(getActiveParticipants(participants).length, $screenSharing)
     })
   )
 
@@ -342,6 +342,12 @@
     }
   }
   $: if (((document.fullscreenElement && !$isFullScreen) || $isFullScreen) && roomEl) toggleFullscreen()
+
+  function getActiveParticipants (participants: ParticipantData[]): ParticipantData[] {
+    return participants.filter((p) => !p.isAgent || $infos.some(({ person }) => person === p._id))
+  }
+
+  $: activeParticipants = getActiveParticipants(participants)
 </script>
 
 <div bind:this={roomEl} class="flex-col-center w-full h-full right-navpanel-border" class:theme-dark={$isFullScreen}>
@@ -376,7 +382,7 @@
         style={$screenSharing ? '' : gridStyle}
         class:scroll-m-0={$screenSharing}
       >
-        {#each participants as participant, i (participant._id)}
+        {#each activeParticipants as participant, i (participant._id)}
           <ParticipantView
             bind:this={participantElements[i]}
             {...participant}

plugins/recruit-assets/lang/en.json

@@ -98,7 +98,6 @@
     "HasNoActiveApplicant": "No Active",
     "NoneApplications": "None",
     "VacancyList": "Vacancies",
-    "MatchVacancy": "Match to vacancy",
     "VacancyMatching": "Match Talents to vacancy",
     "Score": "Score",
     "Match": "Match",

plugins/recruit-assets/lang/es.json

@@ -98,7 +98,6 @@
     "HasNoActiveApplicant": "Sin Activos",
     "NoneApplications": "Ninguna",
     "VacancyList": "Vacantes",
-    "MatchVacancy": "Asignar a vacante",
     "VacancyMatching": "Asignar Talentos a vacante",
     "Score": "Puntuación",
     "Match": "Coincidencia",

plugins/recruit-assets/lang/fr.json

@@ -98,7 +98,6 @@
     "HasNoActiveApplicant": "Aucun actif",
     "NoneApplications": "Aucun",
     "VacancyList": "Liste des postes vacants",
-    "MatchVacancy": "Associer au poste vacant",
     "VacancyMatching": "Associer les talents au poste vacant",
     "Score": "Score",
     "Match": "Associer",

plugins/recruit-assets/lang/pt.json

@@ -98,7 +98,6 @@
     "HasNoActiveApplicant": "Sem Ativos",
     "NoneApplications": "Nenhuma",
     "VacancyList": "Vagas",
-    "MatchVacancy": "Atribuir à vaga",
     "VacancyMatching": "Atribuir Talentos à vaga",
     "Score": "Pontuação",
     "Match": "Correspondência",

plugins/recruit-assets/lang/ru.json

@@ -100,7 +100,6 @@
     "HasNoActiveApplicant": "Не активные",
     "NoneApplications": "Отсутствуют",
     "VacancyList": "Вакансии",
-    "MatchVacancy": "Проверить на вакансию",
     "VacancyMatching": "Подбор кандидатов на вакансию",
     "Score": "Оценка",
     "Match": "Совпадение",

plugins/recruit-assets/lang/zh.json

@@ -98,7 +98,6 @@
     "HasNoActiveApplicant": "无活跃",
     "NoneApplications": "无",
     "VacancyList": "职位空缺",
-    "MatchVacancy": "匹配职位",
     "VacancyMatching": "匹配人才到职位",
     "Score": "评分",
     "Match": "匹配",

plugins/recruit-resources/src/components/ApplicantNamePresenter.svelte

@@ -0,0 +1,64 @@
+<!--
+// Copyright © 2024 Anticrm Platform Contributors.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+-->
+
+<script lang="ts">
+  import contactPlugin, { getFirstName, getLastName } from '@hcengineering/contact'
+  import { getMetadata } from '@hcengineering/platform'
+
+  export let parentName: string | null | undefined = undefined
+  export let spaceName: string | null | undefined = undefined
+
+  function getDisplayName (name: string | null | undefined): string {
+    if (name == null) {
+      return ''
+    }
+
+    const lastFirst = getMetadata(contactPlugin.metadata.LastNameFirst) === true
+    const fname = getFirstName(name ?? '') ?? ''
+    const lname = getLastName(name ?? '') ?? ''
+
+    return lastFirst ? lname + ' ' + fname : fname + ' ' + lname
+  }
+
+  $: displayName = getDisplayName(parentName)
+</script>
+
+<div class="flex-row-center">
+  <span>{displayName}</span>
+  {#if spaceName.trim().length > 0}
+    <span class="p-1">·</span>
+    <span class="text-sm">{spaceName}</span>
+  {/if}
+</div>
+
+<style lang="scss">
+  .icon-place {
+    width: 1.75rem;
+  }
+  .searchResult {
+    display: flex;
+    flex-direction: row;
+
+    .shortTitle {
+      display: flex;
+      padding-right: 0.5rem;
+      color: var(--theme-darker-color);
+    }
+    .name {
+      display: flex;
+      flex: 1;
+    }
+  }
+</style>

plugins/recruit-resources/src/components/MatchVacancy.svelte

@@ -1,272 +0,0 @@
-<script lang="ts">
-  import contact from '@hcengineering/contact'
-  import core, { Doc, DocIndexState, FindOptions, Ref } from '@hcengineering/core'
-  import presentation, {
-    Card,
-    createQuery,
-    getClient,
-    HTMLViewer,
-    IndexedDocumentCompare,
-    MessageViewer,
-    SpaceSelect
-  } from '@hcengineering/presentation'
-  import { Applicant, ApplicantMatch, Candidate, Vacancy } from '@hcengineering/recruit'
-  import { Button, IconActivity, IconAdd, Label, resizeObserver, showPopup, Spinner, tooltip } from '@hcengineering/ui'
-  import Scroller from '@hcengineering/ui/src/components/Scroller.svelte'
-  import { MarkupPreviewPopup, ObjectPresenter } from '@hcengineering/view-resources'
-  import { calcSørensenDiceCoefficient, cosinesim } from '@hcengineering/view-resources/src/utils'
-  import { createEventDispatcher } from 'svelte'
-  import recruit from '../plugin'
-  import CreateApplication from './CreateApplication.svelte'
-  import VacancyCard from './VacancyCard.svelte'
-  import VacancyOrgPresenter from './VacancyOrgPresenter.svelte'
-
-  export let objects: Candidate[] | Candidate
-
-  $: _objects = Array.isArray(objects) ? objects : [objects]
-
-  const orgOptions: FindOptions<Vacancy> = {
-    lookup: {
-      company: contact.class.Organization
-    }
-  }
-  let _space: Ref<Vacancy> | undefined
-  let vacancy: Vacancy | undefined
-
-  const vacancyQuery = createQuery()
-  $: vacancyQuery.query(recruit.class.Vacancy, { _id: _space }, (res) => {
-    ;[vacancy] = res
-  })
-
-  const indexDataQuery = createQuery()
-  let state = new Map<Ref<Doc>, DocIndexState>()
-  $: indexDataQuery.query(
-    core.class.DocIndexState,
-    {
-      _id: {
-        $in: [_space as unknown as Ref<DocIndexState>, ..._objects.map((it) => it._id as unknown as Ref<DocIndexState>)]
-      }
-    },
-    (res) => {
-      state = new Map(res.map((it) => [it._id, it] ?? []))
-    }
-  )
-
-  $: vacancyState = state.get(_space as unknown as Ref<DocIndexState>)
-
-  $: scoreState = new Map(
-    _objects.map((it) => [
-      it._id,
-      Math.round(
-        calcSørensenDiceCoefficient(state.get(it._id)?.fullSummary ?? '', vacancyState?.fullSummary ?? '') * 100
-      ) / 100
-    ])
-  )
-
-  $: _sortedObjects = [..._objects].sort((a, b) => (scoreState.get(b._id) ?? 0) - (scoreState.get(a._id) ?? 0))
-
-  const matchQuery = createQuery()
-  let matches = new Map<Ref<Doc>, ApplicantMatch>()
-
-  $: matchQuery.query(
-    recruit.class.ApplicantMatch,
-    {
-      attachedTo: { $in: [..._objects.map((it) => it._id)] },
-      space: _space
-    },
-    (res) => {
-      matches = new Map(res.map((it) => [it.attachedTo, it] ?? []))
-    }
-  )
-
-  const applicationQuery = createQuery()
-  let applications = new Map<Ref<Doc>, Applicant>()
-
-  $: applicationQuery.query(
-    recruit.class.Applicant,
-    {
-      attachedTo: { $in: [..._objects.map((it) => it._id)] },
-      space: _space
-    },
-    (res) => {
-      applications = new Map(res.map((it) => [it.attachedTo, it] ?? []))
-    }
-  )
-
-  function getEmbedding (doc: DocIndexState): number[] | undefined {
-    for (const [k, v] of Object.entries(doc.attributes)) {
-      if (k.startsWith('openai_embedding_') && doc.attributes[k + '_use'] === true) {
-        return v
-      }
-    }
-  }
-  $: vacancyEmbedding = vacancyState && getEmbedding(vacancyState)
-
-  const dispatch = createEventDispatcher()
-
-  const client = getClient()
-  const matching = new Set<string>()
-
-  async function requestMatch (doc: Candidate, docState: DocIndexState): Promise<void> {
-    try {
-      matching.add(doc._id)
-      if (_space === undefined) {
-        return
-      }
-      const oldMatch = matches.get(doc._id)
-      if (oldMatch) {
-        await client.remove(oldMatch)
-      }
-      await client.addCollection(recruit.class.ApplicantMatch, _space, doc._id, doc._class, 'vacancyMatch', {
-        complete: false,
-        vacancy: vacancyState?.fullSummary ?? '',
-        summary: docState.fullSummary ?? '',
-        response: ''
-      })
-    } finally {
-      matching.delete(doc._id)
-    }
-  }
-  async function createApplication (doc: Candidate, match?: ApplicantMatch): Promise<void> {
-    showPopup(
-      CreateApplication,
-      {
-        space: _space,
-        candidate: doc._id,
-        preserveCandidate: true,
-        preserveVacancy: true,
-        comment: match?.response ?? ''
-      },
-      'top'
-    )
-  }
-  async function showSummary (left: DocIndexState, right?: DocIndexState): Promise<void> {
-    showPopup(IndexedDocumentCompare, { left, right }, 'centered')
-  }
-</script>
-
-<Card
-  label={recruit.string.VacancyMatching}
-  okLabel={presentation.string.Ok}
-  on:close
-  okAction={() => {}}
-  canSave={true}
-  on:changeContent
->
-  <Scroller horizontal>
-    <div
-      class="flex-row flex-nowrap"
-      use:resizeObserver={() => {
-        dispatch('changeContent')
-      }}
-    >
-      <div class="flex-row-center antiEmphasized">
-        <div class="p-1 flex-grow">
-          <SpaceSelect
-            size={'large'}
-            _class={recruit.class.Vacancy}
-            spaceQuery={{ archived: false }}
-            spaceOptions={orgOptions}
-            label={recruit.string.Vacancy}
-            create={{
-              component: recruit.component.CreateVacancy,
-              label: recruit.string.CreateVacancy
-            }}
-            bind:value={_space}
-            component={VacancyOrgPresenter}
-            componentProps={{ inline: true }}
-          >
-            <svelte:fragment slot="content">
-              <VacancyCard {vacancy} disabled={true} />
-            </svelte:fragment>
-          </SpaceSelect>
-        </div>
-        <div class="p-1">
-          {#if vacancy}
-            <Scroller>
-              <div class="flex-col max-h-60 select-text">
-                {#if vacancy.description}
-                  {vacancy.description}
-                {/if}
-                {#if vacancyState?.fullSummary}
-                  <HTMLViewer value={vacancyState?.fullSummary.split('\n').join('<br/>')} />
-                {/if}
-              </div>
-            </Scroller>
-          {/if}
-        </div>
-      </div>
-
-      <Scroller>
-        <table class="antiTable mt-2">
-          <thead class="scroller-thead">
-            <tr class="scroller-thead__tr">
-              <td><Label label={recruit.string.Talent} /></td>
-              <td><Label label={recruit.string.Score} /></td>
-              <td><Label label={recruit.string.Match} /></td>
-              <td>#</td>
-            </tr>
-          </thead>
-
-          <tbody>
-            {#each _sortedObjects as doc}
-              {@const docState = state.get(doc._id)}
-              {@const docEmbedding = docState && getEmbedding(docState)}
-              {@const match = matches.get(doc._id)}
-              {@const appl = applications.get(doc._id)}
-              <tr class="antiTable-body__row">
-                <td>
-                  <div class="flex-row-center">
-                    <ObjectPresenter objectId={doc._id} _class={doc._class} value={doc} />
-                    {#if appl}
-                      <div class="ml-2 flex-row-center">
-                        <ObjectPresenter objectId={appl._id} _class={appl._class} value={appl} />
-                      </div>
-                    {/if}
-                  </div>
-                </td>
-                <td class="whitespace-nowrap">
-                  {#if docEmbedding && vacancyEmbedding}
-                    {Math.round(cosinesim(docEmbedding, vacancyEmbedding) * 100)}
-                    /
-                  {/if}
-                  {scoreState.get(doc._id) ?? 0}
-                </td>
-                <td>
-                  {#if match?.complete}
-                    <div
-                      class="lines-limit-4"
-                      use:tooltip={{ component: MarkupPreviewPopup, props: { value: match.response } }}
-                    >
-                      <MessageViewer message={match.response} />
-                    </div>
-                  {/if}
-                </td>
-                <td class="flex-row-center gap-2">
-                  {#if docState}
-                    <Button
-                      label={recruit.string.PerformMatch}
-                      icon={matching.has(doc._id) || !(match?.complete ?? true) ? Spinner : IconActivity}
-                      on:click={() => requestMatch(doc, docState)}
-                    />
-                    <Button
-                      icon={IconActivity}
-                      showTooltip={{ label: presentation.string.DocumentPreview }}
-                      on:click={() => showSummary(docState, vacancyState)}
-                    />
-                    <Button
-                      icon={IconAdd}
-                      disabled={appl !== undefined}
-                      showTooltip={{ label: recruit.string.CreateVacancy }}
-                      on:click={() => createApplication(doc, match)}
-                    />
-                  {/if}
-                </td>
-              </tr>
-            {/each}
-          </tbody>
-        </table>
-      </Scroller>
-    </div>
-  </Scroller>
-</Card>

plugins/recruit-resources/src/index.ts

@@ -42,7 +42,6 @@ import CreateVacancy from './components/CreateVacancy.svelte'
 import EditApplication from './components/EditApplication.svelte'
 import EditVacancy from './components/EditVacancy.svelte'
 import KanbanCard from './components/KanbanCard.svelte'
-import MatchVacancy from './components/MatchVacancy.svelte'
 import NewCandidateHeader from './components/NewCandidateHeader.svelte'
 import NotificationApplicantPresenter from './components/NotificationApplicantPresenter.svelte'
 import Organizations from './components/Organizations.svelte'
@@ -65,6 +64,7 @@ import Opinions from './components/review/Opinions.svelte'
 import OpinionsPresenter from './components/review/OpinionsPresenter.svelte'
 import ReviewPresenter from './components/review/ReviewPresenter.svelte'
 import Reviews from './components/review/Reviews.svelte'
+import ApplicantNamePresenter from './components/ApplicantNamePresenter.svelte'
 import recruit from './plugin'
 import {
   getAppIdentifier,
@@ -386,9 +386,9 @@ export default async (): Promise<Resources> => ({
     VacancyList,
     VacancyTemplateEditor,
 
-    MatchVacancy,
     NotificationApplicantPresenter,
-    VacancyEditor
+    VacancyEditor,
+    ApplicantNamePresenter
   },
   completion: {
     ApplicationQuery: async (

plugins/recruit-resources/src/plugin.ts

@@ -114,7 +114,6 @@ export default mergeIds(recruitId, recruit, {
     HasNoActiveApplicant: '' as IntlString,
     NoneApplications: '' as IntlString,
 
-    MatchVacancy: '' as IntlString,
     VacancyMatching: '' as IntlString,
     Score: '' as IntlString,
     Match: '' as IntlString,

plugins/view-resources/src/components/TableBrowser.svelte

@@ -14,15 +14,13 @@
 -->
 <script lang="ts">
   import type { Class, Doc, DocumentQuery, FindOptions, Ref } from '@hcengineering/core'
-  import { getEmbeddedLabel } from '@hcengineering/platform'
-  import { Scroller, tableSP, FadeOptions } from '@hcengineering/ui'
+  import { ActionContext } from '@hcengineering/presentation'
+  import { FadeOptions, Scroller, tableSP } from '@hcengineering/ui'
   import { BuildModelKey } from '@hcengineering/view'
   import { onMount } from 'svelte'
   import { focusStore, ListSelectionProvider, SelectDirection } from '../selection'
   import { LoadingProps } from '../utils'
-  import SourcePresenter from './inference/SourcePresenter.svelte'
   import Table from './Table.svelte'
-  import { ActionContext } from '@hcengineering/presentation'
 
   export let _class: Ref<Class<Doc>>
   export let query: DocumentQuery<Doc>
@@ -55,26 +53,12 @@
   })
 
   // Search config
-  let _config = config
+  const _config = config
 
   let prefferedSorting: string = 'modifiedOn'
 
   function updateConfig (config: Array<BuildModelKey | string>, search?: string): void {
     const useSearch = search !== '' && search != null
-    _config = [
-      ...(useSearch
-        ? [
-            {
-              key: '',
-              presenter: SourcePresenter,
-              label: getEmbeddedLabel('#'),
-              sortingKey: '#score',
-              props: { search }
-            }
-          ]
-        : []),
-      ...config
-    ]
     prefferedSorting = !useSearch ? 'modifiedOn' : '#score'
   }
 

plugins/view-resources/src/components/inference/SourcePresenter.svelte

@@ -1,38 +0,0 @@
-<!--
-// Copyright © 2022 Hardcore Engineering Inc.
-//
-// Licensed under the Eclipse Public License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License. You may
-// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//
-// See the License for the specific language governing permissions and
-// limitations under the License.
--->
-<script lang="ts">
-  import { Doc, WithLookup } from '@hcengineering/core'
-  import { IndexedDocumentPreview } from '@hcengineering/presentation'
-  import { showPopup, tooltip } from '@hcengineering/ui'
-  import plugin from '../../plugin'
-
-  export let value: WithLookup<Doc>
-  export let search: string
-</script>
-
-<!-- svelte-ignore a11y-click-events-have-key-events -->
-<!-- svelte-ignore a11y-no-static-element-interactions -->
-<span
-  use:tooltip={{ label: plugin.string.ShowPreviewOnClick }}
-  on:click={() => {
-    showPopup(IndexedDocumentPreview, { objectId: value._id, search }, 'centered')
-  }}
->
-  {#if value.$source?.$score}
-    {Math.round(value.$source?.$score * 100) / 100}
-  {:else}
-    *
-  {/if}
-</span>

plugins/view-resources/src/components/list/List.svelte

@@ -104,7 +104,7 @@
     { ...categoryQueryOptions, limit: 1000 }
   )
 
-  $: if (fastDocs.length === 1000) {
+  $: if (fastDocs.length === 1000 && queryNoLookup.$search == null) {
     docsQuerySlow.query(
       _class,
       queryNoLookup,
@@ -113,6 +113,8 @@
       },
       categoryQueryOptions
     )
+  } else {
+    slowDocs = []
   }
 
   $: docs = [...fastDocs, ...slowDocs.filter((it) => !fastQueryIds.has(it._id))]

plugins/view-resources/src/components/list/ListCategory.svelte

@@ -34,10 +34,10 @@
   import { AttributeModel, BuildModelKey, ViewOptionModel, ViewOptions, Viewlet } from '@hcengineering/view'
   import { createEventDispatcher } from 'svelte'
   import { fade } from 'svelte/transition'
+  import { showMenu } from '../../actions'
   import { FocusSelection, SelectionFocusProvider, focusStore } from '../../selection'
   import ListHeader from './ListHeader.svelte'
   import ListItem from './ListItem.svelte'
-  import { showMenu } from '../../actions'
 
   export let category: PrimitiveType | AggregateValue
   export let headerComponent: AttributeModel | undefined
@@ -108,11 +108,27 @@
     }
   }
 
+  function noSearch (query: DocumentQuery<Doc>): DocumentQuery<Doc> {
+    const result = query
+    if ('$search' in result) {
+      delete result.$search
+    }
+    return result
+  }
+
+  $: finalResultQuery =
+    itemProj.length < 20 && resultQuery.$search !== null
+      ? noSearch({
+        ...resultQuery,
+        _id: { $in: itemProj.map((it) => it._id) }
+      })
+      : resultQuery
+
   $: if (lastLevel) {
     void limiter.add(async () => {
       loading = docsQuery.query(
         _class,
-        { ...resultQuery, ...docKeys },
+        { ...finalResultQuery, ...docKeys },
         (res) => {
           items = res
           loading = false

plugins/view-resources/src/index.ts

@@ -126,7 +126,6 @@ import {
   valueNinResult
 } from './filter'
 
-import { IndexedDocumentPreview } from '@hcengineering/presentation'
 import { AggregationMiddleware, AnalyticsMiddleware } from './middleware'
 import { showEmptyGroups } from './viewOptions'
 import {
@@ -162,7 +161,6 @@ export { default as ViewletSelector } from './components/ViewletSelector.svelte'
 export { default as ViewletsSettingButton } from './components/ViewletsSettingButton.svelte'
 export { default as FilterButton } from './components/filter/FilterButton.svelte'
 export { default as FilterRemovedNotification } from './components/filter/FilterRemovedNotification.svelte'
-export { default as SourcePresenter } from './components/inference/SourcePresenter.svelte'
 export { default as List } from './components/list/List.svelte'
 export { default as NavLink } from './components/navigator/NavLink.svelte'
 export { default as StatusPresenter } from './components/status/StatusPresenter.svelte'
@@ -281,7 +279,6 @@ export default async (): Promise<Resources> => ({
     ListView,
     GrowPresenter,
     DividerPresenter,
-    IndexedDocumentPreview,
     SpaceRefPresenter,
     SpaceTypeSelector,
     EnumArrayEditor,

pods/backup/src/index.ts

@@ -53,12 +53,8 @@ startBackup(
     return factory
   },
   (ctx, dbUrl, workspace, branding, externalStorage) => {
-    return getConfig(ctx, dbUrl, workspace, branding, ctx, {
+    return getConfig(ctx, dbUrl, ctx, {
       externalStorage,
-      fullTextUrl: '',
-      indexParallel: 0,
-      indexProcessing: 0,
-      rekoniUrl: '',
       disableTriggers: true
     })
   }

pods/front/package.json

@@ -22,7 +22,7 @@
     "docker:tbuild": "docker build -t hardcoreeng/front . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/front",
     "docker:abuild": "docker build -t hardcoreeng/front . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/front",
     "format": "format src",
-    "run-local": "cross-env MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost SERVER_SECRET='secret' ACCOUNTS_URL=http://localhost:3000 UPLOAD_URL=/files ELASTIC_URL=http://localhost:9200 MODEL_VERSION=$(node ../../common/scripts/show_version.js) VERSION=$(node ../../common/scripts/show_tag.js) PUBLIC_DIR='.' ts-node ./src/__start.ts",
+    "run-local": "cross-env MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost SERVER_SECRET='secret' ACCOUNTS_URL=http://localhost:3000 UPLOAD_URL=/files MODEL_VERSION=$(node ../../common/scripts/show_version.js) VERSION=$(node ../../common/scripts/show_tag.js) PUBLIC_DIR='.' ts-node ./src/__start.ts",
     "test": "jest --passWithNoTests --silent --forceExit",
     "_phase:build": "compile transpile src",
     "_phase:test": "jest --passWithNoTests --silent --forceExit",
@@ -64,7 +64,6 @@
     "express-fileupload": "^1.5.1",
     "uuid": "^8.3.2",
     "cors": "^2.8.5",
-    "@hcengineering/elastic": "^0.6.0",
     "@hcengineering/server-core": "^0.6.1",
     "@hcengineering/server-token": "^0.6.11",
     "@hcengineering/attachment": "^0.6.14",

pods/fulltext/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+  extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: './tsconfig.json'
+  }
+}

pods/fulltext/.npmignore

@@ -0,0 +1,4 @@
+*
+!/lib/**
+!CHANGELOG.md
+/lib/**/__tests__/

pods/fulltext/Dockerfile

@@ -0,0 +1,20 @@
+
+FROM node:20
+
+WORKDIR /usr/src/app
+
+RUN npm install --ignore-scripts=false --verbose bufferutil utf-8-validate @mongodb-js/zstd snappy --unsafe-perm
+
+RUN apt-get update
+RUN apt-get install libjemalloc2
+RUN apt-get clean
+
+ENV LD_PRELOAD=libjemalloc.so.2
+ENV MALLOC_CONF=dirty_decay_ms:1000,narenas:2,background_thread:true
+
+COPY bundle/bundle.js ./
+COPY bundle/bundle.js.map ./
+COPY bundle/model.json ./
+
+EXPOSE 4700
+CMD [ "node", "--expose-gc", "bundle.js" ]

pods/fulltext/build.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+#  Copyright © 2022 Hardcore Engineering Inc.
+#  
+#  Licensed under the Eclipse Public License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License. You may
+#  obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+rushx bundle
+rushx docker:build
+rushx docker:push

pods/fulltext/config/rig.json

@@ -0,0 +1,5 @@
+{  
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+  "rigPackageName": "@hcengineering/platform-rig",
+  "rigProfile": "node"
+}

pods/fulltext/jest.config.js

@@ -0,0 +1,7 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
+  roots: ["./src"],
+  coverageReporters: ["text-summary", "html"]
+}

pods/fulltext/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "@hcengineering/pod-fulltext",
+  "version": "0.6.0",
+  "main": "lib/index.js",
+  "svelte": "src/index.ts",
+  "types": "types/index.d.ts",
+  "author": "Anticrm Platform Contributors",
+  "template": "@hcengineering/node-package",
+  "license": "EPL-2.0",
+  "scripts": {
+    "start": "ts-node src/index.ts",
+    "build": "compile",
+    "build:watch": "compile",
+    "_phase:bundle": "rushx bundle",
+    "_phase:docker-build": "rushx docker:build",
+    "_phase:docker-staging": "rushx docker:staging",
+    "get-model": "mkdir -p bundle && esbuild src/get-model.ts --bundle --keep-names --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.VERSION=$(node ../../common/scripts/show_tag.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error && node ./bundle/bundle.js > ./bundle/model.json",
+    "bundle": "mkdir -p bundle && rushx get-model && esbuild src/index.ts --keep-names --bundle --platform=node --external:*.node --outfile=bundle/bundle.js --log-level=error --sourcemap=external",
+    "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/fulltext",
+    "docker:tbuild": "docker build -t hardcoreeng/fulltext . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/fulltext",
+    "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/fulltext staging",
+    "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/fulltext",
+    "format": "format src",
+    "test": "jest --passWithNoTests --silent --forceExit",
+    "_phase:build": "compile transpile src",
+    "_phase:test": "jest --passWithNoTests --silent --forceExit",
+    "_phase:format": "format src",
+    "_phase:validate": "compile validate"
+  },
+  "devDependencies": {
+    "cross-env": "~7.0.3",
+    "@hcengineering/platform-rig": "^0.6.0",
+    "@types/node": "~20.11.16",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint": "^8.54.0",
+    "esbuild": "^0.20.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "prettier": "^3.1.0",
+    "ts-node": "^10.8.0",
+    "typescript": "^5.3.3",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.1",
+    "@types/jest": "^29.5.5",
+    "@hcengineering/model-all": "^0.6.0",
+    "@types/koa-bodyparser": "^4.3.12",
+    "@types/koa-router": "^7.4.8",
+    "@types/koa": "^2.15.0",
+    "@types/koa__cors": "^5.0.0"
+  },
+  "dependencies": {
+    "@hcengineering/platform": "^0.6.11",
+    "@hcengineering/server-token": "^0.6.11",
+    "@hcengineering/server-core": "^0.6.1",
+    "@hcengineering/server-pipeline": "^0.6.0",
+    "@hcengineering/client": "^0.6.18",
+    "@hcengineering/client-resources": "^0.6.27",
+    "@hcengineering/core": "^0.6.32",
+    "dotenv": "~16.0.0",
+    "@hcengineering/backup-service": "^0.6.0",
+    "@hcengineering/analytics": "^0.6.0",
+    "@hcengineering/analytics-service": "^0.6.0",
+    "koa": "^2.15.3",
+    "koa-router": "^12.0.1",
+    "koa-bodyparser": "^4.4.1",
+    "@koa/cors": "^5.0.0",
+    "@hcengineering/server-indexer": "^0.6.0",
+    "@hcengineering/elastic": "^0.6.0",
+    "@hcengineering/server-collaboration": "^0.6.0",
+    "@hcengineering/middleware": "^0.6.0",
+    "@hcengineering/server-client": "^0.6.0",
+    "@hcengineering/server-storage": "^0.6.0"
+  }
+}

pods/fulltext/run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+export FULLTEXT_DB_URL=http://localhost:9200
+export DB_URL=mongodb://localhost:27017
+
+# "DB_URL": "postgresql://postgres:example@localhost:5432",
+export STORAGE_CONFIG="minio|localhost?accessKey=minioadmin&secretKey=minioadmin"
+export SERVER_SECRET=secret
+export REKONI_URL=http://localhost:4004
+export MODEL_JSON=./bundle/model.json
+export ELASTIC_INDEX_NAME=local_storage_index
+export STATS_URL=http://host.docker.internal:4900
+export ACCOUNTS_URL=http://localhost:3000
+rushx bundle
+node --inspect ./bundle/bundle.js

pods/fulltext/src/get-model.ts

@@ -0,0 +1 @@
+import '@hcengineering/model-all/src/show'

pods/fulltext/src/index.ts

@@ -0,0 +1,138 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { Analytics } from '@hcengineering/analytics'
+import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service'
+import { MeasureMetricsContext, newMetrics, type Tx } from '@hcengineering/core'
+import { initStatisticsContext, type StorageConfiguration } from '@hcengineering/server-core'
+import { join } from 'path'
+
+import { createElasticAdapter } from '@hcengineering/elastic'
+import { createRekoniAdapter, createYDocAdapter, type FulltextDBConfiguration } from '@hcengineering/server-indexer'
+import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
+import { readFileSync } from 'fs'
+import { startIndexer } from './server'
+
+const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[]
+
+const serverSecret = process.env.SERVER_SECRET
+if (serverSecret === undefined) {
+  console.info('Please provide server secret')
+  process.exit(1)
+}
+
+const metricsContext = initStatisticsContext('fulltext', {
+  factory: () =>
+    new MeasureMetricsContext(
+      'fulltext',
+      {},
+      {},
+      newMetrics(),
+      new SplitLogger('fulltext', {
+        root: join(process.cwd(), 'logs'),
+        enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true'
+      })
+    )
+})
+
+const sentryDSN = process.env.SENTRY_DSN
+
+configureAnalytics(sentryDSN, {})
+Analytics.setTag('application', 'fulltext')
+
+const dbURL = process.env.DB_URL
+if (dbURL === undefined) {
+  console.error('DB_URL should be specified')
+  process.exit(1)
+}
+
+const fullTextDbURL = process.env.FULLTEXT_DB_URL
+if (fullTextDbURL === undefined) {
+  console.error('FULLTEXT_DB_URL should be specified')
+  process.exit(1)
+}
+
+const rekoniUrl = process.env.REKONI_URL
+if (rekoniUrl === undefined) {
+  console.error('REKONI_URL should be specified')
+  process.exit(1)
+}
+
+const config: FulltextDBConfiguration = {
+  fulltextAdapter: {
+    factory: createElasticAdapter,
+    url: fullTextDbURL
+  },
+  contentAdapters: {
+    Rekoni: {
+      factory: createRekoniAdapter,
+      contentType: '*',
+      url: rekoniUrl
+    },
+    YDoc: {
+      factory: createYDocAdapter,
+      contentType: 'application/ydoc',
+      url: ''
+    }
+  },
+  defaultContentAdapter: 'Rekoni'
+}
+
+const elasticIndexName = process.env.ELASTIC_INDEX_NAME
+if (elasticIndexName === undefined) {
+  console.log('Please provide ELASTIC_INDEX_NAME')
+  process.exit(1)
+}
+
+const servicePort = parseInt(process.env.PORT ?? '4700')
+metricsContext.info('Starting stats service')
+
+const accountsUrl = process.env.ACCOUNTS_URL
+if (accountsUrl === undefined) {
+  console.error('please provide account url')
+  process.exit(1)
+}
+
+const storageConfig: StorageConfiguration = storageConfigFromEnv()
+const externalStorage = buildStorageFromConfig(storageConfig)
+
+const onClose = startIndexer(metricsContext, {
+  model,
+  config,
+  externalStorage,
+  elasticIndexName,
+  dbURL,
+  port: servicePort,
+  serverSecret,
+  accountsUrl
+})
+
+process.on('uncaughtException', (e) => {
+  metricsContext.error('uncaughtException', { error: e })
+})
+
+process.on('unhandledRejection', (reason, promise) => {
+  metricsContext.error('Unhandled Rejection at:', { reason, promise })
+})
+
+const close = (): void => {
+  void onClose.then((res) => {
+    res()
+  })
+}
+
+process.on('SIGINT', close)
+process.on('SIGTERM', close)
+process.on('exit', close)

pods/fulltext/src/server.ts

@@ -0,0 +1,400 @@
+/* eslint-disable @typescript-eslint/unbound-method */
+import { Analytics } from '@hcengineering/analytics'
+import type {
+  Class,
+  Doc,
+  DocumentQuery,
+  IndexingUpdateEvent,
+  MeasureContext,
+  Ref,
+  SearchOptions,
+  SearchQuery,
+  Tx,
+  TxWorkspaceEvent,
+  WorkspaceId,
+  WorkspaceIdWithUrl
+} from '@hcengineering/core'
+import core, {
+  DOMAIN_DOC_INDEX_STATE,
+  generateId,
+  Hierarchy,
+  ModelDb,
+  systemAccountEmail,
+  WorkspaceEvent
+} from '@hcengineering/core'
+import {
+  ContextNameMiddleware,
+  DBAdapterInitMiddleware,
+  DBAdapterMiddleware,
+  DomainFindMiddleware,
+  LowLevelMiddleware,
+  ModelMiddleware
+} from '@hcengineering/middleware'
+import { PlatformError, setMetadata, unknownError } from '@hcengineering/platform'
+import serverClientPlugin, { getTransactorEndpoint, getWorkspaceInfo } from '@hcengineering/server-client'
+import serverCore, {
+  createContentAdapter,
+  createPipeline,
+  type ContentTextAdapter,
+  type FullTextAdapter,
+  type MiddlewareCreator,
+  type Pipeline,
+  type PipelineContext,
+  type StorageAdapter
+} from '@hcengineering/server-core'
+import { FullTextIndexPipeline, searchFulltext, type FulltextDBConfiguration } from '@hcengineering/server-indexer'
+import { getConfig, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline'
+import serverToken, { decodeToken, generateToken, type Token } from '@hcengineering/server-token'
+import cors from '@koa/cors'
+import Koa from 'koa'
+import bodyParser from 'koa-bodyparser'
+import Router from 'koa-router'
+
+class WorkspaceIndexer {
+  fulltext!: FullTextIndexPipeline
+  pipeline!: Pipeline
+
+  lastUpdate: number = Date.now()
+
+  constructor (readonly fulltextAdapter: FullTextAdapter) {}
+
+  static async create (
+    ctx: MeasureContext,
+    model: Tx[],
+    workspace: WorkspaceIdWithUrl,
+    dbURL: string,
+    externalStorage: StorageAdapter,
+    ftadapter: FullTextAdapter,
+    contentAdapter: ContentTextAdapter
+  ): Promise<WorkspaceIndexer> {
+    const result = new WorkspaceIndexer(ftadapter)
+    const dbConf = getConfig(ctx, dbURL, ctx, {
+      disableTriggers: true,
+      externalStorage
+    })
+
+    const middlewares: MiddlewareCreator[] = [
+      LowLevelMiddleware.create,
+      ContextNameMiddleware.create,
+      DomainFindMiddleware.create,
+      DBAdapterInitMiddleware.create,
+      ModelMiddleware.create(model),
+      DBAdapterMiddleware.create(dbConf)
+    ]
+
+    const hierarchy = new Hierarchy()
+    const modelDb = new ModelDb(hierarchy)
+
+    const context: PipelineContext = {
+      workspace,
+      branding: null,
+      modelDb,
+      hierarchy,
+      storageAdapter: externalStorage
+    }
+    result.pipeline = await createPipeline(ctx, middlewares, context)
+
+    const defaultAdapter = result.pipeline.context.adapterManager?.getDefaultAdapter()
+    if (defaultAdapter === undefined) {
+      throw new PlatformError(unknownError('Default adapter should be set'))
+    }
+
+    const token = generateToken(systemAccountEmail, workspace)
+    const transactorEndpoint = (await getTransactorEndpoint(token, 'internal'))
+      .replace('wss://', 'https://')
+      .replace('ws://', 'http://')
+
+    result.fulltext = new FullTextIndexPipeline(
+      ftadapter,
+      defaultAdapter,
+      hierarchy,
+      workspace,
+      ctx,
+      modelDb,
+      externalStorage,
+      contentAdapter,
+      (ctx: MeasureContext, classes: Ref<Class<Doc>>[]) => {
+        ctx.info('broadcast indexing update', { classes, workspace })
+        const evt: IndexingUpdateEvent = {
+          _class: classes
+        }
+        const tx: TxWorkspaceEvent = {
+          _class: core.class.TxWorkspaceEvent,
+          _id: generateId(),
+          event: WorkspaceEvent.IndexingUpdate,
+          modifiedBy: core.account.System,
+          modifiedOn: Date.now(),
+          objectSpace: core.space.DerivedTx,
+          space: core.space.DerivedTx,
+          params: evt
+        }
+        // Send tx to pipeline
+        // TODO: Fix me
+        void fetch(transactorEndpoint + `/api/v1/broadcast?token=${token}&workspace=${workspace.name}`, {
+          method: 'PUT',
+          body: JSON.stringify(tx)
+        })
+      },
+      async () => {
+        const helper = result.pipeline.context.adapterManager?.domainHelper
+        if (helper !== undefined) {
+          const dhelper = result.pipeline.context.adapterManager?.getAdapter(DOMAIN_DOC_INDEX_STATE, true).helper?.()
+          if (dhelper !== undefined) {
+            // Force creation of indexes for search domain
+            await helper.checkDomain(ctx, DOMAIN_DOC_INDEX_STATE, 10000, dhelper)
+          }
+        }
+      },
+      async (ctx: MeasureContext) => {
+        await result.pipeline.context.adapterManager?.closeContext?.(ctx)
+      }
+    )
+    await result.fulltext.startIndexing(() => {
+      result.lastUpdate = Date.now()
+    })
+    return result
+  }
+
+  async close (): Promise<void> {
+    await this.fulltext.cancel()
+    await this.pipeline.close()
+  }
+}
+
+interface IndexDocuments {
+  token: string
+  requests: {
+    _class: Ref<Class<Doc>>
+    _id: Ref<Doc>
+  }[]
+}
+
+interface FulltextSearch {
+  token: string
+  query: SearchQuery
+  options: SearchOptions
+}
+
+interface Search {
+  token: string
+  _classes: Ref<Class<Doc>>[]
+  query: DocumentQuery<Doc>
+  fullTextLimit: number
+}
+
+export async function startIndexer (
+  ctx: MeasureContext,
+  opt: {
+    model: Tx[]
+    dbURL: string
+    config: FulltextDBConfiguration
+    externalStorage: StorageAdapter
+    elasticIndexName: string
+    port: number
+    serverSecret: string
+    accountsUrl: string
+  }
+): Promise<() => void> {
+  const closeTimeout = 5 * 60 * 1000
+
+  setMetadata(serverToken.metadata.Secret, opt.serverSecret)
+  setMetadata(serverCore.metadata.ElasticIndexName, opt.elasticIndexName)
+  setMetadata(serverClientPlugin.metadata.Endpoint, opt.accountsUrl)
+
+  registerServerPlugins()
+  registerStringLoaders()
+
+  const sysHierarchy = new Hierarchy()
+  for (const tx of opt.model) {
+    sysHierarchy.tx(tx)
+  }
+
+  const app = new Koa()
+  const router = new Router()
+
+  const indexers = new Map<string, WorkspaceIndexer | Promise<WorkspaceIndexer>>()
+
+  const contentAdapter = await ctx.with('create content adapter', {}, (ctx) =>
+    createContentAdapter(opt.config.contentAdapters, opt.config.defaultContentAdapter)
+  )
+  const fulltextAdapter = await opt.config.fulltextAdapter.factory(opt.config.fulltextAdapter.url)
+
+  const shutdownInterval = setInterval(() => {
+    for (const [k, v] of [...indexers.entries()]) {
+      if (v instanceof Promise) {
+        continue
+      }
+      if (Date.now() - v.lastUpdate > closeTimeout) {
+        indexers.delete(k)
+        void v.close()
+      }
+    }
+  }, closeTimeout) // Every 5 minutes we should close unused indexes.
+
+  async function getIndexer (
+    ctx: MeasureContext,
+    workspace: WorkspaceId,
+    token: string,
+    create: boolean = false
+  ): Promise<WorkspaceIndexer | undefined> {
+    const workspaceInfo = await getWorkspaceInfo(token)
+    let idx = indexers.get(workspace.name)
+    if (idx === undefined && create) {
+      if (workspaceInfo === undefined) {
+        ctx.error('Workspace not available for token')
+        return
+      }
+      idx = WorkspaceIndexer.create(
+        ctx,
+        opt.model,
+        {
+          ...workspace,
+          workspaceName: workspaceInfo.workspaceName ?? workspaceInfo.workspace,
+          workspaceUrl: workspaceInfo.workspaceUrl ?? workspaceInfo.workspace
+        },
+        opt.dbURL,
+        opt.externalStorage,
+        fulltextAdapter,
+        contentAdapter
+      )
+      indexers.set(workspace.name, idx)
+    }
+    if (idx instanceof Promise) {
+      idx = await idx
+      indexers.set(workspace.name, idx)
+    }
+    return idx
+  }
+
+  app.use(
+    cors({
+      credentials: true
+    })
+  )
+  app.use(bodyParser())
+
+  router.put('/api/v1/search', async (req, res) => {
+    try {
+      const request = req.request.body as Search
+      const decoded = decodeToken(request.token) // Just to be safe
+
+      ctx.info('search', { classes: request._classes, query: request.query, workspace: decoded.workspace })
+      await ctx.with('search', {}, async (ctx) => {
+        const docs = await ctx.with('search', { workspace: decoded.workspace.name }, (ctx) =>
+          fulltextAdapter.search(ctx, decoded.workspace, request._classes, request.query, request.fullTextLimit)
+        )
+        req.body = docs
+      })
+
+      void triggerIndexer(decoded, request.token)
+    } catch (err: any) {
+      Analytics.handleError(err)
+      console.error(err)
+      req.res.writeHead(404, {})
+      req.res.end()
+    }
+  })
+
+  router.put('/api/v1/full-text-search', async (req, res) => {
+    try {
+      const request = req.request.body as FulltextSearch
+      const decoded = decodeToken(request.token) // Just to be safe
+      ctx.info('fulltext-search', { ...request.query, workspace: decoded.workspace })
+      await ctx.with('full-text-search', {}, async (ctx) => {
+        const result = await ctx.with('searchFulltext', {}, (ctx) =>
+          searchFulltext(ctx, decoded.workspace, sysHierarchy, fulltextAdapter, request.query, request.options)
+        )
+        req.body = result
+      })
+
+      void triggerIndexer(decoded, request.token)
+    } catch (err: any) {
+      Analytics.handleError(err)
+      console.error(err)
+      req.res.writeHead(404, {})
+      req.res.end()
+    }
+  })
+
+  router.put('/api/v1/warmup', async (req, res) => {
+    try {
+      const request = req.request.body as IndexDocuments
+      const decoded = decodeToken(request.token) // Just to be safe
+      req.body = {}
+
+      ctx.info('warm-up', { workspace: decoded.workspace })
+      void triggerIndexer(decoded, request.token, true)
+    } catch (err: any) {
+      Analytics.handleError(err)
+      console.error(err)
+      req.res.writeHead(404, {})
+      req.res.end()
+    }
+  })
+
+  router.put('/api/v1/close', async (req, res) => {
+    try {
+      const request = req.request.body as IndexDocuments
+      const decoded = decodeToken(request.token) // Just to be safe
+      req.body = {}
+
+      ctx.info('close', { workspace: decoded.workspace })
+      const idx = indexers.get(decoded.workspace.name)
+      indexers.delete(decoded.workspace.name)
+      if (idx !== undefined && idx instanceof Promise) {
+        void idx.then((res) => {
+          void res.close()
+        })
+      } else if (idx !== undefined) {
+        void idx.close()
+      }
+    } catch (err: any) {
+      Analytics.handleError(err)
+      console.error(err)
+      req.res.writeHead(404, {})
+      req.res.end()
+    }
+  })
+
+  router.put('/api/v1/index-documents', async (req, res) => {
+    try {
+      const request = req.request.body as IndexDocuments
+      const decoded = decodeToken(request.token) // Just to be safe
+
+      const indexer = await getIndexer(ctx, decoded.workspace, request.token)
+      if (indexer !== undefined) {
+        indexer.lastUpdate = Date.now()
+        await ctx.with('index-documents', {}, (ctx) => indexer.fulltext.indexDocuments(ctx, request.requests))
+      }
+      req.body = {}
+    } catch (err: any) {
+      Analytics.handleError(err)
+      console.error(err)
+      req.res.writeHead(404, {})
+      req.res.end()
+    }
+  })
+
+  app.use(router.routes()).use(router.allowedMethods())
+
+  const server = app.listen(opt.port, () => {
+    console.log(`server started on port ${opt.port}`)
+  })
+
+  const close = (): void => {
+    clearInterval(shutdownInterval)
+    server.close()
+  }
+
+  return close
+
+  async function triggerIndexer (decoded: Token, token: string, trigger: boolean = false): Promise<void> {
+    const indexer = await getIndexer(ctx, decoded.workspace, token, trigger)
+    if (indexer !== undefined) {
+      indexer.lastUpdate = Date.now()
+      if (trigger) {
+        indexer.fulltext.triggerIndexing()
+      }
+    }
+  }
+}

pods/fulltext/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json",
+
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./lib",
+    "declarationDir": "./types",
+    "tsBuildInfoFile": ".build/build.tsbuildinfo"
+  }
+}

pods/server/package.json

@@ -8,9 +8,9 @@
   "template": "@hcengineering/node-package",
   "license": "EPL-2.0",
   "scripts": {
-    "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false MODEL_JSON=../../models/all/bundle/model.json node --inspect bundle/bundle.js",
-    "start-u": "rush bundle --to @hcengineering/pod-server && ./bundle/ && cross-env NODE_ENV=production SERVER_PROVIDER=uweb ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json node bundle/bundle.js",
-    "start-flame": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json clinic flame --dest ./out -- node --nolazy -r ts-node/register --enable-source-maps src/__start.ts",
+    "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false MODEL_JSON=../../models/all/bundle/model.json node --inspect bundle/bundle.js",
+    "start-u": "rush bundle --to @hcengineering/pod-server && ./bundle/ && cross-env NODE_ENV=production MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json node bundle/bundle.js",
+    "start-flame": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json clinic flame --dest ./out -- node --nolazy -r ts-node/register --enable-source-maps src/__start.ts",
     "build": "compile",
     "_phase:bundle": "rushx bundle",
     "_phase:docker-build": "rushx docker:build",
@@ -61,7 +61,6 @@
     "@hcengineering/server-storage": "^0.6.0",
     "@hcengineering/server-pipeline": "^0.6.0",
     "@hcengineering/mongo": "^0.6.1",
-    "@hcengineering/elastic": "^0.6.0",
     "@hcengineering/server-token": "^0.6.11",
     "@hcengineering/middleware": "^0.6.0",
     "@hcengineering/minio": "^0.6.0",

pods/server/src/__start.ts

@@ -68,20 +68,16 @@ setMetadata(serverNotification.metadata.SesUrl, config.sesUrl ?? '')
 setMetadata(notification.metadata.PushPublicKey, config.pushPublicKey)
 setMetadata(serverNotification.metadata.PushPrivateKey, config.pushPrivateKey)
 setMetadata(serverNotification.metadata.PushSubject, config.pushSubject)
-setMetadata(serverCore.metadata.ElasticIndexName, config.elasticIndexName)
 setMetadata(serverCore.metadata.ElasticIndexVersion, 'v1')
 setMetadata(serverTelegram.metadata.BotUrl, process.env.TELEGRAM_BOT_URL)
 setMetadata(serverAiBot.metadata.SupportWorkspaceId, process.env.SUPPORT_WORKSPACE)
 setMetadata(serverAiBot.metadata.EndpointURL, process.env.AI_BOT_URL)
 
 const { shutdown, sessionManager } = start(metricsContext, config.dbUrl, {
-  fullTextUrl: config.elasticUrl,
+  fulltextUrl: config.fulltextUrl,
   storageConfig,
-  rekoniUrl: config.rekoniUrl,
   port: config.serverPort,
   serverFactory: startHttpServer,
-  indexParallel: 2,
-  indexProcessing: 500,
   brandingMap: loadBrandingMap(config.brandingPath),
   accountsUrl: config.accountsUrl,
   enableCompression: config.enableCompression,

pods/server/src/server.ts

@@ -47,16 +47,12 @@ export function start (
   metrics: MeasureContext,
   dbUrl: string,
   opt: {
-    fullTextUrl: string
+    fulltextUrl: string
     storageConfig: StorageConfiguration
-    rekoniUrl: string
     port: number
     brandingMap: BrandingMap
     serverFactory: ServerFactory
 
-    indexProcessing: number // 1000
-    indexParallel: number // 2
-
     enableCompression?: boolean
 
     accountsUrl: string

qms-tests/docker-compose.yaml

@@ -116,7 +116,6 @@ services:
     pull_policy: never
     links:
       - mongodb
-      - elastic
       - minio
       - rekoni
       - account
@@ -128,7 +127,6 @@ services:
       - SERVER_PROVIDER=${SERVER_PROVIDER}
       - SERVER_PORT=3334
       - SERVER_SECRET=secret
-      - ELASTIC_URL=http://elastic:9200
       - DB_URL=mongodb://mongodb:27018
       - MONGO_URL=mongodb://mongodb:27018
       - METRICS_CONSOLE=false
@@ -139,7 +137,7 @@ services:
       - UPLOAD_URL=http://host.docker.internal:8083/files
       - ACCOUNTS_URL=http://account:3003
       - LAST_NAME_FIRST=true
-      - ELASTIC_INDEX_NAME=local_storage_index
+      - FULLTEXT_URL=http://fulltext:4710
       - BRANDING_PATH=/var/cfg/branding-test.json
   collaborator:
     image: hardcoreeng/collaborator
@@ -203,5 +201,21 @@ services:
       resources:
         limits:
           memory: 300M
-volumes:
-  files:
+  fulltext:
+    image: hardcoreeng/fulltext
+    restart: unless-stopped
+    links:
+      - elastic
+      - mongodb
+      - rekoni
+    ports:
+      - 4710:4710
+    environment:
+      - PORT=4710
+      - SERVER_SECRET=secret
+      - DB_URL=mongodb://mongodb:27018
+      - FULLTEXT_DB_URL=http://elastic:9200
+      - ELASTIC_INDEX_NAME=local_storage_index
+      - STORAGE_CONFIG=${STORAGE_CONFIG}
+      - REKONI_URL=http://rekoni:4007
+      - ACCOUNTS_URL=http://account:3003          

rush.json

@@ -1441,6 +1441,26 @@
       "projectFolder": "pods/backup",
       "shouldPublish": false
     },
+    {
+      "packageName": "@hcengineering/pod-fulltext",
+      "projectFolder": "pods/fulltext",
+      "shouldPublish": false
+    },
+    {
+      "packageName": "@hcengineering/server-fulltext",
+      "projectFolder": "server-plugins/fulltext",
+      "shouldPublish": false
+    },
+    {
+      "packageName": "@hcengineering/server-fulltext-resources",
+      "projectFolder": "server-plugins/fulltext-resources",
+      "shouldPublish": false
+    },
+    {
+      "packageName": "@hcengineering/model-server-fulltext",
+      "projectFolder": "models/server-fulltext",
+      "shouldPublish": false
+    },
     {
       "packageName": "@hcengineering/hr",
       "projectFolder": "plugins/hr",

server-plugins/collaboration/src/fulltext.ts

@@ -1,165 +0,0 @@
-//
-// Copyright © 2024 Hardcore Engineering Inc.
-//
-// Licensed under the Eclipse Public License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License. You may
-// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-import core, {
-  Blob,
-  Class,
-  CollaborativeDoc,
-  Doc,
-  DocIndexState,
-  DocumentQuery,
-  DocumentUpdate,
-  MeasureContext,
-  Ref,
-  WorkspaceId,
-  collaborativeDocParse,
-  getFullTextIndexableAttributes
-} from '@hcengineering/core'
-import { ContentTextAdapter, DbAdapter, IndexedDoc, StorageAdapter } from '@hcengineering/server-core'
-
-import {
-  DocUpdateHandler,
-  FullTextPipeline,
-  FullTextPipelineStage,
-  collabStageId,
-  contentStageId,
-  docKey,
-  docUpdKey,
-  fieldStateId
-} from '@hcengineering/server-indexer'
-
-/**
- * @public
- */
-export class CollaborativeContentRetrievalStage implements FullTextPipelineStage {
-  require = []
-  stageId = collabStageId
-
-  extra = ['content', 'base64']
-
-  enabled = true
-
-  // Clear all except following.
-  clearExcept: string[] = [fieldStateId, contentStageId]
-
-  updateFields: DocUpdateHandler[] = []
-
-  textLimit = 100 * 1024
-
-  constructor (
-    readonly storageAdapter: StorageAdapter | undefined,
-    readonly workspace: WorkspaceId,
-    readonly metrics: MeasureContext,
-    private readonly contentAdapter: ContentTextAdapter
-  ) {}
-
-  async initialize (ctx: MeasureContext, storage: DbAdapter, pipeline: FullTextPipeline): Promise<void> {
-    // Just do nothing
-  }
-
-  async search (
-    _classes: Ref<Class<Doc>>[],
-    search: DocumentQuery<Doc>,
-    size?: number,
-    from?: number
-  ): Promise<{ docs: IndexedDoc[], pass: boolean }> {
-    return { docs: [], pass: true }
-  }
-
-  async collect (toIndex: DocIndexState[], pipeline: FullTextPipeline): Promise<void> {
-    for (const doc of toIndex) {
-      if (pipeline.cancelling) {
-        return
-      }
-      await this.updateContent(doc, pipeline)
-    }
-  }
-
-  async updateContent (doc: DocIndexState, pipeline: FullTextPipeline): Promise<void> {
-    const attributes = getFullTextIndexableAttributes(pipeline.hierarchy, doc.objectClass)
-    // Copy content attributes as well.
-    const update: DocumentUpdate<DocIndexState> = {}
-
-    if (pipeline.cancelling) {
-      return
-    }
-
-    try {
-      for (const [, val] of Object.entries(attributes)) {
-        if (val.type._class === core.class.TypeCollaborativeDoc) {
-          const collaborativeDoc = doc.attributes[docKey(val.name, { _class: val.attributeOf })] as CollaborativeDoc
-          if (collaborativeDoc !== undefined && collaborativeDoc !== '') {
-            const { documentId } = collaborativeDocParse(collaborativeDoc)
-
-            const docInfo: Blob | undefined = await this.storageAdapter?.stat(this.metrics, this.workspace, documentId)
-
-            if (docInfo !== undefined) {
-              const digest = docInfo.etag
-              const digestKey = docKey(val.name, { _class: val.attributeOf, digest: true })
-              if (doc.attributes[digestKey] !== digest) {
-                ;(update as any)[docUpdKey(digestKey)] = digest
-
-                const contentType = (docInfo.contentType ?? '').split(';')[0]
-                const readable = await this.storageAdapter?.get(this.metrics, this.workspace, documentId)
-
-                if (readable !== undefined) {
-                  let textContent = await this.metrics.with(
-                    'fetch',
-                    {},
-                    async () => await this.contentAdapter.content(documentId, contentType, readable)
-                  )
-                  readable?.destroy()
-
-                  textContent = textContent
-                    .split(/ +|\t+|\f+/)
-                    .filter((it) => it)
-                    .join(' ')
-                    .split(/\n\n+/)
-                    .join('\n')
-
-                  // trim to large content
-                  if (textContent.length > this.textLimit) {
-                    textContent = textContent.slice(0, this.textLimit)
-                  }
-                  textContent = Buffer.from(textContent).toString('base64')
-                  ;(update as any)[docUpdKey(val.name, { _class: val.attributeOf, extra: this.extra })] = textContent
-                }
-              }
-            }
-          }
-        }
-      }
-    } catch (err: any) {
-      const wasError = (doc as any).error !== undefined
-
-      await pipeline.update(doc._id, false, { [docKey('error')]: JSON.stringify({ message: err.message, err }) })
-      if (wasError) {
-        return
-      }
-      // Print error only first time, and update it in doc index
-      console.error(err)
-      return
-    }
-
-    await pipeline.update(doc._id, true, update)
-  }
-
-  async remove (docs: DocIndexState[], pipeline: FullTextPipeline): Promise<void> {
-    // will be handled by field processor
-    for (const doc of docs) {
-      await pipeline.update(doc._id, true, {})
-    }
-  }
-}

server-plugins/collaboration/src/index.ts

@@ -17,8 +17,6 @@ import type { Plugin, Resource } from '@hcengineering/platform'
 import { plugin } from '@hcengineering/platform'
 import type { TriggerFunc } from '@hcengineering/server-core'
 
-export * from './fulltext'
-
 /**
  * @public
  */

server-plugins/contact-resources/src/index.ts

@@ -31,7 +31,6 @@ import contact, {
 } from '@hcengineering/contact'
 import core, {
   Account,
-  Class,
   Doc,
   Hierarchy,
   Ref,
@@ -43,7 +42,8 @@ import core, {
   TxProcessor,
   TxRemoveDoc,
   TxUpdateDoc,
-  concatLink
+  concatLink,
+  type Space
 } from '@hcengineering/core'
 import notification, { Collaborators } from '@hcengineering/notification'
 import { getMetadata } from '@hcengineering/platform'
@@ -285,9 +285,17 @@ export function organizationTextPresenter (doc: Doc): string {
 /**
  * @public
  */
-export function contactNameProvider (hierarchy: Hierarchy, props: Record<string, string>): string {
-  const _class = props._class !== undefined ? (props._class as Ref<Class<Doc>>) : contact.class.Contact
-  return formatContactName(hierarchy, _class, props.name ?? '', props.lastNameFirst)
+export function contactNameProvider (
+  doc: Doc,
+  parent: Doc | undefined,
+  space: Space | undefined,
+  hierarchy: Hierarchy,
+  mode: string
+): string {
+  if (parent !== undefined && hierarchy.isDerived(parent._class, contact.class.Contact)) {
+    return formatContactName(hierarchy, parent._class, (parent as Contact).name ?? '', mode)
+  }
+  return formatContactName(hierarchy, doc._class, (doc as Contact).name ?? '', mode)
 }
 
 export async function getCurrentEmployeeName (control: TriggerControl, context: Record<string, Doc>): Promise<string> {

server-plugins/fulltext-resources/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+  extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: './tsconfig.json'
+  }
+}

server-plugins/fulltext-resources/.npmignore

@@ -0,0 +1,4 @@
+*
+!/lib/**
+!CHANGELOG.md
+/lib/**/__tests__/

server-plugins/fulltext-resources/config/rig.json

@@ -0,0 +1,4 @@
+{  
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+  "rigPackageName": "@hcengineering/platform-rig"
+}

server-plugins/fulltext-resources/jest.config.js

@@ -0,0 +1,7 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
+  roots: ["./src"],
+  coverageReporters: ["text-summary", "html"]
+}

server-plugins/fulltext-resources/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@hcengineering/server-fulltext-resources",
+  "version": "0.6.0",
+  "main": "lib/index.js",
+  "svelte": "src/index.ts",
+  "types": "types/index.d.ts",
+  "files": [
+    "lib/**/*",
+    "types/**/*",
+    "tsconfig.json"
+  ],
+  "author": "Anticrm Platform Contributors",
+  "license": "EPL-2.0",
+  "scripts": {
+    "build": "compile",
+    "build:watch": "compile",
+    "format": "format src",
+    "test": "jest --passWithNoTests --silent",
+    "_phase:build": "compile transpile src",
+    "_phase:test": "jest --passWithNoTests --silent",
+    "_phase:format": "format src",
+    "_phase:validate": "compile validate"
+  },
+  "devDependencies": {
+    "@hcengineering/platform-rig": "^0.6.0",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint": "^8.54.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "prettier": "^3.1.0",
+    "typescript": "^5.3.3",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.1",
+    "@types/jest": "^29.5.5"
+  },
+  "dependencies": {
+    "@hcengineering/core": "^0.6.32",
+    "@hcengineering/platform": "^0.6.11",
+    "@hcengineering/server-core": "^0.6.1"
+  }
+}