diff --git a/.vscode/launch.json b/.vscode/launch.json index 64dd01593a..d00b23d65a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -34,7 +34,7 @@ "request": "launch", "args": ["src/__start.ts"], "env": { - "ELASTIC_URL": "http://localhost:9200", + "FULLTEXT_URL": "http://localhost:4700", "MONGO_URL": "mongodb://localhost:27017", "DB_URL": "mongodb://localhost:27017", // "DB_URL": "postgresql://postgres:example@localhost:5432", @@ -65,6 +65,34 @@ "cwd": "${workspaceRoot}/pods/server", "protocol": "inspector" }, + { + "name": "Debug Fulltext", + "type": "node", + "request": "launch", + "args": ["src/index.ts"], + "env": { + "PORT": "4700",// For mongo + // "PORT": "4701", // for pg + "FULLTEXT_DB_URL": "http://localhost:9200", + "DB_URL": "mongodb://localhost:27017", + // "DB_URL": "postgresql://postgres:example@localhost:5432", + "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin", + "SERVER_SECRET": "secret", + "REKONI_URL": "http://localhost:4004", + "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", + "REGION":"pg", + "ELASTIC_INDEX_NAME": "local_storage_index", + "STATS_URL":"http://host.docker.internal:4900", + "ACCOUNTS_URL": "http://localhost:3000", + }, + "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], + "runtimeVersion": "20", + "showAsyncStacks": true, + "outputCapture": "std", + "sourceMaps": true, + "cwd": "${workspaceRoot}/pods/fulltext", + "protocol": "inspector" + }, { "name": "Debug Account", "type": "node", diff --git a/common/config/rush/command-line.json b/common/config/rush/command-line.json index c6e75745b4..137cedd174 100644 --- a/common/config/rush/command-line.json +++ b/common/config/rush/command-line.json @@ -238,7 +238,7 @@ "summary": "Build docker with platform", "description": "use to build all docker containers required for platform", "safeForSimultaneousRushProcesses": true, - "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats" + "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext" }, { "commandKind": "global", diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 8d0d9c7d4a..43d4561238 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -21,8 +21,8 @@ dependencies: specifier: ^4.20240729.0 version: 4.20241004.0 '@elastic/elasticsearch': - specifier: ^7.14.0 - version: 7.17.13 + specifier: ^7.17.14 + version: 7.17.14 '@electron/notarize': specifier: ^2.3.2 version: 2.3.2 @@ -467,6 +467,9 @@ dependencies: '@rush-temp/model-server-drive': specifier: file:./projects/model-server-drive.tgz version: file:projects/model-server-drive.tgz + '@rush-temp/model-server-fulltext': + specifier: file:./projects/model-server-fulltext.tgz + version: file:projects/model-server-fulltext.tgz '@rush-temp/model-server-gmail': specifier: file:./projects/model-server-gmail.tgz version: file:projects/model-server-gmail.tgz @@ -617,6 +620,9 @@ dependencies: '@rush-temp/pod-front': specifier: file:./projects/pod-front.tgz version: file:projects/pod-front.tgz + '@rush-temp/pod-fulltext': + specifier: file:./projects/pod-fulltext.tgz + version: file:projects/pod-fulltext.tgz '@rush-temp/pod-github': specifier: file:./projects/pod-github.tgz version: file:projects/pod-github.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-prosemirror@1.2.12) @@ -812,6 +818,12 @@ dependencies: '@rush-temp/server-drive-resources': specifier: file:./projects/server-drive-resources.tgz version: file:projects/server-drive-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2) + '@rush-temp/server-fulltext': + specifier: file:./projects/server-fulltext.tgz + version: file:projects/server-fulltext.tgz(esbuild@0.20.1)(ts-node@10.9.2) + '@rush-temp/server-fulltext-resources': + specifier: file:./projects/server-fulltext-resources.tgz + version: file:projects/server-fulltext-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2) '@rush-temp/server-github': specifier: file:./projects/server-github.tgz version: file:projects/server-github.tgz(esbuild@0.20.1)(ts-node@10.9.2) @@ -3071,11 +3083,11 @@ packages: '@elastic/ecs-helpers': 2.1.1 dev: false - /@elastic/elasticsearch@7.17.13: - resolution: {integrity: sha512-GMXtFVqd3FgUlTtPL/GDc+3GhwvfZ0kSuegCvVVqb58kd+0I6U6u7PL8QFRLHtwzqLEBmYLdwr4PRkBAWKGlzA==} + /@elastic/elasticsearch@7.17.14: + resolution: {integrity: sha512-6uQ1pVXutwz1Krwooo67W+3K8BwH1ASMh1WoHTpomUzw8EXecXN5lHIJ9EPqTHuv1WqR2LKkSJyagcq0HYUJpg==} engines: {node: '>=12'} dependencies: - debug: 4.3.4 + debug: 4.3.5 hpagent: 0.1.2 ms: 2.1.3 secure-json-parse: 2.7.0 @@ -23193,12 +23205,12 @@ packages: dev: false file:projects/elastic.tgz(@types/node@20.11.19)(esbuild@0.20.1): - resolution: {integrity: sha512-pNrJ3byQAXmu2LIoiS7ph3F/dImZtZBe/tSV3ATcOLuPiPxtd3f+3rMK6Ajszw1mxvlVdXBiDtHcaL3iwdx5ZQ==, tarball: file:projects/elastic.tgz} + resolution: {integrity: sha512-WGiDJdr4on2TaKPfaKntlenxzUJ5uuxtibsd/ZkppUh0BghpFSH37C8WGps7kQRS4oqpb+mB3QhxVhxIz2IvGg==, tarball: file:projects/elastic.tgz} id: file:projects/elastic.tgz name: '@rush-temp/elastic' version: 0.0.0 dependencies: - '@elastic/elasticsearch': 7.17.13 + '@elastic/elasticsearch': 7.17.14 '@types/jest': 29.5.12 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) @@ -23226,7 +23238,7 @@ packages: dev: false file:projects/front.tgz(esbuild@0.20.1): - resolution: {integrity: sha512-kbnGRdB+2sGaVlIVQlK7RJo8OwjxEgTaNzWOg+3CSzpLuJFGE29mFog3D7BB2Q2g7hkEip1B6SiXHR/leCVvVQ==, tarball: file:projects/front.tgz} + resolution: {integrity: sha512-Y14f9YJTEcruCyze5lu63lxO9ODS2tUcO4WvCVNnRlro0vT13XvmOw+eDiTvohHI2aHROX+NuJuIEtGlzyqLBw==, tarball: file:projects/front.tgz} id: file:projects/front.tgz name: '@rush-temp/front' version: 0.0.0 @@ -24355,7 +24367,7 @@ packages: dev: false file:projects/middleware.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-+LHNVG0cNOCtW49PLe0rmRphnhY2+TLFzsqVmzNahdBTv3PHdSQ0YtW94U8eomL9gd6h1hWnMjNlfbWlFKGEwA==, tarball: file:projects/middleware.tgz} + resolution: {integrity: sha512-XcnDd4BMYa4b8U2G4S6Hr2OAIMGQFNwzs8/EYYaEp+dmEx9lSibJByGSfaeZaMkrbVxO6auouuC1rWuZnmiieg==, tarball: file:projects/middleware.tgz} id: file:projects/middleware.tgz name: '@rush-temp/middleware' version: 0.0.0 @@ -24456,7 +24468,7 @@ packages: dev: false file:projects/model-all.tgz: - resolution: {integrity: sha512-FHdenDxGsdlbQ6948ecdTtmUq5jHHwbY/ZXllcSYgV5WoGxFG/kJvPH6mqrh2y5MMNDIkcQTkf2J65mAxxdL8w==, tarball: file:projects/model-all.tgz} + resolution: {integrity: sha512-lZSE4CGfPJX/cTsbqnSoz/ge496Cwl3zeRFHM2wrpgfCFyeCNp00xpRv62LPGe38dvHo/IIzy+Uu0RSVPHUOPw==, tarball: file:projects/model-all.tgz} name: '@rush-temp/model-all' version: 0.0.0 dependencies: @@ -25212,6 +25224,24 @@ packages: - supports-color dev: false + file:projects/model-server-fulltext.tgz: + resolution: {integrity: sha512-qOs1G3E2P64j6xsyD1w2+ZPLclBeAyx1xmGEI19kneyA8IB8NNRcpI5GIB9vIrYJ2lJUi0jyI+sHkLoxmgtBaA==, tarball: file:projects/model-server-fulltext.tgz} + name: '@rush-temp/model-server-fulltext' + version: 0.0.0 + dependencies: + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + prettier: 3.2.5 + typescript: 5.6.2 + transitivePeerDependencies: + - supports-color + dev: false + file:projects/model-server-gmail.tgz: resolution: {integrity: sha512-vdwW7KKXUWLji9gEb2uEGwbjyJfeQr2g4G/vhZH15sy87rI5f7+M7x5p7zZq5IOF9dk6YArAbILD/yl7u+E3Xg==, tarball: file:projects/model-server-gmail.tgz} name: '@rush-temp/model-server-gmail' @@ -26539,7 +26569,7 @@ packages: dev: false file:projects/pod-front.tgz: - resolution: {integrity: sha512-YkIJf+SF6JuDvwC0ANFW/F/DLjC7M/5vm+aXzIK/CJUnojLex6nCeyDNzLqfl0E3oecLOKz13pRHsDzgNHYAGQ==, tarball: file:projects/pod-front.tgz} + resolution: {integrity: sha512-717gPfBibQBUuH6G8gN+SGt3xRVSHqMAc4pq2R+xmc4y0bb8pmROILpwF5pdLWzvIwtc0VeLKbk0fNxD+jeQ7A==, tarball: file:projects/pod-front.tgz} name: '@rush-temp/pod-front' version: 0.0.0 dependencies: @@ -26585,6 +26615,47 @@ packages: - supports-color dev: false + file:projects/pod-fulltext.tgz: + resolution: {integrity: sha512-TNpmIc39Ot7eD4avFPov9XJHNfcOjSAVFKtWlkkeWSBLte8BdQ2zYC2JZ3+Tb8Dzu1nX01UQm5WvRVWJY78R3A==, tarball: file:projects/pod-fulltext.tgz} + name: '@rush-temp/pod-fulltext' + version: 0.0.0 + dependencies: + '@koa/cors': 5.0.0 + '@types/jest': 29.5.12 + '@types/koa': 2.15.0 + '@types/koa-bodyparser': 4.3.12 + '@types/koa-router': 7.4.8 + '@types/koa__cors': 5.0.0 + '@types/node': 20.11.19 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) + cross-env: 7.0.3 + dotenv: 16.0.3 + esbuild: 0.20.1 + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) + koa: 2.15.3 + koa-bodyparser: 4.4.1 + koa-router: 12.0.1 + prettier: 3.2.5 + ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.6.2) + ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.6.2) + typescript: 5.6.2 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - '@swc/core' + - '@swc/wasm' + - babel-jest + - babel-plugin-macros + - node-notifier + - supports-color + dev: false + file:projects/pod-github.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-prosemirror@1.2.12): resolution: {integrity: sha512-jSnbzFjyWZA1Hnjtyotd09jvLvQ+oGX0V4UOaXqZfyxM65JFeoit/DSrKlQTWJjSCbw9y6S90CYZN+pK3h8InA==, tarball: file:projects/pod-github.tgz} id: file:projects/pod-github.tgz @@ -26825,7 +26896,7 @@ packages: dev: false file:projects/pod-server.tgz(utf-8-validate@6.0.4): - resolution: {integrity: sha512-boGyT8HkEJkWqcXXAAZ869HB68/GpHcMTUBZlhkUS9oztHADMfpAbGQ+yVVvypye4e9nsYTF1LhpsDrToOtPYw==, tarball: file:projects/pod-server.tgz} + resolution: {integrity: sha512-BV8NwL5NYGZtthvNwCy8vAMyZNBjbT8FmzRMaKq40wTer+rQ7T0xDrl96WBsIVsyv60knHY31oXjYkjE4P3M6A==, tarball: file:projects/pod-server.tgz} id: file:projects/pod-server.tgz name: '@rush-temp/pod-server' version: 0.0.0 @@ -28443,7 +28514,7 @@ packages: dev: false file:projects/server-backup.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-oDyT5+30r1kpzb+Cx/HTsY1f4fRjjtSy7n13JjzYPmkn37nVcHYb1X1RImHxhoiPehYWdftTsE5v5l2on0mOPg==, tarball: file:projects/server-backup.tgz} + resolution: {integrity: sha512-e0MNgA1LeSOikxK3b6X/HMuzpyQ4MjAoJtHi84X28nxD+49sEy1NctCTptyqz2qWWicAZAghKV46Qi9N9+RnEw==, tarball: file:projects/server-backup.tgz} id: file:projects/server-backup.tgz name: '@rush-temp/server-backup' version: 0.0.0 @@ -28977,6 +29048,66 @@ packages: - ts-node dev: false + file:projects/server-fulltext-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2): + resolution: {integrity: sha512-NebNMPNgIeG8N0qA5Q4ROcs5Jn/dmE0WX92uxw/qVUO/k8DLMxRsrFvyKrrMGtYgeivbxJPe6PEd3FijUAMfiw==, tarball: file:projects/server-fulltext-resources.tgz} + id: file:projects/server-fulltext-resources.tgz + name: '@rush-temp/server-fulltext-resources' + version: 0.0.0 + dependencies: + '@types/jest': 29.5.12 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) + prettier: 3.2.5 + ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.6.2) + typescript: 5.6.2 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - '@types/node' + - babel-jest + - babel-plugin-macros + - esbuild + - node-notifier + - supports-color + - ts-node + dev: false + + file:projects/server-fulltext.tgz(esbuild@0.20.1)(ts-node@10.9.2): + resolution: {integrity: sha512-QYDVer6XKHqKih+CXofNkysDl9TafFIlN5sw4GdJjLT3/AJt7L2FeCKMuPgMtYQhihj2kodtBfNUbnSeLA8qJw==, tarball: file:projects/server-fulltext.tgz} + id: file:projects/server-fulltext.tgz + name: '@rush-temp/server-fulltext' + version: 0.0.0 + dependencies: + '@types/jest': 29.5.12 + '@types/node': 20.11.19 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) + prettier: 3.2.5 + ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.6.2) + typescript: 5.6.2 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - babel-jest + - babel-plugin-macros + - esbuild + - node-notifier + - supports-color + - ts-node + dev: false + file:projects/server-github-model.tgz: resolution: {integrity: sha512-1qQ8IXhyHSiV5P+M7sGq0QHkkAG2n2DK3knDQJv/xzOQamJ6kcr2nlaqtAD48LfO1wVTtUkjwYwE9RbAEgpl5w==, tarball: file:projects/server-github-model.tgz} name: '@rush-temp/server-github-model' @@ -29239,7 +29370,7 @@ packages: dev: false file:projects/server-indexer.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-rMMlimV5wzW1ZPKeGdkdpeODEn0cyYDQJWti1GmagrteQed+ZH0DL51uPP0z7aioLG/ODxgFZKj8GWD+tMSrnA==, tarball: file:projects/server-indexer.tgz} + resolution: {integrity: sha512-H/iX6BGnUy04J35NoefNnWyoCqEd5RFve5SMXioIuA4tWqglHcAz8JqyPPLlfN+onrsh4Pz0tTVRV6lOZXSXWw==, tarball: file:projects/server-indexer.tgz} id: file:projects/server-indexer.tgz name: '@rush-temp/server-indexer' version: 0.0.0 @@ -29520,7 +29651,7 @@ packages: dev: false file:projects/server-pipeline.tgz: - resolution: {integrity: sha512-gaF/29tMBmt1rcHqrdJ87g7xHkb39eb8oZfDEVzyeK7qwf1vRjOYE0el4Pp/5OvkBrZSQGEwKgfUK3lHI3iLJQ==, tarball: file:projects/server-pipeline.tgz} + resolution: {integrity: sha512-VCAeDWNsMsyk2flDq7dhTEuS204IxDIMmB8t44NfPWwciiA/9dwla/bq1Fewnmb/28XS1xdw0UuE+X+qX0AHGA==, tarball: file:projects/server-pipeline.tgz} name: '@rush-temp/server-pipeline' version: 0.0.0 dependencies: @@ -31515,12 +31646,12 @@ packages: dev: false file:projects/tool.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4): - resolution: {integrity: sha512-Xiug9crD6dZ0BSoLCOQVEQE05/LitZ2QEPOeqZP0k9K31mffLqyYbCZfbRMvWMzBoFolNR8yUC38wIPvkRal1g==, tarball: file:projects/tool.tgz} + resolution: {integrity: sha512-VMtn5vIgitHobfk3a7XXFR5cr1nCbF/KasUqkn+o+EBDB3H+1n9krwH41lyS9KRHDe6HelYQVcBAPbCn87oPcA==, tarball: file:projects/tool.tgz} id: file:projects/tool.tgz name: '@rush-temp/tool' version: 0.0.0 dependencies: - '@elastic/elasticsearch': 7.17.13 + '@elastic/elasticsearch': 7.17.14 '@types/jest': 29.5.12 '@types/mime-types': 2.1.4 '@types/minio': 7.0.18 diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 2878dc9f77..1ac0aa0855 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -99,8 +99,6 @@ services: - "host.docker.internal:host-gateway" ports: - 4900:4900 - volumes: - - ./branding.json:/var/cfg/branding.json environment: - PORT=4900 - SERVER_SECRET=secret @@ -222,7 +220,6 @@ services: - "host.docker.internal:host-gateway" links: - mongodb - - elastic - minio - rekoni - account @@ -240,21 +237,19 @@ services: - SERVER_SECRET=secret - ENABLE_COMPRESSION=true - STATS_URL=http://host.docker.internal:4900 - - ELASTIC_URL=http://host.docker.internal:9200 + - FULLTEXT_URL=http://host.docker.internal:4700 # - DB_URL=postgresql://postgres:example@postgres:5432 - DB_URL=${MONGO_URL} - MONGO_URL=${MONGO_URL} - 'MONGO_OPTIONS={"appName": "transactor", "maxPoolSize": 10}' - METRICS_CONSOLE=false - METRICS_FILE=metrics.txt - - STORAGE_CONFIG=${STORAGE_CONFIG} - - REKONI_URL=http://host.docker.internal:4004 + - STORAGE_CONFIG=${STORAGE_CONFIG} - FRONT_URL=http://host.docker.internal:8087 # - APM_SERVER_URL=http://apm-server:8200 - SES_URL='' - ACCOUNTS_URL=http://host.docker.internal:3000 - - LAST_NAME_FIRST=true - - ELASTIC_INDEX_NAME=local_storage_index + - LAST_NAME_FIRST=true - BRANDING_PATH=/var/cfg/branding.json - SUPPORT_WORKSPACE=support - AI_BOT_URL=http://host.docker.internal:4010 @@ -265,7 +260,6 @@ services: - "host.docker.internal:host-gateway" links: - postgres - - elastic - minio - rekoni - account @@ -282,7 +276,7 @@ services: - SERVER_PORT=3331 - SERVER_SECRET=secret - ENABLE_COMPRESSION=true - - ELASTIC_URL=http://host.docker.internal:9200 + - FULLTEXT_URL=http://host.docker.internal:4701 - STATS_URL=http://host.docker.internal:4900 - DB_URL=postgresql://postgres:example@postgres:5432 - MONGO_URL=${MONGO_URL} @@ -290,13 +284,11 @@ services: - METRICS_CONSOLE=false - METRICS_FILE=metrics.txt - STORAGE_CONFIG=${STORAGE_CONFIG} - - REKONI_URL=http://host.docker.internal:4004 - FRONT_URL=http://host.docker.internal:8087 # - APM_SERVER_URL=http://apm-server:8200 - SES_URL='' - ACCOUNTS_URL=http://host.docker.internal:3000 - - LAST_NAME_FIRST=true - - ELASTIC_INDEX_NAME=local_storage_index + - LAST_NAME_FIRST=true - BRANDING_PATH=/var/cfg/branding.json restart: unless-stopped rekoni: @@ -308,6 +300,53 @@ services: resources: limits: memory: 1024M + fulltext: + image: hardcoreeng/fulltext + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped + links: + - elastic + - mongodb + ports: + - 4700:4700 + environment: + - SERVER_SECRET=secret + - DB_URL=${MONGO_URL} + - FULLTEXT_DB_URL=http://host.docker.internal:9200 + - ELASTIC_INDEX_NAME=local_storage_index + - STORAGE_CONFIG=${STORAGE_CONFIG} + - STATS_URL=http://host.docker.internal:4900 + - REKONI_URL=http://host.docker.internal:4004 + - ACCOUNTS_URL=http://host.docker.internal:3000 + deploy: + resources: + limits: + memory: 300M + fulltext_pg: + image: hardcoreeng/fulltext + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped + links: + - elastic + - postgres + ports: + - 4701:4701 + environment: + - PORT=4701 + - SERVER_SECRET=secret + - DB_URL=postgresql://postgres:example@postgres:5432 + - FULLTEXT_DB_URL=http://host.docker.internal:9200 + - ELASTIC_INDEX_NAME=local_storage_index + - STORAGE_CONFIG=${STORAGE_CONFIG} + - STATS_URL=http://host.docker.internal:4900 + - REKONI_URL=http://host.docker.internal:4004 + - ACCOUNTS_URL=http://host.docker.internal:3000 + deploy: + resources: + limits: + memory: 300M print: image: hardcoreeng/print extra_hosts: diff --git a/dev/tool/package.json b/dev/tool/package.json index 05c099b0bb..314bfbde9e 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -55,7 +55,7 @@ "@types/jest": "^29.5.5" }, "dependencies": { - "@elastic/elasticsearch": "^7.14.0", + "@elastic/elasticsearch": "^7.17.14", "@hcengineering/account": "^0.6.0", "@hcengineering/workspace-service": "^0.6.0", "@hcengineering/attachment": "^0.6.14", diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts index 14a8d357b4..e3022a3d1a 100644 --- a/dev/tool/src/clean.ts +++ b/dev/tool/src/clean.ts @@ -1240,7 +1240,6 @@ async function updateId ( const { _id, space, modifiedBy, modifiedOn, createdBy, createdOn, _class, ...data } = docIndexState await txop.createDoc(docIndexState._class, docIndexState.space, { ...data, - stages: {}, removed: false }) await txop.update(docIndexState, { removed: true, needIndex: true }) diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 762eb4ee9e..2a2e5df7ac 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -115,7 +115,7 @@ import { } from './clean' import { changeConfiguration } from './configuration' import { moveAccountDbFromMongoToPG, moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db' -import { fixJsonMarkup, migrateMarkup, restoreLostMarkup } from './markup' +import { fixJsonMarkup, migrateMarkup } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { fixAccountEmails, renameAccount } from './renameAccount' import { moveFiles, showLostFiles } from './storage' @@ -1282,38 +1282,6 @@ export function devTool ( }) }) - program.command('show-lost-markup ').action(async (workspace: string, cmd: any) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { - await withStorage(async (adapter) => { - try { - const workspaceId = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, workspaceId) - const endpoint = await getTransactorEndpoint(token) - await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'show' }) - } catch (err: any) { - console.error(err) - } - }) - }) - }) - - program.command('restore-lost-markup ').action(async (workspace: string, cmd: any) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { - await withStorage(async (adapter) => { - try { - const workspaceId = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, workspaceId) - const endpoint = await getTransactorEndpoint(token) - await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'restore' }) - } catch (err: any) { - console.error(err) - } - }) - }) - }) - program.command('fix-bw-workspace ').action(async (workspace: string) => { await withStorage(async (adapter) => { await fixMinioBW(toolCtx, getWorkspaceId(workspace), adapter) diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index 8c2f7d154a..649d78e959 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -3,28 +3,20 @@ import core, { type AnyAttribute, type Class, type Client as CoreClient, - type CollaborativeDoc, type Doc, - type DocIndexState, type Domain, type Hierarchy, - type Markup, type MeasureContext, type Ref, - type TxMixin, - type TxCreateDoc, - type TxUpdateDoc, type WorkspaceId, - RateLimiter, collaborativeDocParse, makeCollaborativeDoc, - SortingOrder, - TxProcessor + RateLimiter } from '@hcengineering/core' import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core' import { connect } from '@hcengineering/server-tool' -import { isEmptyMarkup, jsonToText, markupToYDoc } from '@hcengineering/text' +import { jsonToText, markupToYDoc } from '@hcengineering/text' import { type Db, type FindCursor, type MongoClient } from 'mongodb' export async function fixJsonMarkup ( @@ -213,124 +205,3 @@ async function processMigrateMarkupFor ( console.log('processed', processed) } - -export async function restoreLostMarkup ( - ctx: MeasureContext, - workspaceId: WorkspaceId, - transactorUrl: string, - storageAdapter: StorageAdapter, - { command }: { command: 'show' | 'restore' } -): Promise { - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup' - })) as unknown as CoreClient - - try { - const hierarchy = connection.getHierarchy() - const classes = hierarchy.getDescendants(core.class.Doc) - - for (const _class of classes) { - const isAttachedDoc = hierarchy.isDerived(_class, core.class.AttachedDoc) - - const attributes = hierarchy.getAllAttributes(_class) - const attrs = Array.from(attributes.values()).filter((p) => p.type._class === core.class.TypeCollaborativeDoc) - - // ignore classes with no collaborative attributes - if (attrs.length === 0) continue - - const docs = await connection.findAll(_class, { _class }) - for (const doc of docs) { - for (const attr of attrs) { - const value = hierarchy.isMixin(attr.attributeOf) - ? ((doc as any)[attr.attributeOf]?.[attr.name] as CollaborativeDoc) - : ((doc as any)[attr.name] as CollaborativeDoc) - - if (value == null || value === '') continue - - const { documentId } = collaborativeDocParse(value) - const stat = await storageAdapter.stat(ctx, workspaceId, documentId) - if (stat !== undefined) continue - - const query = isAttachedDoc - ? { - 'tx.objectId': doc._id, - 'tx._class': { $in: [core.class.TxCreateDoc, core.class.TxUpdateDoc] } - } - : { - objectId: doc._id - } - - let restored = false - - // try to restore by txes - // we need last tx that modified the attribute - - const txes = await connection.findAll(isAttachedDoc ? core.class.TxCollectionCUD : core.class.TxCUD, query, { - sort: { modifiedOn: SortingOrder.Descending } - }) - for (const tx of txes) { - const innerTx = TxProcessor.extractTx(tx) - - let markup: string | undefined - if (innerTx._class === core.class.TxMixin) { - const mixinTx = innerTx as TxMixin - markup = (mixinTx.attributes as any)[attr.name] - } else if (innerTx._class === core.class.TxCreateDoc) { - const createTx = innerTx as TxCreateDoc - markup = (createTx.attributes as any)[attr.name] - } else if (innerTx._class === core.class.TxUpdateDoc) { - const updateTex = innerTx as TxUpdateDoc - markup = (updateTex.operations as any)[attr.name] - } else { - continue - } - - if (markup === undefined || !markup.startsWith('{')) continue - if (isEmptyMarkup(markup)) continue - - console.log(doc._class, doc._id, attr.name, markup) - if (command === 'restore') { - const ydoc = markupToYDoc(markup, attr.name) - await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, value, ydoc) - } - restored = true - break - } - - if (restored) continue - - // try to restore by doc index state - const docIndexState = await connection.findOne(core.class.DocIndexState, { - _id: doc._id as Ref - }) - if (docIndexState !== undefined) { - // document:class:Document%content#content#base64 - const attrName = `${doc._class}%${attr.name}#content#base64` - const base64: string | undefined = docIndexState.attributes[attrName] - if (base64 !== undefined) { - const text = Buffer.from(base64, 'base64').toString() - if (text !== '') { - const markup: Markup = JSON.stringify({ - type: 'doc', - content: [ - { - type: 'paragraph', - content: [{ type: 'text', text, marks: [] }] - } - ] - }) - console.log(doc._class, doc._id, attr.name, markup) - if (command === 'restore') { - const ydoc = markupToYDoc(markup, attr.name) - await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, value, ydoc) - } - } - } - } - } - } - } - } finally { - await connection.close() - } -} diff --git a/dev/tool/src/workspace.ts b/dev/tool/src/workspace.ts index fe0d9d98af..9852311668 100644 --- a/dev/tool/src/workspace.ts +++ b/dev/tool/src/workspace.ts @@ -104,7 +104,7 @@ export async function recreateElastic (mongoUrl: string, workspaceId: WorkspaceI const db = getWorkspaceMongoDB(_client, workspaceId) await db .collection(DOMAIN_DOC_INDEX_STATE) - .updateMany({ _class: core.class.DocIndexState }, { $set: { stages: {}, needIndex: true } }) + .updateMany({ _class: core.class.DocIndexState }, { $set: { needIndex: true } }) } finally { client.close() } diff --git a/models/activity/src/index.ts b/models/activity/src/index.ts index 6d63a993c0..d9a0149cbd 100644 --- a/models/activity/src/index.ts +++ b/models/activity/src/index.ts @@ -31,8 +31,8 @@ import { type DocUpdateMessageViewletAttributesConfig, type IgnoreActivity, type Reaction, - type SavedMessage, - type ReplyProvider + type ReplyProvider, + type SavedMessage } from '@hcengineering/activity' import contact, { type Person } from '@hcengineering/contact' import core, { @@ -67,19 +67,21 @@ import { } from '@hcengineering/model' import { TAttachedDoc, TClass, TDoc } from '@hcengineering/model-core' import preference, { TPreference } from '@hcengineering/model-preference' +import presentation from '@hcengineering/model-presentation' import view from '@hcengineering/model-view' import type { Asset, IntlString, Resource } from '@hcengineering/platform' import { type AnyComponent } from '@hcengineering/ui/src/types' -import presentation from '@hcengineering/model-presentation' -import activity from './plugin' import { buildActions } from './actions' import { buildNotifications } from './notification' +import activity from './plugin' export { activityId } from '@hcengineering/activity' export { activityOperation, migrateMessagesSpace } from './migration' export const DOMAIN_ACTIVITY = 'activity' as Domain +export const DOMAIN_USER_MENTION = 'user_mention' as Domain +export const DOMAIN_REACTION = 'reaction' as Domain @Mixin(activity.mixin.ActivityDoc, core.class.Class) export class TActivityDoc extends TClass implements ActivityDoc { @@ -213,7 +215,7 @@ export class TActivityMessagesFilter extends TDoc implements ActivityMessagesFil filter!: Resource<(message: ActivityMessage, _class?: Ref) => boolean> } -@Model(activity.class.Reaction, core.class.AttachedDoc, DOMAIN_ACTIVITY) +@Model(activity.class.Reaction, core.class.AttachedDoc, DOMAIN_REACTION) @UX(activity.string.Reactions) export class TReaction extends TAttachedDoc implements Reaction { @Prop(TypeRef(activity.class.ActivityMessage), core.string.AttachedTo) @@ -248,7 +250,7 @@ export class TReplyProvider extends TDoc implements ReplyProvider { function!: Resource<(message: ActivityMessage) => Promise> } -@Model(activity.class.UserMentionInfo, core.class.AttachedDoc, DOMAIN_ACTIVITY) +@Model(activity.class.UserMentionInfo, core.class.AttachedDoc, DOMAIN_USER_MENTION) export class TUserMentionInfo extends TAttachedDoc { user!: Ref content!: string diff --git a/models/activity/src/migration.ts b/models/activity/src/migration.ts index 3562a8f731..898984596f 100644 --- a/models/activity/src/migration.ts +++ b/models/activity/src/migration.ts @@ -14,6 +14,7 @@ // import { type ActivityMessage, type DocUpdateMessage, type Reaction } from '@hcengineering/activity' +import contact from '@hcengineering/contact' import core, { type Class, type Doc, type Domain, groupByArray, type Ref, type Space } from '@hcengineering/core' import { type MigrateOperation, @@ -25,10 +26,8 @@ import { tryMigrate } from '@hcengineering/model' import { htmlToMarkup } from '@hcengineering/text' -import contact from '@hcengineering/contact' - +import { activityId, DOMAIN_ACTIVITY, DOMAIN_REACTION, DOMAIN_USER_MENTION } from './index' import activity from './plugin' -import { activityId, DOMAIN_ACTIVITY } from './index' const DOMAIN_CHUNTER = 'chunter' as Domain @@ -223,6 +222,13 @@ export const activityOperation: MigrateOperation = { func: async (client: MigrationClient): Promise => { await client.update(DOMAIN_ACTIVITY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) } + }, + { + state: 'move-reactions', + func: async (client: MigrationClient): Promise => { + await client.move(DOMAIN_ACTIVITY, { _class: activity.class.Reaction }, DOMAIN_REACTION) + await client.move(DOMAIN_ACTIVITY, { _class: activity.class.UserMentionInfo }, DOMAIN_USER_MENTION) + } } ]) }, diff --git a/models/all/package.json b/models/all/package.json index e5b825c0fc..044e908168 100644 --- a/models/all/package.json +++ b/models/all/package.json @@ -109,6 +109,7 @@ "@hcengineering/model-server-controlled-documents": "^0.1.0", "@hcengineering/model-analytics-collector": "^0.6.0", "@hcengineering/model-server-ai-bot": "^0.6.0", - "@hcengineering/model-ai-bot": "^0.6.0" + "@hcengineering/model-ai-bot": "^0.6.0", + "@hcengineering/model-server-fulltext": "^0.6.0" } } diff --git a/models/all/src/index.ts b/models/all/src/index.ts index a958108caf..0563d6749f 100644 --- a/models/all/src/index.ts +++ b/models/all/src/index.ts @@ -99,6 +99,7 @@ import { serverDocumentsId, createModel as serverDocumentsModel } from '@hcengineering/model-server-controlled-documents' +import { serverFulltextId, createModel as serverFulltextModel } from '@hcengineering/model-server-fulltext' import { type Plugin } from '@hcengineering/platform' @@ -434,7 +435,8 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[ [serverProductsModel, serverProductsId], [serverTrainingModel, serverTrainingId], [serverDocumentsModel, serverDocumentsId], - [serverAiBotModel, serverAiBotId] + [serverAiBotModel, serverAiBotId], + [serverFulltextModel, serverFulltextId] ] for (const [b, id, config] of builders) { diff --git a/models/attachment/src/migration.ts b/models/attachment/src/migration.ts index 5fead855a8..73b55dd12b 100644 --- a/models/attachment/src/migration.ts +++ b/models/attachment/src/migration.ts @@ -19,7 +19,7 @@ import { type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model' -import { attachmentId, DOMAIN_ATTACHMENT } from '.' +import attachment, { attachmentId, DOMAIN_ATTACHMENT } from '.' export const attachmentOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { @@ -29,6 +29,20 @@ export const attachmentOperation: MigrateOperation = { func: async (client: MigrationClient): Promise => { await client.update(DOMAIN_ATTACHMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) } + }, + { + state: 'fix-attachedTo', + func: async (client: MigrationClient): Promise => { + await client.update( + DOMAIN_ATTACHMENT, + { _class: attachment.class.Attachment, attachedToClass: 'chunter:class:Comment' }, + { + $set: { + attachedToClass: 'chunter:class:ChatMessage' + } + } + ) + } } ]) }, diff --git a/models/bitrix/src/index.ts b/models/bitrix/src/index.ts index 28f3bcaaaf..071002797a 100644 --- a/models/bitrix/src/index.ts +++ b/models/bitrix/src/index.ts @@ -145,4 +145,12 @@ export function createModel (builder: Builder): void { { modifiedOn: 1 } ] }) + builder.mixin(bitrix.class.FieldMapping, core.class.Class, core.mixin.IndexConfiguration, { + indexes: [], + searchDisabled: true + }) + builder.mixin(bitrix.class.EntityMapping, core.class.Class, core.mixin.IndexConfiguration, { + indexes: [], + searchDisabled: true + }) } diff --git a/models/chunter/src/index.ts b/models/chunter/src/index.ts index a59cb8b959..d714ebcee3 100644 --- a/models/chunter/src/index.ts +++ b/models/chunter/src/index.ts @@ -314,6 +314,16 @@ export function createModel (builder: Builder): void { defineActions(builder) defineNotifications(builder) + + builder.mixin(chunter.class.InlineButton, core.class.Class, core.mixin.IndexConfiguration, { + indexes: [], + searchDisabled: true + }) + + builder.mixin(chunter.class.ChatSyncInfo, core.class.Class, core.mixin.IndexConfiguration, { + indexes: [], + searchDisabled: true + }) } export default chunter diff --git a/models/controlled-documents/src/index.ts b/models/controlled-documents/src/index.ts index 14f81efa24..9e14cf94f9 100644 --- a/models/controlled-documents/src/index.ts +++ b/models/controlled-documents/src/index.ts @@ -888,8 +888,7 @@ export function defineNotifications (builder: Builder): void { export function defineSearch (builder: Builder): void { builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { - toClass: documents.class.Document, - parentPropagate: true + toClass: documents.class.Document }) builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { diff --git a/models/core/src/core.ts b/models/core/src/core.ts index 2cff517172..60512e0cf2 100644 --- a/models/core/src/core.ts +++ b/models/core/src/core.ts @@ -66,7 +66,6 @@ import { TypeCollaborativeDoc, TypeFileSize, TypeIntlString, - TypeRecord, TypeRef, TypeString, TypeTimestamp, @@ -330,19 +329,6 @@ export class TDocIndexState extends TDoc implements DocIndexState { @Hidden() objectClass!: Ref> - @Prop(TypeRef(core.class.Doc), core.string.AttachedTo) - @Index(IndexKind.Indexed) - @Hidden() - attachedTo?: Ref - - @Prop(TypeRef(core.class.Class), core.string.AttachedToClass) - @Index(IndexKind.Indexed) - @Hidden() - attachedToClass?: Ref> - - // Indexable attributes of document. - attributes!: Record - @Prop(TypeBoolean(), getEmbeddedLabel('Removed')) @Hidden() removed!: boolean @@ -350,16 +336,6 @@ export class TDocIndexState extends TDoc implements DocIndexState { @Prop(TypeBoolean(), getEmbeddedLabel('NeedIndexing')) @Hidden() needIndex!: boolean - - // States for different stages - @Prop(TypeRecord(), getEmbeddedLabel('Stages')) - // @Index(IndexKind.Indexed) - @Hidden() - stages!: Record - - @Prop(TypeString(), getEmbeddedLabel('Generation')) - @Hidden() - generationId?: string } @Model(core.class.FullTextSearchContext, core.class.Doc, DOMAIN_MODEL) diff --git a/models/core/src/index.ts b/models/core/src/index.ts index e7dfc70eca..ef1b6c943a 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -295,16 +295,20 @@ export function createModel (builder: Builder): void { domain: DOMAIN_DOC_INDEX_STATE, indexes: [ { - keys: { needIndex: 1 } + keys: { needIndex: 1, objectClass: 1 } } ], disabled: [ { attachedToClass: 1 }, { stages: 1 }, - { generationId: 1 }, { space: 1 }, { _class: 1 }, + { needIndex: 1 }, + { objectClass: 1 }, + { _class: 1 }, + { attachedTo: 1 }, { modifiedBy: 1 }, + { modifiedOn: 1 }, { createdBy: 1 }, { createdBy: -1 }, { createdOn: -1 } @@ -318,4 +322,17 @@ export function createModel (builder: Builder): void { definePermissions(builder) defineSpaceType(builder) + + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: core.class.MigrationState, + forceIndex: false + }) + builder.mixin(core.class.Configuration, core.class.Class, core.mixin.IndexConfiguration, { + indexes: [], + searchDisabled: true + }) + builder.mixin(core.class.MigrationState, core.class.Class, core.mixin.IndexConfiguration, { + indexes: [], + searchDisabled: true + }) } diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 29e7311bf0..a692f66ab1 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -15,7 +15,6 @@ import { saveCollaborativeDoc } from '@hcengineering/collaboration' import core, { - DOMAIN_DOC_INDEX_STATE, DOMAIN_SPACE, DOMAIN_STATUS, DOMAIN_TX, @@ -24,7 +23,6 @@ import core, { collaborativeDocParse, coreId, generateId, - isClassIndexable, makeCollaborativeDoc, type AnyAttribute, type Doc, @@ -260,25 +258,6 @@ async function processMigrateContentFor ( export const coreOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { - // We need to delete all documents in doc index state for missing classes - const allClasses = client.hierarchy.getDescendants(core.class.Doc) - const contexts = new Map( - client.model.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it]) - ) - - const allIndexed = allClasses.filter((it) => isClassIndexable(client.hierarchy, it, contexts)) - - // Next remove all non indexed classes and missing classes as well. - await client.update( - DOMAIN_DOC_INDEX_STATE, - { objectClass: { $nin: allIndexed } }, - { - $set: { - removed: true, - needIndex: true - } - } - ) await tryMigrate(client, coreId, [ { state: 'statuses-to-model', @@ -296,12 +275,6 @@ export const coreOperation: MigrateOperation = { state: 'old-statuses-transactions', func: migrateStatusTransactions }, - { - state: 'add-need-index', - func: async (client: MigrationClient) => { - await client.update(DOMAIN_DOC_INDEX_STATE, {}, { $set: { needIndex: true } }) - } - }, { state: 'collaborative-content-to-storage', func: migrateCollaborativeContentToStorage diff --git a/models/gmail/src/index.ts b/models/gmail/src/index.ts index c7853dcdb4..4133a0060c 100644 --- a/models/gmail/src/index.ts +++ b/models/gmail/src/index.ts @@ -215,8 +215,7 @@ export function createModel (builder: Builder): void { ) builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { - toClass: gmail.class.Message, - parentPropagate: false + toClass: gmail.class.Message }) builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, { diff --git a/models/love/src/index.ts b/models/love/src/index.ts index 25d4611fb4..abd60ac349 100644 --- a/models/love/src/index.ts +++ b/models/love/src/index.ts @@ -391,4 +391,19 @@ export function createModel (builder: Builder): void { builder.mixin(love.class.MeetingMinutes, core.class.Class, view.mixin.ObjectPresenter, { presenter: love.component.MeetingMinutesPresenter }) + + builder.mixin(love.class.Room, core.class.Class, core.mixin.IndexConfiguration, { + indexes: [], + searchDisabled: true + }) + + builder.mixin(love.class.Office, core.class.Class, core.mixin.IndexConfiguration, { + indexes: [], + searchDisabled: true + }) + + builder.mixin(love.class.Floor, core.class.Class, core.mixin.IndexConfiguration, { + indexes: [], + searchDisabled: true + }) } diff --git a/models/recruit/src/index.ts b/models/recruit/src/index.ts index 8d0a97046f..fe4b43e16b 100644 --- a/models/recruit/src/index.ts +++ b/models/recruit/src/index.ts @@ -1469,26 +1469,6 @@ export function createModel (builder: Builder): void { propagate: [] }) - createAction(builder, { - label: recruit.string.MatchVacancy, - icon: recruit.icon.Vacancy, - action: view.actionImpl.ShowPopup, - actionProps: { - component: recruit.component.MatchVacancy, - element: 'top', - fillProps: { - _objects: 'objects' - } - }, - input: 'any', - category: recruit.category.Recruit, - target: recruit.mixin.Candidate, - context: { - mode: ['context', 'browser'], - group: 'create' - } - }) - builder.mixin(recruit.mixin.Candidate, core.class.Class, view.mixin.ObjectEditorFooter, { editor: tracker.component.RelatedIssuesSection, props: { diff --git a/models/recruit/src/plugin.ts b/models/recruit/src/plugin.ts index 13c0f704db..59af01b4ae 100644 --- a/models/recruit/src/plugin.ts +++ b/models/recruit/src/plugin.ts @@ -126,9 +126,9 @@ export default mergeIds(recruitId, recruit, { VacancyTemplateEditor: '' as AnyComponent, ApplicationMatchPresenter: '' as AnyComponent, - MatchVacancy: '' as AnyComponent, NotificationApplicantPresenter: '' as AnyComponent, - VacancyEditor: '' as AnyComponent + VacancyEditor: '' as AnyComponent, + ApplicantNamePresenter: '' as AnyComponent }, template: { DefaultVacancy: '' as Ref diff --git a/models/server-contact/src/index.ts b/models/server-contact/src/index.ts index 6e822e8128..8476613c83 100644 --- a/models/server-contact/src/index.ts +++ b/models/server-contact/src/index.ts @@ -43,15 +43,15 @@ export function createModel (builder: Builder): void { }) builder.mixin(contact.class.Contact, core.class.Class, serverCore.mixin.SearchPresenter, { - searchConfig: { - iconConfig: { - component: contact.component.AvatarRef, - props: ['_id'] - }, - title: { props: ['name'] } + iconConfig: { + component: contact.component.AvatarRef, + fields: [['_id']] }, - getSearchTitle: { - name: serverContact.function.ContactNameProvider + title: { + component: contact.component.ContactNamePresenter, // Will present on UI. + fields: [['name']], + template: [['func', serverContact.function.ContactNameProvider, 'true']], + extraFields: [[['func', serverContact.function.ContactNameProvider, 'false']]] } }) diff --git a/models/server-controlled-documents/src/index.ts b/models/server-controlled-documents/src/index.ts index 643d2b069f..1bd4370205 100644 --- a/models/server-controlled-documents/src/index.ts +++ b/models/server-controlled-documents/src/index.ts @@ -58,13 +58,10 @@ export function createModel (builder: Builder): void { }) builder.mixin(documents.class.DocumentMeta, core.class.Class, serverCore.mixin.SearchPresenter, { - searchConfig: { - iconConfig: { - component: documents.component.DocumentIcon, - props: [] - }, - title: 'title' - } + iconConfig: { + component: documents.component.DocumentIcon + }, + title: [['title']] }) builder.mixin(documents.class.ControlledDocument, core.class.Class, serverNotification.mixin.TextPresenter, { diff --git a/models/server-core/src/index.ts b/models/server-core/src/index.ts index 919cfd38d4..aff508963b 100644 --- a/models/server-core/src/index.ts +++ b/models/server-core/src/index.ts @@ -20,11 +20,10 @@ import type { Resource } from '@hcengineering/platform' import core, { DOMAIN_MODEL } from '@hcengineering/core' import type { - ClassSearchConfig, + FieldTemplateComponent, ObjectDDParticipant, ObjectDDParticipantFunc, SearchPresenter, - SearchPresenterFunc, Trigger, TriggerFunc } from '@hcengineering/server-core' @@ -44,9 +43,7 @@ export class TObjectDDParticipant extends TClass implements ObjectDDParticipant @Mixin(serverCore.mixin.SearchPresenter, core.class.Class) export class TSearchPresenter extends TClass implements SearchPresenter { - searchConfig!: ClassSearchConfig - getSearchShortTitle!: SearchPresenterFunc - getSearchTitle!: SearchPresenterFunc + title!: FieldTemplateComponent } export function createModel (builder: Builder): void { diff --git a/models/server-document/src/index.ts b/models/server-document/src/index.ts index 10bd0b9f71..90cc1f56e6 100644 --- a/models/server-document/src/index.ts +++ b/models/server-document/src/index.ts @@ -28,13 +28,11 @@ export function createModel (builder: Builder): void { }) builder.mixin(document.class.Document, core.class.Class, serverCore.mixin.SearchPresenter, { - searchConfig: { - iconConfig: { - component: document.component.DocumentSearchIcon, - props: ['icon', 'color'] - }, - title: 'title' - } + iconConfig: { + component: document.component.DocumentSearchIcon, + fields: [['icon'], ['color']] + }, + title: [['title']] }) builder.mixin, ObjectDDParticipant>( diff --git a/models/server-drive/src/index.ts b/models/server-drive/src/index.ts index acca4db1c1..15dea09acc 100644 --- a/models/server-drive/src/index.ts +++ b/models/server-drive/src/index.ts @@ -41,16 +41,12 @@ export function createModel (builder: Builder): void { ) builder.mixin(drive.class.File, core.class.Class, serverCore.mixin.SearchPresenter, { - searchConfig: { - icon: drive.icon.File, - title: 'title' - } + searchIcon: drive.icon.File, + title: [['title']] }) builder.mixin(drive.class.Folder, core.class.Class, serverCore.mixin.SearchPresenter, { - searchConfig: { - icon: drive.icon.Folder, - title: 'title' - } + searchIcon: drive.icon.Folder, + title: [['title']] }) } diff --git a/models/server-fulltext/.eslintrc.js b/models/server-fulltext/.eslintrc.js new file mode 100644 index 0000000000..c1cf82cba0 --- /dev/null +++ b/models/server-fulltext/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/model/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/models/server-fulltext/.npmignore b/models/server-fulltext/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/models/server-fulltext/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/models/server-fulltext/config/rig.json b/models/server-fulltext/config/rig.json new file mode 100644 index 0000000000..2f6be36605 --- /dev/null +++ b/models/server-fulltext/config/rig.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig", + "rigProfile": "model" +} diff --git a/models/server-fulltext/package.json b/models/server-fulltext/package.json new file mode 100644 index 0000000000..e4dcf66b17 --- /dev/null +++ b/models/server-fulltext/package.json @@ -0,0 +1,37 @@ +{ + "name": "@hcengineering/model-server-fulltext", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "author": "Anticrm Platform Contributors", + "template": "@hcengineering/model-package", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "format": "format src", + "_phase:build": "compile transpile src", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "^0.6.0", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "typescript": "^5.3.3" + }, + "dependencies": { + "@hcengineering/core": "^0.6.32", + "@hcengineering/model": "^0.6.11", + "@hcengineering/platform": "^0.6.11", + "@hcengineering/server-fulltext": "^0.6.0", + "@hcengineering/server-core": "^0.6.1" + } +} diff --git a/models/server-fulltext/src/index.ts b/models/server-fulltext/src/index.ts new file mode 100644 index 0000000000..b9716d0b8a --- /dev/null +++ b/models/server-fulltext/src/index.ts @@ -0,0 +1,27 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type Builder } from '@hcengineering/model' + +import core from '@hcengineering/core' +import serverCore from '@hcengineering/server-core' +import serverFulltext from '@hcengineering/server-fulltext' +export { serverFulltextId } from '@hcengineering/server-fulltext' +export function createModel (builder: Builder): void { + builder.createDoc(serverCore.class.Trigger, core.space.Model, { + trigger: serverFulltext.trigger.OnChange, + arrays: true + }) +} diff --git a/models/server-fulltext/tsconfig.json b/models/server-fulltext/tsconfig.json new file mode 100644 index 0000000000..9765b6086d --- /dev/null +++ b/models/server-fulltext/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/model/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/models/server-products/src/index.ts b/models/server-products/src/index.ts index 52d8a30d99..1383345994 100644 --- a/models/server-products/src/index.ts +++ b/models/server-products/src/index.ts @@ -14,12 +14,10 @@ export const serverProductsId = 'server-products' as Plugin export function createModel (builder: Builder): void { builder.mixin(products.class.Product, core.class.Class, serverCore.mixin.SearchPresenter, { - searchConfig: { - iconConfig: { - component: products.component.ProductSearchIcon, - props: ['icon', 'color'] - }, - title: 'name' - } + iconConfig: { + component: products.component.ProductSearchIcon, + fields: [['icon'], ['color']] + }, + title: [['name']] }) } diff --git a/models/server-recruit/src/index.ts b/models/server-recruit/src/index.ts index b627a5ae91..9e9d3ea94e 100644 --- a/models/server-recruit/src/index.ts +++ b/models/server-recruit/src/index.ts @@ -15,15 +15,15 @@ import { type Builder } from '@hcengineering/model' +import contact from '@hcengineering/contact' import core from '@hcengineering/core' import recruit from '@hcengineering/model-recruit' import notification from '@hcengineering/notification' import serverCore from '@hcengineering/server-core' import serverNotification from '@hcengineering/server-notification' import serverRecruit from '@hcengineering/server-recruit' -import serverContact from '@hcengineering/server-contact' -import contact from '@hcengineering/contact' import serverView from '@hcengineering/server-view' +import serverContact from '@hcengineering/server-contact' export { serverRecruitId } from '@hcengineering/server-recruit' @@ -68,26 +68,32 @@ export function createModel (builder: Builder): void { }) builder.mixin(recruit.class.Vacancy, core.class.Class, serverCore.mixin.SearchPresenter, { - searchConfig: { - icon: recruit.icon.Vacancy, - title: 'name' - } + searchIcon: recruit.icon.Vacancy, + title: [['name']] }) builder.mixin(recruit.class.Applicant, core.class.Class, serverCore.mixin.SearchPresenter, { - searchConfig: { - iconConfig: { - component: contact.component.AvatarRef, - props: [{ _id: ['attachedTo'] }] - }, - shortTitle: 'identifier', - title: { - tmpl: '{name} - {vacName}', - props: [{ _class: ['attachedTo', '_class'] }, { name: ['attachedTo', 'name'] }, { vacName: ['space', 'name'] }] - } + iconConfig: { + component: contact.component.AvatarRef, + fields: [['attachedTo']] }, - getSearchTitle: { - name: serverContact.function.ContactNameProvider + shortTitle: [['identifier']], + title: { + component: recruit.component.ApplicantNamePresenter, // Will present on UI. + fields: [ + ['parent', 'name'], + ['space', 'name'] + ], + template: [ + ['func', serverContact.function.ContactNameProvider, 'true'], + ['space', 'name'] + ], + extraFields: [ + [ + ['func', serverContact.function.ContactNameProvider, 'false'], + ['space', 'name'] + ] + ] } }) diff --git a/models/server-tracker/src/index.ts b/models/server-tracker/src/index.ts index 5cbf00948b..e41a90d5f6 100644 --- a/models/server-tracker/src/index.ts +++ b/models/server-tracker/src/index.ts @@ -13,6 +13,7 @@ // limitations under the License. // +import contact from '@hcengineering/contact' import core from '@hcengineering/core' import { type Builder } from '@hcengineering/model' import tracker from '@hcengineering/model-tracker' @@ -20,7 +21,6 @@ import notification from '@hcengineering/notification' import serverCore from '@hcengineering/server-core' import serverNotification from '@hcengineering/server-notification' import serverTracker from '@hcengineering/server-tracker' -import contact from '@hcengineering/contact' import serverView from '@hcengineering/server-view' export { serverTrackerId } from '@hcengineering/server-tracker' @@ -43,14 +43,12 @@ export function createModel (builder: Builder): void { }) builder.mixin(tracker.class.Issue, core.class.Class, serverCore.mixin.SearchPresenter, { - searchConfig: { - iconConfig: { - component: tracker.component.IssueSearchIcon, - props: ['status', 'space'] - }, - shortTitle: 'identifier', - title: 'title' - } + iconConfig: { + component: tracker.component.IssueSearchIcon, + fields: [['status'], ['space']] + }, + shortTitle: [['identifier']], + title: [['title']] }) builder.createDoc(serverCore.class.Trigger, core.space.Model, { diff --git a/models/setting/src/index.ts b/models/setting/src/index.ts index 3272c01439..6692f10e02 100644 --- a/models/setting/src/index.ts +++ b/models/setting/src/index.ts @@ -16,7 +16,7 @@ import activity from '@hcengineering/activity' import contact from '@hcengineering/contact' import { AccountRole, DOMAIN_MODEL, type Account, type Blob, type Domain, type Ref } from '@hcengineering/core' -import { Mixin, Model, type Builder, UX } from '@hcengineering/model' +import { Mixin, Model, UX, type Builder } from '@hcengineering/model' import core, { TClass, TConfiguration, TDoc } from '@hcengineering/model-core' import view, { createAction } from '@hcengineering/model-view' import notification from '@hcengineering/notification' @@ -28,12 +28,12 @@ import { type Integration, type IntegrationType, type InviteSettings, - type WorkspaceSetting, type SettingsCategory, - type UserMixin, + type SpaceTypeCreator, type SpaceTypeEditor, type SpaceTypeEditorSection, - type SpaceTypeCreator + type UserMixin, + type WorkspaceSetting } from '@hcengineering/setting' import templates from '@hcengineering/templates' import setting from './plugin' diff --git a/models/telegram/src/index.ts b/models/telegram/src/index.ts index 91878b222b..2a23222535 100644 --- a/models/telegram/src/index.ts +++ b/models/telegram/src/index.ts @@ -182,7 +182,6 @@ export function createModel (builder: Builder): void { builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { toClass: telegram.class.Message, - parentPropagate: false, childProcessingAllowed: true }) diff --git a/models/view/src/index.ts b/models/view/src/index.ts index 81b1dc09f9..e3f974c4a2 100644 --- a/models/view/src/index.ts +++ b/models/view/src/index.ts @@ -1209,6 +1209,10 @@ export function createModel (builder: Builder): void { builder.mixin(core.class.Space, core.class.Class, view.mixin.IgnoreActions, { actions: [view.action.Open, view.action.OpenInNewTab, view.action.Delete] }) + builder.mixin(view.class.FilteredView, core.class.Class, core.mixin.IndexConfiguration, { + indexes: [], + searchDisabled: true + }) } export default view diff --git a/models/view/src/plugin.ts b/models/view/src/plugin.ts index 4f71ba9b84..0fa27261a0 100644 --- a/models/view/src/plugin.ts +++ b/models/view/src/plugin.ts @@ -78,7 +78,6 @@ export default mergeIds(viewId, view, { MarkupEditor: '' as AnyComponent, MarkupEditorPopup: '' as AnyComponent, ListView: '' as AnyComponent, - IndexedDocumentPreview: '' as AnyComponent, SpaceRefPresenter: '' as AnyComponent, EnumPresenter: '' as AnyComponent, StatusPresenter: '' as AnyComponent, diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index 3c37f52773..f032beebc2 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -360,16 +360,6 @@ export interface TransientConfiguration extends Class { */ export const DOMAIN_BLOB = 'blob' as Domain -/** - * Special domain to access s3 blob data. - * @public - */ -export const DOMAIN_FULLTEXT_BLOB = 'fulltext-blob' as Domain - -/** - * Special domain to access s3 blob data. - * @public - */ export const DOMAIN_DOC_INDEX_STATE = 'doc-index-state' as Domain // S P A C E @@ -526,25 +516,8 @@ export function versionToString (version: Version | Data): string { */ export interface DocIndexState extends Doc { objectClass: Ref> - - attachedTo?: Ref - attachedToClass?: Ref> - - generationId?: string - - // States for stages - stages: Record - needIndex: boolean - removed: boolean - - // Indexable attributes, including child ones. - attributes: Record - mixins?: Ref>[] - // Full Summary - fullSummary?: string | null - shortSummary?: string | null } /** @@ -594,9 +567,6 @@ export interface FullTextSearchContext extends Doc { // If defined, will propagate all document from child's based on class propagateClasses?: Ref>[] - // Do we need to propagate child value to parent one. Default(true) - parentPropagate?: boolean - childProcessingAllowed?: boolean } diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 5a16bfa67d..130f651a00 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -57,6 +57,8 @@ export interface SessionData { workspace: WorkspaceIdWithUrl branding: Branding | null + + needWarmupFulltext?: boolean } /** @@ -78,7 +80,12 @@ export interface LowLevelStorage { clean: (ctx: MeasureContext, domain: Domain, docs: Ref[]) => Promise // Low level direct group API - groupBy: (ctx: MeasureContext, domain: Domain, field: string) => Promise> + groupBy: ( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery

+ ) => Promise> // migrations rawFindAll: (domain: Domain, query: DocumentQuery, options?: FindOptions) => Promise diff --git a/packages/core/src/storage.ts b/packages/core/src/storage.ts index c26bf2d595..c3dee93106 100644 --- a/packages/core/src/storage.ts +++ b/packages/core/src/storage.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import type { Asset } from '@hcengineering/platform' +import type { Asset, Resource } from '@hcengineering/platform' import type { KeysByType } from 'simplytyped' import type { AttachedDoc, Class, Doc, Ref, Space } from './classes' @@ -221,16 +221,23 @@ export interface SearchOptions { limit?: number } +export interface SearchComponentWithProps { + component?: Resource + props?: Record +} + /** * @public */ export interface SearchResultDoc { id: Ref + icon?: Asset - iconComponent?: string - iconProps?: Record + iconComponent?: SearchComponentWithProps shortTitle?: string + shortTitleComponent?: SearchComponentWithProps title?: string + titleComponent?: SearchComponentWithProps score?: number doc: Pick } diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index 506201eba5..36e250c1a5 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -26,7 +26,6 @@ import { Collection, Doc, DocData, - DocIndexState, DOMAIN_BLOB, DOMAIN_DOC_INDEX_STATE, DOMAIN_MODEL, @@ -41,6 +40,7 @@ import { Space, TypedSpace, WorkspaceMode, + type Domain, type PluginConfiguration } from './classes' import core from './component' @@ -165,72 +165,11 @@ export function isWorkspaceCreating (mode?: WorkspaceMode): boolean { return ['pending-creation', 'creating'].includes(mode) } -const attributesPrefix = 'attributes.' - /** * @public */ -export interface IndexKeyOptions { - _class?: Ref> - docId?: Ref - extra?: string[] - digest?: boolean -} -/** - * @public - */ - -export function docUpdKey (name: string, opt?: IndexKeyOptions): string { - return attributesPrefix + docKey(name, opt) -} -/** - * @public - */ -export function docKey (name: string, opt?: IndexKeyOptions): string { - const extra = opt?.extra !== undefined && opt?.extra?.length > 0 ? `#${opt.extra?.join('#') ?? ''}` : '' - const digestName = opt?.digest === true ? name + '^digest' : name - return opt?._class === undefined ? digestName : `${opt?._class}%${digestName}${extra}` -} - -/** - * @public - */ -export function extractDocKey (key: string): { - _class?: Ref> - attr: string - docId?: Ref - extra: string[] - digest: boolean -} { - let k = key - if (k.startsWith(attributesPrefix)) { - k = k.slice(attributesPrefix.length) - } - let docId: Ref | undefined - let _class: Ref> | undefined - let attr = '' - const docSepPos = k.indexOf('|') - if (docSepPos !== -1) { - docId = k.substring(0, docSepPos).replace('_', '.') as Ref - k = k.substring(docSepPos + 1) - } - const clPos = k.indexOf('%') - if (clPos !== -1) { - _class = k.substring(0, clPos) as Ref> - attr = k.substring(clPos + 1) - } else { - attr = k - } - const extra = attr.split('#') - attr = extra.splice(0, 1)[0] - const digestPos = attr.indexOf('^digest') - let digest = false - if (digestPos !== -1) { - attr = attr.substring(0, digestPos) - digest = true - } - - return { docId, attr, _class, extra, digest } +export function docKey (name: string, _class?: Ref>): string { + return _class === undefined || _class !== core.class.Doc ? name : `${_class}%${name}` } /** @@ -744,7 +683,9 @@ export function isClassIndexable ( domain === DOMAIN_TX || domain === DOMAIN_MODEL || domain === DOMAIN_BLOB || + domain === ('preference' as Domain) || domain === DOMAIN_TRANSIENT || + domain === ('settings' as Domain) || domain === DOMAIN_BENCHMARK ) { hierarchy.setClassifierProp(c, 'class_indexed', false) diff --git a/packages/platform/src/resource.ts b/packages/platform/src/resource.ts index 2152d003b0..e679a8e32c 100644 --- a/packages/platform/src/resource.ts +++ b/packages/platform/src/resource.ts @@ -137,6 +137,30 @@ export function getResourceP (resource: Resource): T | Promise { return cachedResource.get(resource) ?? getResource(resource) } +/** + * @public + * @param resource - + * @returns + */ +export function getResourceC (resource: Resource | undefined, callback: (resource: T | undefined) => void): void { + if (resource === undefined) { + callback(undefined) + return + } + const cached = cachedResource.get(resource) + if (cached !== undefined) { + callback(cached) + } else { + void getResource(resource) + .then((r) => { + callback(r) + }) + .catch(() => { + callback(undefined) + }) + } +} + /** * @public */ diff --git a/packages/presentation/src/components/IndexedDocumentCompare.svelte b/packages/presentation/src/components/IndexedDocumentCompare.svelte deleted file mode 100644 index 4156e88be9..0000000000 --- a/packages/presentation/src/components/IndexedDocumentCompare.svelte +++ /dev/null @@ -1,33 +0,0 @@ - - -

- -
-
- -
- {#if right !== undefined} -
- -
- {/if} -
- - - diff --git a/packages/presentation/src/components/IndexedDocumentContent.svelte b/packages/presentation/src/components/IndexedDocumentContent.svelte deleted file mode 100644 index 00dbc69645..0000000000 --- a/packages/presentation/src/components/IndexedDocumentContent.svelte +++ /dev/null @@ -1,97 +0,0 @@ - - -{#if summary} - {#if search.length > 0} - Result: - {#each summary.split('\n').filter((line, idx, arr) => { - return line.toLowerCase().includes(search.toLowerCase()) || arr[idx - 1] - ?.toLowerCase() - .includes(search.toLowerCase()) - }) as line} - {line} - {/each} -
- {/if} - Summary: - {#each summary.split('\n') as line} - {@const hl = search.length > 0 && line.toLowerCase().includes(search.toLowerCase())} - {line} - {/each} -{:else if indexDoc} - {#each attributes as attr} - {@const clOf = client.getHierarchy().getClass(attr[0].attributeOf)} -
- {#if clOf.icon} -
- -
- {/if} -
-
- {#each attr[1] as doc} -
1}> - {#if search.length > 0} - Result: - {#each doc.filter((line) => line.toLowerCase().includes(search.toLowerCase())) as line} - {line} - {/each} -
- {/if} - {#each doc as line} - {@const hl = search.length > 0 && line.toLowerCase().includes(search.toLowerCase())} - {line} - {/each} -
- {/each} -
- {/each} -{/if} - - diff --git a/packages/presentation/src/components/IndexedDocumentPreview.svelte b/packages/presentation/src/components/IndexedDocumentPreview.svelte deleted file mode 100644 index 555e86aaf9..0000000000 --- a/packages/presentation/src/components/IndexedDocumentPreview.svelte +++ /dev/null @@ -1,52 +0,0 @@ - - -{#if noPanel} -
-
-
- {#if indexDoc} - - {/if} -
-{:else} - - -
- {#if indexDoc} - - {/if} -
- -{/if} - - diff --git a/packages/presentation/src/components/SearchResult.svelte b/packages/presentation/src/components/SearchResult.svelte index 7f8b49a1c8..87b790682f 100644 --- a/packages/presentation/src/components/SearchResult.svelte +++ b/packages/presentation/src/components/SearchResult.svelte @@ -15,33 +15,51 @@ -->
- {#if icon !== undefined} - - {/if} {#if iconComponent}
- {#await getResource(iconComponent) then component} - - {/await} +
+ {:else if icon !== undefined} + {/if}
- - {#if value.shortTitle !== undefined} + + {#if shortTitleComponent} + + {:else if value.shortTitle !== undefined} {value.shortTitle} {/if} - {value.title} + {#if titleComponent} + + {:else} + {value.title} + {/if}
diff --git a/packages/presentation/src/index.ts b/packages/presentation/src/index.ts index c43c424b20..8bffb3c21d 100644 --- a/packages/presentation/src/index.ts +++ b/packages/presentation/src/index.ts @@ -35,8 +35,6 @@ export { default as SpaceSelect } from './components/SpaceSelect.svelte' export { default as SpaceSelector } from './components/SpaceSelector.svelte' export { default as SpacesMultiPopup } from './components/SpacesMultiPopup.svelte' export { default as ObjectSearchPopup } from './components/ObjectSearchPopup.svelte' -export { default as IndexedDocumentPreview } from './components/IndexedDocumentPreview.svelte' -export { default as IndexedDocumentCompare } from './components/IndexedDocumentCompare.svelte' export { default as NavLink } from './components/NavLink.svelte' export { default as IconDownload } from './components/icons/Download.svelte' export { default as FilePreview } from './components/FilePreview.svelte' diff --git a/plugins/ai-bot/src/rest.ts b/plugins/ai-bot/src/rest.ts index 04b243ba28..d6a71da8a7 100644 --- a/plugins/ai-bot/src/rest.ts +++ b/plugins/ai-bot/src/rest.ts @@ -74,4 +74,10 @@ export interface PostTranscriptRequest { transcript: string participant: Ref roomName: string + final: boolean +} + +export interface IdentityResponse { + identity: Ref + name: string } diff --git a/plugins/contact-resources/src/components/ContactNamePresenter.svelte b/plugins/contact-resources/src/components/ContactNamePresenter.svelte new file mode 100644 index 0000000000..ed72d45dec --- /dev/null +++ b/plugins/contact-resources/src/components/ContactNamePresenter.svelte @@ -0,0 +1,37 @@ + + + + +{displayName} diff --git a/plugins/contact-resources/src/index.ts b/plugins/contact-resources/src/index.ts index fd40412ebe..cd7875e4bc 100644 --- a/plugins/contact-resources/src/index.ts +++ b/plugins/contact-resources/src/index.ts @@ -123,6 +123,7 @@ import NameChangedActivityMessage from './components/activity/NameChangedActivit import IconAddMember from './components/icons/AddMember.svelte' import ExpandRightDouble from './components/icons/ExpandRightDouble.svelte' import IconMembers from './components/icons/Members.svelte' +import ContactNamePresenter from './components/ContactNamePresenter.svelte' import { get, writable } from 'svelte/store' import contact from './plugin' @@ -380,7 +381,8 @@ export default async (): Promise => ({ PersonIcon, EditOrganizationPanel, ChannelIcon, - SpaceMembersEditor + SpaceMembersEditor, + ContactNamePresenter }, completion: { EmployeeQuery: async ( diff --git a/plugins/contact/src/index.ts b/plugins/contact/src/index.ts index 90e490c60a..bb6d5f077b 100644 --- a/plugins/contact/src/index.ts +++ b/plugins/contact/src/index.ts @@ -221,7 +221,8 @@ export const contactPlugin = plugin(contactId, { EditOrganizationPanel: '' as AnyComponent, CollaborationUserAvatar: '' as AnyComponent, CreateGuest: '' as AnyComponent, - SpaceMembersEditor: '' as AnyComponent + SpaceMembersEditor: '' as AnyComponent, + ContactNamePresenter: '' as AnyComponent }, channelProvider: { Email: '' as Ref, diff --git a/plugins/love-resources/src/components/Room.svelte b/plugins/love-resources/src/components/Room.svelte index 8fc32ddd3e..66a18590dc 100644 --- a/plugins/love-resources/src/components/Room.svelte +++ b/plugins/love-resources/src/components/Room.svelte @@ -58,6 +58,7 @@ connecting: boolean muted: boolean mirror: boolean + isAgent: boolean } let participants: ParticipantData[] = [] @@ -80,7 +81,7 @@ const element = track.attach() attachTrack(element, participant) } - updateStyle(participants.length, $screenSharing) + updateStyle(getActiveParticipants(participants).length, $screenSharing) } else { const part = participants.find((p) => p._id === participant.identity) if (part !== undefined) { @@ -105,7 +106,7 @@ } else { track.detach(screen) } - updateStyle(participants.length, $screenSharing) + updateStyle(getActiveParticipants(participants).length, $screenSharing) } } @@ -117,7 +118,7 @@ const element = publication.track.attach() void attachTrack(element, participant) } - updateStyle(participants.length, $screenSharing) + updateStyle(getActiveParticipants(participants).length, $screenSharing) } else { const part = participants.find((p) => p._id === participant.identity) if (part !== undefined) { @@ -135,7 +136,8 @@ name: participant.name ?? '', muted: !participant.isMicrophoneEnabled, mirror: participant.isLocal, - connecting: false + connecting: false, + isAgent: participant.isAgent }) } participants = participants @@ -166,11 +168,12 @@ name: participant.name ?? '', muted: !participant.isMicrophoneEnabled, mirror: participant.isLocal, - connecting: false + connecting: false, + isAgent: participant.isAgent } participants.push(value) participants = participants - updateStyle(participants.length, $screenSharing) + updateStyle(getActiveParticipants(participants).length, $screenSharing) } function handleParticipantDisconnected (participant: RemoteParticipant): void { @@ -179,7 +182,7 @@ participants.splice(index, 1) participants = participants } - updateStyle(participants.length, $screenSharing) + updateStyle(getActiveParticipants(participants).length, $screenSharing) } function muteHandler (publication: TrackPublication, participant: Participant): void { @@ -207,7 +210,7 @@ if (publication?.track?.kind === Track.Kind.Video) { if (publication.track.source === Track.Source.ScreenShare) { publication.track.detach(screen) - updateStyle(participants.length, $screenSharing) + updateStyle(getActiveParticipants(participants).length, $screenSharing) } else { const index = participants.findIndex((p) => p._id === participant.identity) if (index !== -1) { @@ -285,10 +288,6 @@ onDestroy( infos.subscribe((data) => { - const aiParticipant = aiPersonId !== undefined ? participants.find(({ _id }) => _id === aiPersonId) : undefined - if (aiParticipant && !data.some((it) => it.room === room._id && it.person === aiParticipant._id)) { - participants = participants.filter(({ _id }) => _id !== aiPersonId) - } for (const info of data) { if (info.room !== room._id) continue const current = participants.find((p) => p._id === info.person) @@ -298,12 +297,13 @@ name: info.name, muted: true, mirror: false, - connecting: true + connecting: true, + isAgent: aiPersonId === info.person } participants.push(value) } participants = participants - updateStyle(participants.length, $screenSharing) + updateStyle(getActiveParticipants(participants).length, $screenSharing) }) ) @@ -342,6 +342,12 @@ } } $: if (((document.fullscreenElement && !$isFullScreen) || $isFullScreen) && roomEl) toggleFullscreen() + + function getActiveParticipants (participants: ParticipantData[]): ParticipantData[] { + return participants.filter((p) => !p.isAgent || $infos.some(({ person }) => person === p._id)) + } + + $: activeParticipants = getActiveParticipants(participants)
@@ -376,7 +382,7 @@ style={$screenSharing ? '' : gridStyle} class:scroll-m-0={$screenSharing} > - {#each participants as participant, i (participant._id)} + {#each activeParticipants as participant, i (participant._id)} + + + +
+ {displayName} + {#if spaceName.trim().length > 0} + · + {spaceName} + {/if} +
+ + diff --git a/plugins/recruit-resources/src/components/MatchVacancy.svelte b/plugins/recruit-resources/src/components/MatchVacancy.svelte deleted file mode 100644 index 0bacc3e171..0000000000 --- a/plugins/recruit-resources/src/components/MatchVacancy.svelte +++ /dev/null @@ -1,272 +0,0 @@ - - - {}} - canSave={true} - on:changeContent -> - -
{ - dispatch('changeContent') - }} - > -
-
- - - - - -
-
- {#if vacancy} - -
- {#if vacancy.description} - {vacancy.description} - {/if} - {#if vacancyState?.fullSummary} - ')} /> - {/if} -
- - {/if} -
-
- - - - - - - - - - - - - - {#each _sortedObjects as doc} - {@const docState = state.get(doc._id)} - {@const docEmbedding = docState && getEmbedding(docState)} - {@const match = matches.get(doc._id)} - {@const appl = applications.get(doc._id)} - - - - - - - {/each} - -
#
-
- - {#if appl} -
- -
- {/if} -
- 		- {#if docEmbedding && vacancyEmbedding} - {Math.round(cosinesim(docEmbedding, vacancyEmbedding) * 100)} - / - {/if} - {scoreState.get(doc._id) ?? 0} - - {#if match?.complete} -
- -
- {/if} - 		- {#if docState} -
- -
- - diff --git a/plugins/recruit-resources/src/index.ts b/plugins/recruit-resources/src/index.ts index c7d74a698d..1b870990b4 100644 --- a/plugins/recruit-resources/src/index.ts +++ b/plugins/recruit-resources/src/index.ts @@ -42,7 +42,6 @@ import CreateVacancy from './components/CreateVacancy.svelte' import EditApplication from './components/EditApplication.svelte' import EditVacancy from './components/EditVacancy.svelte' import KanbanCard from './components/KanbanCard.svelte' -import MatchVacancy from './components/MatchVacancy.svelte' import NewCandidateHeader from './components/NewCandidateHeader.svelte' import NotificationApplicantPresenter from './components/NotificationApplicantPresenter.svelte' import Organizations from './components/Organizations.svelte' @@ -65,6 +64,7 @@ import Opinions from './components/review/Opinions.svelte' import OpinionsPresenter from './components/review/OpinionsPresenter.svelte' import ReviewPresenter from './components/review/ReviewPresenter.svelte' import Reviews from './components/review/Reviews.svelte' +import ApplicantNamePresenter from './components/ApplicantNamePresenter.svelte' import recruit from './plugin' import { getAppIdentifier, @@ -386,9 +386,9 @@ export default async (): Promise => ({ VacancyList, VacancyTemplateEditor, - MatchVacancy, NotificationApplicantPresenter, - VacancyEditor + VacancyEditor, + ApplicantNamePresenter }, completion: { ApplicationQuery: async ( diff --git a/plugins/recruit-resources/src/plugin.ts b/plugins/recruit-resources/src/plugin.ts index 7964203d81..c6e4c67081 100644 --- a/plugins/recruit-resources/src/plugin.ts +++ b/plugins/recruit-resources/src/plugin.ts @@ -114,7 +114,6 @@ export default mergeIds(recruitId, recruit, { HasNoActiveApplicant: '' as IntlString, NoneApplications: '' as IntlString, - MatchVacancy: '' as IntlString, VacancyMatching: '' as IntlString, Score: '' as IntlString, Match: '' as IntlString, diff --git a/plugins/view-resources/src/components/TableBrowser.svelte b/plugins/view-resources/src/components/TableBrowser.svelte index 8cba6ae5a6..44334835ad 100644 --- a/plugins/view-resources/src/components/TableBrowser.svelte +++ b/plugins/view-resources/src/components/TableBrowser.svelte @@ -14,15 +14,13 @@ --> - - - - { - showPopup(IndexedDocumentPreview, { objectId: value._id, search }, 'centered') - }} -> - {#if value.$source?.$score} - {Math.round(value.$source?.$score * 100) / 100} - {:else} - * - {/if} - diff --git a/plugins/view-resources/src/components/list/List.svelte b/plugins/view-resources/src/components/list/List.svelte index 6f123be806..87e3c05fd0 100644 --- a/plugins/view-resources/src/components/list/List.svelte +++ b/plugins/view-resources/src/components/list/List.svelte @@ -104,7 +104,7 @@ { ...categoryQueryOptions, limit: 1000 } ) - $: if (fastDocs.length === 1000) { + $: if (fastDocs.length === 1000 && queryNoLookup.$search == null) { docsQuerySlow.query( _class, queryNoLookup, @@ -113,6 +113,8 @@ }, categoryQueryOptions ) + } else { + slowDocs = [] } $: docs = [...fastDocs, ...slowDocs.filter((it) => !fastQueryIds.has(it._id))] diff --git a/plugins/view-resources/src/components/list/ListCategory.svelte b/plugins/view-resources/src/components/list/ListCategory.svelte index 0159abf192..96a3da701d 100644 --- a/plugins/view-resources/src/components/list/ListCategory.svelte +++ b/plugins/view-resources/src/components/list/ListCategory.svelte @@ -34,10 +34,10 @@ import { AttributeModel, BuildModelKey, ViewOptionModel, ViewOptions, Viewlet } from '@hcengineering/view' import { createEventDispatcher } from 'svelte' import { fade } from 'svelte/transition' + import { showMenu } from '../../actions' import { FocusSelection, SelectionFocusProvider, focusStore } from '../../selection' import ListHeader from './ListHeader.svelte' import ListItem from './ListItem.svelte' - import { showMenu } from '../../actions' export let category: PrimitiveType | AggregateValue export let headerComponent: AttributeModel | undefined @@ -108,11 +108,27 @@ } } + function noSearch (query: DocumentQuery): DocumentQuery { + const result = query + if ('$search' in result) { + delete result.$search + } + return result + } + + $: finalResultQuery = + itemProj.length < 20 && resultQuery.$search !== null + ? noSearch({ + ...resultQuery, + _id: { $in: itemProj.map((it) => it._id) } + }) + : resultQuery + $: if (lastLevel) { void limiter.add(async () => { loading = docsQuery.query( _class, - { ...resultQuery, ...docKeys }, + { ...finalResultQuery, ...docKeys }, (res) => { items = res loading = false diff --git a/plugins/view-resources/src/index.ts b/plugins/view-resources/src/index.ts index 0e08e278c5..98716a246a 100644 --- a/plugins/view-resources/src/index.ts +++ b/plugins/view-resources/src/index.ts @@ -126,7 +126,6 @@ import { valueNinResult } from './filter' -import { IndexedDocumentPreview } from '@hcengineering/presentation' import { AggregationMiddleware, AnalyticsMiddleware } from './middleware' import { showEmptyGroups } from './viewOptions' import { @@ -162,7 +161,6 @@ export { default as ViewletSelector } from './components/ViewletSelector.svelte' export { default as ViewletsSettingButton } from './components/ViewletsSettingButton.svelte' export { default as FilterButton } from './components/filter/FilterButton.svelte' export { default as FilterRemovedNotification } from './components/filter/FilterRemovedNotification.svelte' -export { default as SourcePresenter } from './components/inference/SourcePresenter.svelte' export { default as List } from './components/list/List.svelte' export { default as NavLink } from './components/navigator/NavLink.svelte' export { default as StatusPresenter } from './components/status/StatusPresenter.svelte' @@ -281,7 +279,6 @@ export default async (): Promise => ({ ListView, GrowPresenter, DividerPresenter, - IndexedDocumentPreview, SpaceRefPresenter, SpaceTypeSelector, EnumArrayEditor, diff --git a/pods/backup/src/index.ts b/pods/backup/src/index.ts index 9474550ead..279fe5beff 100644 --- a/pods/backup/src/index.ts +++ b/pods/backup/src/index.ts @@ -53,12 +53,8 @@ startBackup( return factory }, (ctx, dbUrl, workspace, branding, externalStorage) => { - return getConfig(ctx, dbUrl, workspace, branding, ctx, { + return getConfig(ctx, dbUrl, ctx, { externalStorage, - fullTextUrl: '', - indexParallel: 0, - indexProcessing: 0, - rekoniUrl: '', disableTriggers: true }) } diff --git a/pods/front/package.json b/pods/front/package.json index fe860f4f4b..88366245a7 100644 --- a/pods/front/package.json +++ b/pods/front/package.json @@ -22,7 +22,7 @@ "docker:tbuild": "docker build -t hardcoreeng/front . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/front", "docker:abuild": "docker build -t hardcoreeng/front . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/front", "format": "format src", - "run-local": "cross-env MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost SERVER_SECRET='secret' ACCOUNTS_URL=http://localhost:3000 UPLOAD_URL=/files ELASTIC_URL=http://localhost:9200 MODEL_VERSION=$(node ../../common/scripts/show_version.js) VERSION=$(node ../../common/scripts/show_tag.js) PUBLIC_DIR='.' ts-node ./src/__start.ts", + "run-local": "cross-env MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost SERVER_SECRET='secret' ACCOUNTS_URL=http://localhost:3000 UPLOAD_URL=/files MODEL_VERSION=$(node ../../common/scripts/show_version.js) VERSION=$(node ../../common/scripts/show_tag.js) PUBLIC_DIR='.' ts-node ./src/__start.ts", "test": "jest --passWithNoTests --silent --forceExit", "_phase:build": "compile transpile src", "_phase:test": "jest --passWithNoTests --silent --forceExit", @@ -64,7 +64,6 @@ "express-fileupload": "^1.5.1", "uuid": "^8.3.2", "cors": "^2.8.5", - "@hcengineering/elastic": "^0.6.0", "@hcengineering/server-core": "^0.6.1", "@hcengineering/server-token": "^0.6.11", "@hcengineering/attachment": "^0.6.14", diff --git a/pods/fulltext/.eslintrc.js b/pods/fulltext/.eslintrc.js new file mode 100644 index 0000000000..ce90fb9646 --- /dev/null +++ b/pods/fulltext/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/pods/fulltext/.npmignore b/pods/fulltext/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/pods/fulltext/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/pods/fulltext/Dockerfile b/pods/fulltext/Dockerfile new file mode 100644 index 0000000000..ad9de1b47a --- /dev/null +++ b/pods/fulltext/Dockerfile @@ -0,0 +1,20 @@ + +FROM node:20 + +WORKDIR /usr/src/app + +RUN npm install --ignore-scripts=false --verbose bufferutil utf-8-validate @mongodb-js/zstd snappy --unsafe-perm + +RUN apt-get update +RUN apt-get install libjemalloc2 +RUN apt-get clean + +ENV LD_PRELOAD=libjemalloc.so.2 +ENV MALLOC_CONF=dirty_decay_ms:1000,narenas:2,background_thread:true + +COPY bundle/bundle.js ./ +COPY bundle/bundle.js.map ./ +COPY bundle/model.json ./ + +EXPOSE 4700 +CMD [ "node", "--expose-gc", "bundle.js" ] diff --git a/pods/fulltext/build.sh b/pods/fulltext/build.sh new file mode 100755 index 0000000000..c6f3ac4e30 --- /dev/null +++ b/pods/fulltext/build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright © 2022 Hardcore Engineering Inc. +# +# Licensed under the Eclipse Public License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. You may +# obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# + +rushx bundle +rushx docker:build +rushx docker:push diff --git a/pods/fulltext/config/rig.json b/pods/fulltext/config/rig.json new file mode 100644 index 0000000000..78cc5a1733 --- /dev/null +++ b/pods/fulltext/config/rig.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig", + "rigProfile": "node" +} diff --git a/pods/fulltext/jest.config.js b/pods/fulltext/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/pods/fulltext/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/pods/fulltext/package.json b/pods/fulltext/package.json new file mode 100644 index 0000000000..ed47d1c9e0 --- /dev/null +++ b/pods/fulltext/package.json @@ -0,0 +1,77 @@ +{ + "name": "@hcengineering/pod-fulltext", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "author": "Anticrm Platform Contributors", + "template": "@hcengineering/node-package", + "license": "EPL-2.0", + "scripts": { + "start": "ts-node src/index.ts", + "build": "compile", + "build:watch": "compile", + "_phase:bundle": "rushx bundle", + "_phase:docker-build": "rushx docker:build", + "_phase:docker-staging": "rushx docker:staging", + "get-model": "mkdir -p bundle && esbuild src/get-model.ts --bundle --keep-names --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.VERSION=$(node ../../common/scripts/show_tag.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error && node ./bundle/bundle.js > ./bundle/model.json", + "bundle": "mkdir -p bundle && rushx get-model && esbuild src/index.ts --keep-names --bundle --platform=node --external:*.node --outfile=bundle/bundle.js --log-level=error --sourcemap=external", + "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/fulltext", + "docker:tbuild": "docker build -t hardcoreeng/fulltext . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/fulltext", + "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/fulltext staging", + "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/fulltext", + "format": "format src", + "test": "jest --passWithNoTests --silent --forceExit", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent --forceExit", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "cross-env": "~7.0.3", + "@hcengineering/platform-rig": "^0.6.0", + "@types/node": "~20.11.16", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "esbuild": "^0.20.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "ts-node": "^10.8.0", + "typescript": "^5.3.3", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5", + "@hcengineering/model-all": "^0.6.0", + "@types/koa-bodyparser": "^4.3.12", + "@types/koa-router": "^7.4.8", + "@types/koa": "^2.15.0", + "@types/koa__cors": "^5.0.0" + }, + "dependencies": { + "@hcengineering/platform": "^0.6.11", + "@hcengineering/server-token": "^0.6.11", + "@hcengineering/server-core": "^0.6.1", + "@hcengineering/server-pipeline": "^0.6.0", + "@hcengineering/client": "^0.6.18", + "@hcengineering/client-resources": "^0.6.27", + "@hcengineering/core": "^0.6.32", + "dotenv": "~16.0.0", + "@hcengineering/backup-service": "^0.6.0", + "@hcengineering/analytics": "^0.6.0", + "@hcengineering/analytics-service": "^0.6.0", + "koa": "^2.15.3", + "koa-router": "^12.0.1", + "koa-bodyparser": "^4.4.1", + "@koa/cors": "^5.0.0", + "@hcengineering/server-indexer": "^0.6.0", + "@hcengineering/elastic": "^0.6.0", + "@hcengineering/server-collaboration": "^0.6.0", + "@hcengineering/middleware": "^0.6.0", + "@hcengineering/server-client": "^0.6.0", + "@hcengineering/server-storage": "^0.6.0" + } +} diff --git a/pods/fulltext/run.sh b/pods/fulltext/run.sh new file mode 100755 index 0000000000..2133b2a131 --- /dev/null +++ b/pods/fulltext/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash +export FULLTEXT_DB_URL=http://localhost:9200 +export DB_URL=mongodb://localhost:27017 + +# "DB_URL": "postgresql://postgres:example@localhost:5432", +export STORAGE_CONFIG="minio|localhost?accessKey=minioadmin&secretKey=minioadmin" +export SERVER_SECRET=secret +export REKONI_URL=http://localhost:4004 +export MODEL_JSON=./bundle/model.json +export ELASTIC_INDEX_NAME=local_storage_index +export STATS_URL=http://host.docker.internal:4900 +export ACCOUNTS_URL=http://localhost:3000 +rushx bundle +node --inspect ./bundle/bundle.js \ No newline at end of file diff --git a/pods/fulltext/src/get-model.ts b/pods/fulltext/src/get-model.ts new file mode 100644 index 0000000000..9f9b31b293 --- /dev/null +++ b/pods/fulltext/src/get-model.ts @@ -0,0 +1 @@ +import '@hcengineering/model-all/src/show' diff --git a/pods/fulltext/src/index.ts b/pods/fulltext/src/index.ts new file mode 100644 index 0000000000..542d5b19ce --- /dev/null +++ b/pods/fulltext/src/index.ts @@ -0,0 +1,138 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Analytics } from '@hcengineering/analytics' +import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service' +import { MeasureMetricsContext, newMetrics, type Tx } from '@hcengineering/core' +import { initStatisticsContext, type StorageConfiguration } from '@hcengineering/server-core' +import { join } from 'path' + +import { createElasticAdapter } from '@hcengineering/elastic' +import { createRekoniAdapter, createYDocAdapter, type FulltextDBConfiguration } from '@hcengineering/server-indexer' +import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' +import { readFileSync } from 'fs' +import { startIndexer } from './server' + +const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[] + +const serverSecret = process.env.SERVER_SECRET +if (serverSecret === undefined) { + console.info('Please provide server secret') + process.exit(1) +} + +const metricsContext = initStatisticsContext('fulltext', { + factory: () => + new MeasureMetricsContext( + 'fulltext', + {}, + {}, + newMetrics(), + new SplitLogger('fulltext', { + root: join(process.cwd(), 'logs'), + enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true' + }) + ) +}) + +const sentryDSN = process.env.SENTRY_DSN + +configureAnalytics(sentryDSN, {}) +Analytics.setTag('application', 'fulltext') + +const dbURL = process.env.DB_URL +if (dbURL === undefined) { + console.error('DB_URL should be specified') + process.exit(1) +} + +const fullTextDbURL = process.env.FULLTEXT_DB_URL +if (fullTextDbURL === undefined) { + console.error('FULLTEXT_DB_URL should be specified') + process.exit(1) +} + +const rekoniUrl = process.env.REKONI_URL +if (rekoniUrl === undefined) { + console.error('REKONI_URL should be specified') + process.exit(1) +} + +const config: FulltextDBConfiguration = { + fulltextAdapter: { + factory: createElasticAdapter, + url: fullTextDbURL + }, + contentAdapters: { + Rekoni: { + factory: createRekoniAdapter, + contentType: '*', + url: rekoniUrl + }, + YDoc: { + factory: createYDocAdapter, + contentType: 'application/ydoc', + url: '' + } + }, + defaultContentAdapter: 'Rekoni' +} + +const elasticIndexName = process.env.ELASTIC_INDEX_NAME +if (elasticIndexName === undefined) { + console.log('Please provide ELASTIC_INDEX_NAME') + process.exit(1) +} + +const servicePort = parseInt(process.env.PORT ?? '4700') +metricsContext.info('Starting stats service') + +const accountsUrl = process.env.ACCOUNTS_URL +if (accountsUrl === undefined) { + console.error('please provide account url') + process.exit(1) +} + +const storageConfig: StorageConfiguration = storageConfigFromEnv() +const externalStorage = buildStorageFromConfig(storageConfig) + +const onClose = startIndexer(metricsContext, { + model, + config, + externalStorage, + elasticIndexName, + dbURL, + port: servicePort, + serverSecret, + accountsUrl +}) + +process.on('uncaughtException', (e) => { + metricsContext.error('uncaughtException', { error: e }) +}) + +process.on('unhandledRejection', (reason, promise) => { + metricsContext.error('Unhandled Rejection at:', { reason, promise }) +}) + +const close = (): void => { + void onClose.then((res) => { + res() + }) +} + +process.on('SIGINT', close) +process.on('SIGTERM', close) +process.on('exit', close) diff --git a/pods/fulltext/src/server.ts b/pods/fulltext/src/server.ts new file mode 100644 index 0000000000..79f1b190fd --- /dev/null +++ b/pods/fulltext/src/server.ts @@ -0,0 +1,400 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +import { Analytics } from '@hcengineering/analytics' +import type { + Class, + Doc, + DocumentQuery, + IndexingUpdateEvent, + MeasureContext, + Ref, + SearchOptions, + SearchQuery, + Tx, + TxWorkspaceEvent, + WorkspaceId, + WorkspaceIdWithUrl +} from '@hcengineering/core' +import core, { + DOMAIN_DOC_INDEX_STATE, + generateId, + Hierarchy, + ModelDb, + systemAccountEmail, + WorkspaceEvent +} from '@hcengineering/core' +import { + ContextNameMiddleware, + DBAdapterInitMiddleware, + DBAdapterMiddleware, + DomainFindMiddleware, + LowLevelMiddleware, + ModelMiddleware +} from '@hcengineering/middleware' +import { PlatformError, setMetadata, unknownError } from '@hcengineering/platform' +import serverClientPlugin, { getTransactorEndpoint, getWorkspaceInfo } from '@hcengineering/server-client' +import serverCore, { + createContentAdapter, + createPipeline, + type ContentTextAdapter, + type FullTextAdapter, + type MiddlewareCreator, + type Pipeline, + type PipelineContext, + type StorageAdapter +} from '@hcengineering/server-core' +import { FullTextIndexPipeline, searchFulltext, type FulltextDBConfiguration } from '@hcengineering/server-indexer' +import { getConfig, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline' +import serverToken, { decodeToken, generateToken, type Token } from '@hcengineering/server-token' +import cors from '@koa/cors' +import Koa from 'koa' +import bodyParser from 'koa-bodyparser' +import Router from 'koa-router' + +class WorkspaceIndexer { + fulltext!: FullTextIndexPipeline + pipeline!: Pipeline + + lastUpdate: number = Date.now() + + constructor (readonly fulltextAdapter: FullTextAdapter) {} + + static async create ( + ctx: MeasureContext, + model: Tx[], + workspace: WorkspaceIdWithUrl, + dbURL: string, + externalStorage: StorageAdapter, + ftadapter: FullTextAdapter, + contentAdapter: ContentTextAdapter + ): Promise { + const result = new WorkspaceIndexer(ftadapter) + const dbConf = getConfig(ctx, dbURL, ctx, { + disableTriggers: true, + externalStorage + }) + + const middlewares: MiddlewareCreator[] = [ + LowLevelMiddleware.create, + ContextNameMiddleware.create, + DomainFindMiddleware.create, + DBAdapterInitMiddleware.create, + ModelMiddleware.create(model), + DBAdapterMiddleware.create(dbConf) + ] + + const hierarchy = new Hierarchy() + const modelDb = new ModelDb(hierarchy) + + const context: PipelineContext = { + workspace, + branding: null, + modelDb, + hierarchy, + storageAdapter: externalStorage + } + result.pipeline = await createPipeline(ctx, middlewares, context) + + const defaultAdapter = result.pipeline.context.adapterManager?.getDefaultAdapter() + if (defaultAdapter === undefined) { + throw new PlatformError(unknownError('Default adapter should be set')) + } + + const token = generateToken(systemAccountEmail, workspace) + const transactorEndpoint = (await getTransactorEndpoint(token, 'internal')) + .replace('wss://', 'https://') + .replace('ws://', 'http://') + + result.fulltext = new FullTextIndexPipeline( + ftadapter, + defaultAdapter, + hierarchy, + workspace, + ctx, + modelDb, + externalStorage, + contentAdapter, + (ctx: MeasureContext, classes: Ref>[]) => { + ctx.info('broadcast indexing update', { classes, workspace }) + const evt: IndexingUpdateEvent = { + _class: classes + } + const tx: TxWorkspaceEvent = { + _class: core.class.TxWorkspaceEvent, + _id: generateId(), + event: WorkspaceEvent.IndexingUpdate, + modifiedBy: core.account.System, + modifiedOn: Date.now(), + objectSpace: core.space.DerivedTx, + space: core.space.DerivedTx, + params: evt + } + // Send tx to pipeline + // TODO: Fix me + void fetch(transactorEndpoint + `/api/v1/broadcast?token=${token}&workspace=${workspace.name}`, { + method: 'PUT', + body: JSON.stringify(tx) + }) + }, + async () => { + const helper = result.pipeline.context.adapterManager?.domainHelper + if (helper !== undefined) { + const dhelper = result.pipeline.context.adapterManager?.getAdapter(DOMAIN_DOC_INDEX_STATE, true).helper?.() + if (dhelper !== undefined) { + // Force creation of indexes for search domain + await helper.checkDomain(ctx, DOMAIN_DOC_INDEX_STATE, 10000, dhelper) + } + } + }, + async (ctx: MeasureContext) => { + await result.pipeline.context.adapterManager?.closeContext?.(ctx) + } + ) + await result.fulltext.startIndexing(() => { + result.lastUpdate = Date.now() + }) + return result + } + + async close (): Promise { + await this.fulltext.cancel() + await this.pipeline.close() + } +} + +interface IndexDocuments { + token: string + requests: { + _class: Ref> + _id: Ref + }[] +} + +interface FulltextSearch { + token: string + query: SearchQuery + options: SearchOptions +} + +interface Search { + token: string + _classes: Ref>[] + query: DocumentQuery + fullTextLimit: number +} + +export async function startIndexer ( + ctx: MeasureContext, + opt: { + model: Tx[] + dbURL: string + config: FulltextDBConfiguration + externalStorage: StorageAdapter + elasticIndexName: string + port: number + serverSecret: string + accountsUrl: string + } +): Promise<() => void> { + const closeTimeout = 5 * 60 * 1000 + + setMetadata(serverToken.metadata.Secret, opt.serverSecret) + setMetadata(serverCore.metadata.ElasticIndexName, opt.elasticIndexName) + setMetadata(serverClientPlugin.metadata.Endpoint, opt.accountsUrl) + + registerServerPlugins() + registerStringLoaders() + + const sysHierarchy = new Hierarchy() + for (const tx of opt.model) { + sysHierarchy.tx(tx) + } + + const app = new Koa() + const router = new Router() + + const indexers = new Map>() + + const contentAdapter = await ctx.with('create content adapter', {}, (ctx) => + createContentAdapter(opt.config.contentAdapters, opt.config.defaultContentAdapter) + ) + const fulltextAdapter = await opt.config.fulltextAdapter.factory(opt.config.fulltextAdapter.url) + + const shutdownInterval = setInterval(() => { + for (const [k, v] of [...indexers.entries()]) { + if (v instanceof Promise) { + continue + } + if (Date.now() - v.lastUpdate > closeTimeout) { + indexers.delete(k) + void v.close() + } + } + }, closeTimeout) // Every 5 minutes we should close unused indexes. + + async function getIndexer ( + ctx: MeasureContext, + workspace: WorkspaceId, + token: string, + create: boolean = false + ): Promise { + const workspaceInfo = await getWorkspaceInfo(token) + let idx = indexers.get(workspace.name) + if (idx === undefined && create) { + if (workspaceInfo === undefined) { + ctx.error('Workspace not available for token') + return + } + idx = WorkspaceIndexer.create( + ctx, + opt.model, + { + ...workspace, + workspaceName: workspaceInfo.workspaceName ?? workspaceInfo.workspace, + workspaceUrl: workspaceInfo.workspaceUrl ?? workspaceInfo.workspace + }, + opt.dbURL, + opt.externalStorage, + fulltextAdapter, + contentAdapter + ) + indexers.set(workspace.name, idx) + } + if (idx instanceof Promise) { + idx = await idx + indexers.set(workspace.name, idx) + } + return idx + } + + app.use( + cors({ + credentials: true + }) + ) + app.use(bodyParser()) + + router.put('/api/v1/search', async (req, res) => { + try { + const request = req.request.body as Search + const decoded = decodeToken(request.token) // Just to be safe + + ctx.info('search', { classes: request._classes, query: request.query, workspace: decoded.workspace }) + await ctx.with('search', {}, async (ctx) => { + const docs = await ctx.with('search', { workspace: decoded.workspace.name }, (ctx) => + fulltextAdapter.search(ctx, decoded.workspace, request._classes, request.query, request.fullTextLimit) + ) + req.body = docs + }) + + void triggerIndexer(decoded, request.token) + } catch (err: any) { + Analytics.handleError(err) + console.error(err) + req.res.writeHead(404, {}) + req.res.end() + } + }) + + router.put('/api/v1/full-text-search', async (req, res) => { + try { + const request = req.request.body as FulltextSearch + const decoded = decodeToken(request.token) // Just to be safe + ctx.info('fulltext-search', { ...request.query, workspace: decoded.workspace }) + await ctx.with('full-text-search', {}, async (ctx) => { + const result = await ctx.with('searchFulltext', {}, (ctx) => + searchFulltext(ctx, decoded.workspace, sysHierarchy, fulltextAdapter, request.query, request.options) + ) + req.body = result + }) + + void triggerIndexer(decoded, request.token) + } catch (err: any) { + Analytics.handleError(err) + console.error(err) + req.res.writeHead(404, {}) + req.res.end() + } + }) + + router.put('/api/v1/warmup', async (req, res) => { + try { + const request = req.request.body as IndexDocuments + const decoded = decodeToken(request.token) // Just to be safe + req.body = {} + + ctx.info('warm-up', { workspace: decoded.workspace }) + void triggerIndexer(decoded, request.token, true) + } catch (err: any) { + Analytics.handleError(err) + console.error(err) + req.res.writeHead(404, {}) + req.res.end() + } + }) + + router.put('/api/v1/close', async (req, res) => { + try { + const request = req.request.body as IndexDocuments + const decoded = decodeToken(request.token) // Just to be safe + req.body = {} + + ctx.info('close', { workspace: decoded.workspace }) + const idx = indexers.get(decoded.workspace.name) + indexers.delete(decoded.workspace.name) + if (idx !== undefined && idx instanceof Promise) { + void idx.then((res) => { + void res.close() + }) + } else if (idx !== undefined) { + void idx.close() + } + } catch (err: any) { + Analytics.handleError(err) + console.error(err) + req.res.writeHead(404, {}) + req.res.end() + } + }) + + router.put('/api/v1/index-documents', async (req, res) => { + try { + const request = req.request.body as IndexDocuments + const decoded = decodeToken(request.token) // Just to be safe + + const indexer = await getIndexer(ctx, decoded.workspace, request.token) + if (indexer !== undefined) { + indexer.lastUpdate = Date.now() + await ctx.with('index-documents', {}, (ctx) => indexer.fulltext.indexDocuments(ctx, request.requests)) + } + req.body = {} + } catch (err: any) { + Analytics.handleError(err) + console.error(err) + req.res.writeHead(404, {}) + req.res.end() + } + }) + + app.use(router.routes()).use(router.allowedMethods()) + + const server = app.listen(opt.port, () => { + console.log(`server started on port ${opt.port}`) + }) + + const close = (): void => { + clearInterval(shutdownInterval) + server.close() + } + + return close + + async function triggerIndexer (decoded: Token, token: string, trigger: boolean = false): Promise { + const indexer = await getIndexer(ctx, decoded.workspace, token, trigger) + if (indexer !== undefined) { + indexer.lastUpdate = Date.now() + if (trigger) { + indexer.fulltext.triggerIndexing() + } + } + } +} diff --git a/pods/fulltext/tsconfig.json b/pods/fulltext/tsconfig.json new file mode 100644 index 0000000000..f017cc597c --- /dev/null +++ b/pods/fulltext/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/pods/server/logs/server-combined-2024-08-14.log.gz b/pods/server/logs/server-combined-2024-08-14.log.gz deleted file mode 100644 index 96e0332f96..0000000000 Binary files a/pods/server/logs/server-combined-2024-08-14.log.gz and /dev/null differ diff --git a/pods/server/package.json b/pods/server/package.json index d58215176e..8b7d96ed4e 100644 --- a/pods/server/package.json +++ b/pods/server/package.json @@ -8,9 +8,9 @@ "template": "@hcengineering/node-package", "license": "EPL-2.0", "scripts": { - "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false MODEL_JSON=../../models/all/bundle/model.json node --inspect bundle/bundle.js", - "start-u": "rush bundle --to @hcengineering/pod-server && ./bundle/ && cross-env NODE_ENV=production SERVER_PROVIDER=uweb ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json node bundle/bundle.js", - "start-flame": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json clinic flame --dest ./out -- node --nolazy -r ts-node/register --enable-source-maps src/__start.ts", + "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false MODEL_JSON=../../models/all/bundle/model.json node --inspect bundle/bundle.js", + "start-u": "rush bundle --to @hcengineering/pod-server && ./bundle/ && cross-env NODE_ENV=production MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json node bundle/bundle.js", + "start-flame": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json clinic flame --dest ./out -- node --nolazy -r ts-node/register --enable-source-maps src/__start.ts", "build": "compile", "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", @@ -61,7 +61,6 @@ "@hcengineering/server-storage": "^0.6.0", "@hcengineering/server-pipeline": "^0.6.0", "@hcengineering/mongo": "^0.6.1", - "@hcengineering/elastic": "^0.6.0", "@hcengineering/server-token": "^0.6.11", "@hcengineering/middleware": "^0.6.0", "@hcengineering/minio": "^0.6.0", diff --git a/pods/server/src/__start.ts b/pods/server/src/__start.ts index 43c51895ab..c52052600f 100644 --- a/pods/server/src/__start.ts +++ b/pods/server/src/__start.ts @@ -68,20 +68,16 @@ setMetadata(serverNotification.metadata.SesUrl, config.sesUrl ?? '') setMetadata(notification.metadata.PushPublicKey, config.pushPublicKey) setMetadata(serverNotification.metadata.PushPrivateKey, config.pushPrivateKey) setMetadata(serverNotification.metadata.PushSubject, config.pushSubject) -setMetadata(serverCore.metadata.ElasticIndexName, config.elasticIndexName) setMetadata(serverCore.metadata.ElasticIndexVersion, 'v1') setMetadata(serverTelegram.metadata.BotUrl, process.env.TELEGRAM_BOT_URL) setMetadata(serverAiBot.metadata.SupportWorkspaceId, process.env.SUPPORT_WORKSPACE) setMetadata(serverAiBot.metadata.EndpointURL, process.env.AI_BOT_URL) const { shutdown, sessionManager } = start(metricsContext, config.dbUrl, { - fullTextUrl: config.elasticUrl, + fulltextUrl: config.fulltextUrl, storageConfig, - rekoniUrl: config.rekoniUrl, port: config.serverPort, serverFactory: startHttpServer, - indexParallel: 2, - indexProcessing: 500, brandingMap: loadBrandingMap(config.brandingPath), accountsUrl: config.accountsUrl, enableCompression: config.enableCompression, diff --git a/pods/server/src/server.ts b/pods/server/src/server.ts index 789a2bb05f..4266c65a3e 100644 --- a/pods/server/src/server.ts +++ b/pods/server/src/server.ts @@ -47,16 +47,12 @@ export function start ( metrics: MeasureContext, dbUrl: string, opt: { - fullTextUrl: string + fulltextUrl: string storageConfig: StorageConfiguration - rekoniUrl: string port: number brandingMap: BrandingMap serverFactory: ServerFactory - indexProcessing: number // 1000 - indexParallel: number // 2 - enableCompression?: boolean accountsUrl: string diff --git a/qms-tests/docker-compose.yaml b/qms-tests/docker-compose.yaml index 9606f1d576..965b7ec3a9 100644 --- a/qms-tests/docker-compose.yaml +++ b/qms-tests/docker-compose.yaml @@ -116,7 +116,6 @@ services: pull_policy: never links: - mongodb - - elastic - minio - rekoni - account @@ -128,7 +127,6 @@ services: - SERVER_PROVIDER=${SERVER_PROVIDER} - SERVER_PORT=3334 - SERVER_SECRET=secret - - ELASTIC_URL=http://elastic:9200 - DB_URL=mongodb://mongodb:27018 - MONGO_URL=mongodb://mongodb:27018 - METRICS_CONSOLE=false @@ -139,7 +137,7 @@ services: - UPLOAD_URL=http://host.docker.internal:8083/files - ACCOUNTS_URL=http://account:3003 - LAST_NAME_FIRST=true - - ELASTIC_INDEX_NAME=local_storage_index + - FULLTEXT_URL=http://fulltext:4710 - BRANDING_PATH=/var/cfg/branding-test.json collaborator: image: hardcoreeng/collaborator @@ -203,5 +201,21 @@ services: resources: limits: memory: 300M -volumes: - files: + fulltext: + image: hardcoreeng/fulltext + restart: unless-stopped + links: + - elastic + - mongodb + - rekoni + ports: + - 4710:4710 + environment: + - PORT=4710 + - SERVER_SECRET=secret + - DB_URL=mongodb://mongodb:27018 + - FULLTEXT_DB_URL=http://elastic:9200 + - ELASTIC_INDEX_NAME=local_storage_index + - STORAGE_CONFIG=${STORAGE_CONFIG} + - REKONI_URL=http://rekoni:4007 + - ACCOUNTS_URL=http://account:3003 diff --git a/rush.json b/rush.json index bd5e5ca6a3..3547e832a3 100644 --- a/rush.json +++ b/rush.json @@ -1441,6 +1441,26 @@ "projectFolder": "pods/backup", "shouldPublish": false }, + { + "packageName": "@hcengineering/pod-fulltext", + "projectFolder": "pods/fulltext", + "shouldPublish": false + }, + { + "packageName": "@hcengineering/server-fulltext", + "projectFolder": "server-plugins/fulltext", + "shouldPublish": false + }, + { + "packageName": "@hcengineering/server-fulltext-resources", + "projectFolder": "server-plugins/fulltext-resources", + "shouldPublish": false + }, + { + "packageName": "@hcengineering/model-server-fulltext", + "projectFolder": "models/server-fulltext", + "shouldPublish": false + }, { "packageName": "@hcengineering/hr", "projectFolder": "plugins/hr", diff --git a/server-plugins/collaboration/src/fulltext.ts b/server-plugins/collaboration/src/fulltext.ts deleted file mode 100644 index f07fbab1fc..0000000000 --- a/server-plugins/collaboration/src/fulltext.ts +++ /dev/null @@ -1,165 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import core, { - Blob, - Class, - CollaborativeDoc, - Doc, - DocIndexState, - DocumentQuery, - DocumentUpdate, - MeasureContext, - Ref, - WorkspaceId, - collaborativeDocParse, - getFullTextIndexableAttributes -} from '@hcengineering/core' -import { ContentTextAdapter, DbAdapter, IndexedDoc, StorageAdapter } from '@hcengineering/server-core' - -import { - DocUpdateHandler, - FullTextPipeline, - FullTextPipelineStage, - collabStageId, - contentStageId, - docKey, - docUpdKey, - fieldStateId -} from '@hcengineering/server-indexer' - -/** - * @public - */ -export class CollaborativeContentRetrievalStage implements FullTextPipelineStage { - require = [] - stageId = collabStageId - - extra = ['content', 'base64'] - - enabled = true - - // Clear all except following. - clearExcept: string[] = [fieldStateId, contentStageId] - - updateFields: DocUpdateHandler[] = [] - - textLimit = 100 * 1024 - - constructor ( - readonly storageAdapter: StorageAdapter | undefined, - readonly workspace: WorkspaceId, - readonly metrics: MeasureContext, - private readonly contentAdapter: ContentTextAdapter - ) {} - - async initialize (ctx: MeasureContext, storage: DbAdapter, pipeline: FullTextPipeline): Promise { - // Just do nothing - } - - async search ( - _classes: Ref>[], - search: DocumentQuery, - size?: number, - from?: number - ): Promise<{ docs: IndexedDoc[], pass: boolean }> { - return { docs: [], pass: true } - } - - async collect (toIndex: DocIndexState[], pipeline: FullTextPipeline): Promise { - for (const doc of toIndex) { - if (pipeline.cancelling) { - return - } - await this.updateContent(doc, pipeline) - } - } - - async updateContent (doc: DocIndexState, pipeline: FullTextPipeline): Promise { - const attributes = getFullTextIndexableAttributes(pipeline.hierarchy, doc.objectClass) - // Copy content attributes as well. - const update: DocumentUpdate = {} - - if (pipeline.cancelling) { - return - } - - try { - for (const [, val] of Object.entries(attributes)) { - if (val.type._class === core.class.TypeCollaborativeDoc) { - const collaborativeDoc = doc.attributes[docKey(val.name, { _class: val.attributeOf })] as CollaborativeDoc - if (collaborativeDoc !== undefined && collaborativeDoc !== '') { - const { documentId } = collaborativeDocParse(collaborativeDoc) - - const docInfo: Blob | undefined = await this.storageAdapter?.stat(this.metrics, this.workspace, documentId) - - if (docInfo !== undefined) { - const digest = docInfo.etag - const digestKey = docKey(val.name, { _class: val.attributeOf, digest: true }) - if (doc.attributes[digestKey] !== digest) { - ;(update as any)[docUpdKey(digestKey)] = digest - - const contentType = (docInfo.contentType ?? '').split(';')[0] - const readable = await this.storageAdapter?.get(this.metrics, this.workspace, documentId) - - if (readable !== undefined) { - let textContent = await this.metrics.with( - 'fetch', - {}, - async () => await this.contentAdapter.content(documentId, contentType, readable) - ) - readable?.destroy() - - textContent = textContent - .split(/ +|\t+|\f+/) - .filter((it) => it) - .join(' ') - .split(/\n\n+/) - .join('\n') - - // trim to large content - if (textContent.length > this.textLimit) { - textContent = textContent.slice(0, this.textLimit) - } - textContent = Buffer.from(textContent).toString('base64') - ;(update as any)[docUpdKey(val.name, { _class: val.attributeOf, extra: this.extra })] = textContent - } - } - } - } - } - } - } catch (err: any) { - const wasError = (doc as any).error !== undefined - - await pipeline.update(doc._id, false, { [docKey('error')]: JSON.stringify({ message: err.message, err }) }) - if (wasError) { - return - } - // Print error only first time, and update it in doc index - console.error(err) - return - } - - await pipeline.update(doc._id, true, update) - } - - async remove (docs: DocIndexState[], pipeline: FullTextPipeline): Promise { - // will be handled by field processor - for (const doc of docs) { - await pipeline.update(doc._id, true, {}) - } - } -} diff --git a/server-plugins/collaboration/src/index.ts b/server-plugins/collaboration/src/index.ts index a98c5b3620..7a950e9f23 100644 --- a/server-plugins/collaboration/src/index.ts +++ b/server-plugins/collaboration/src/index.ts @@ -17,8 +17,6 @@ import type { Plugin, Resource } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' import type { TriggerFunc } from '@hcengineering/server-core' -export * from './fulltext' - /** * @public */ diff --git a/server-plugins/contact-resources/src/index.ts b/server-plugins/contact-resources/src/index.ts index 0ccbbb07a6..f1235f9c75 100644 --- a/server-plugins/contact-resources/src/index.ts +++ b/server-plugins/contact-resources/src/index.ts @@ -31,7 +31,6 @@ import contact, { } from '@hcengineering/contact' import core, { Account, - Class, Doc, Hierarchy, Ref, @@ -43,7 +42,8 @@ import core, { TxProcessor, TxRemoveDoc, TxUpdateDoc, - concatLink + concatLink, + type Space } from '@hcengineering/core' import notification, { Collaborators } from '@hcengineering/notification' import { getMetadata } from '@hcengineering/platform' @@ -285,9 +285,17 @@ export function organizationTextPresenter (doc: Doc): string { /** * @public */ -export function contactNameProvider (hierarchy: Hierarchy, props: Record): string { - const _class = props._class !== undefined ? (props._class as Ref>) : contact.class.Contact - return formatContactName(hierarchy, _class, props.name ?? '', props.lastNameFirst) +export function contactNameProvider ( + doc: Doc, + parent: Doc | undefined, + space: Space | undefined, + hierarchy: Hierarchy, + mode: string +): string { + if (parent !== undefined && hierarchy.isDerived(parent._class, contact.class.Contact)) { + return formatContactName(hierarchy, parent._class, (parent as Contact).name ?? '', mode) + } + return formatContactName(hierarchy, doc._class, (doc as Contact).name ?? '', mode) } export async function getCurrentEmployeeName (control: TriggerControl, context: Record): Promise { diff --git a/server-plugins/fulltext-resources/.eslintrc.js b/server-plugins/fulltext-resources/.eslintrc.js new file mode 100644 index 0000000000..72235dc283 --- /dev/null +++ b/server-plugins/fulltext-resources/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/server-plugins/fulltext-resources/.npmignore b/server-plugins/fulltext-resources/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/server-plugins/fulltext-resources/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/server-plugins/fulltext-resources/config/rig.json b/server-plugins/fulltext-resources/config/rig.json new file mode 100644 index 0000000000..0110930f55 --- /dev/null +++ b/server-plugins/fulltext-resources/config/rig.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig" +} diff --git a/server-plugins/fulltext-resources/jest.config.js b/server-plugins/fulltext-resources/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/server-plugins/fulltext-resources/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/server-plugins/fulltext-resources/package.json b/server-plugins/fulltext-resources/package.json new file mode 100644 index 0000000000..a7fdd1933b --- /dev/null +++ b/server-plugins/fulltext-resources/package.json @@ -0,0 +1,44 @@ +{ + "name": "@hcengineering/server-fulltext-resources", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "files": [ + "lib/**/*", + "types/**/*", + "tsconfig.json" + ], + "author": "Anticrm Platform Contributors", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "format": "format src", + "test": "jest --passWithNoTests --silent", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "^0.6.0", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "typescript": "^5.3.3", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5" + }, + "dependencies": { + "@hcengineering/core": "^0.6.32", + "@hcengineering/platform": "^0.6.11", + "@hcengineering/server-core": "^0.6.1" + } +} diff --git a/server-plugins/fulltext-resources/src/index.ts b/server-plugins/fulltext-resources/src/index.ts new file mode 100644 index 0000000000..14b4b83380 --- /dev/null +++ b/server-plugins/fulltext-resources/src/index.ts @@ -0,0 +1,92 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import core, { + DocIndexState, + DOMAIN_DOC_INDEX_STATE, + getFullTextContext, + isClassIndexable, + Tx, + TxProcessor, + type AttachedDoc, + type Class, + type Doc, + type FullTextSearchContext, + type Ref, + type TxCollectionCUD, + type TxCUD +} from '@hcengineering/core' +import { TriggerControl } from '@hcengineering/server-core' + +const indexingCtx = 'indexing_contexts' +export async function OnChange (txes: Tx[], control: TriggerControl): Promise { + let contexts = control.contextCache.get(indexingCtx) as Map>, FullTextSearchContext> + if (contexts === undefined) { + contexts = new Map() + control.contextCache.set(indexingCtx, contexts) + } + + const docsToUpsert = new Map, DocIndexState>() + + for (let tx of txes) { + if (tx._class === core.class.TxCollectionCUD) { + const txcol = tx as TxCollectionCUD + tx = txcol.tx + } + if (TxProcessor.isExtendsCUD(tx._class)) { + const cud = tx as TxCUD + + if (!isClassIndexable(control.hierarchy, cud.objectClass, contexts)) { + // No need, since no indixable fields or attachments. + continue + } + docsToUpsert.set(cud.objectId as Ref, { + _id: cud.objectId as Ref, + _class: core.class.DocIndexState, + objectClass: cud.objectClass, + needIndex: true, + modifiedBy: tx.modifiedBy, + modifiedOn: tx.modifiedOn, + space: cud.space, + removed: cud._class === core.class.TxRemoveDoc + }) + + const propagate = getFullTextContext(control.hierarchy, cud.objectClass, contexts).propagate + if (propagate !== undefined && propagate.length > 0) { + control.ctx.contextData.needWarmupFulltext = true + await control.ctx.with('clean-childs', { _class: cud.objectClass }, () => { + // We need to propagate all changes to all child's of following classes. + return control.lowLevel.rawUpdate( + DOMAIN_DOC_INDEX_STATE, + { attachedTo: cud.objectClass, objectClass: { $in: propagate } }, + { needIndex: true } + ) + }) + } + } + } + if (docsToUpsert.size > 0) { + control.ctx.contextData.needWarmupFulltext = true + await control.lowLevel.upload(control.ctx, DOMAIN_DOC_INDEX_STATE, Array.from(docsToUpsert.values())) + } + return [] +} + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export default async () => ({ + trigger: { + OnChange + } +}) diff --git a/server-plugins/fulltext-resources/tsconfig.json b/server-plugins/fulltext-resources/tsconfig.json new file mode 100644 index 0000000000..59e4fd4297 --- /dev/null +++ b/server-plugins/fulltext-resources/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/server-plugins/fulltext/.eslintrc.js b/server-plugins/fulltext/.eslintrc.js new file mode 100644 index 0000000000..72235dc283 --- /dev/null +++ b/server-plugins/fulltext/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/server-plugins/fulltext/.npmignore b/server-plugins/fulltext/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/server-plugins/fulltext/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/server-plugins/fulltext/config/rig.json b/server-plugins/fulltext/config/rig.json new file mode 100644 index 0000000000..0110930f55 --- /dev/null +++ b/server-plugins/fulltext/config/rig.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig" +} diff --git a/server-plugins/fulltext/jest.config.js b/server-plugins/fulltext/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/server-plugins/fulltext/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/server-plugins/fulltext/package.json b/server-plugins/fulltext/package.json new file mode 100644 index 0000000000..55059569a3 --- /dev/null +++ b/server-plugins/fulltext/package.json @@ -0,0 +1,44 @@ +{ + "name": "@hcengineering/server-fulltext", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "files": [ + "lib/**/*", + "types/**/*", + "tsconfig.json" + ], + "author": "Anticrm Platform Contributors", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "format": "format src", + "test": "jest --passWithNoTests --silent", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "^0.6.0", + "@types/node": "~20.11.16", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "typescript": "^5.3.3", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5" + }, + "dependencies": { + "@hcengineering/platform": "^0.6.11", + "@hcengineering/server-core": "^0.6.1" + } +} diff --git a/server-plugins/fulltext/src/index.ts b/server-plugins/fulltext/src/index.ts new file mode 100644 index 0000000000..b42b59341b --- /dev/null +++ b/server-plugins/fulltext/src/index.ts @@ -0,0 +1,33 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// Copyright © 2021, 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import type { Plugin, Resource } from '@hcengineering/platform' +import { plugin } from '@hcengineering/platform' +import type { TriggerFunc } from '@hcengineering/server-core' + +/** + * @public + */ +export const serverFulltextId = 'server-fulltext' as Plugin + +/** + * @public + */ +export default plugin(serverFulltextId, { + trigger: { + OnChange: '' as Resource + } +}) diff --git a/server-plugins/fulltext/tsconfig.json b/server-plugins/fulltext/tsconfig.json new file mode 100644 index 0000000000..59e4fd4297 --- /dev/null +++ b/server-plugins/fulltext/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/server-plugins/notification-resources/src/index.ts b/server-plugins/notification-resources/src/index.ts index 42b48c936e..20b78adcbd 100644 --- a/server-plugins/notification-resources/src/index.ts +++ b/server-plugins/notification-resources/src/index.ts @@ -38,6 +38,7 @@ import core, { generateId, MeasureContext, MixinUpdate, + RateLimiter, Ref, RefTo, SortingOrder, @@ -602,9 +603,12 @@ export async function createPushNotification ( } webpush.setVapidDetails(subject, publicKey, privateKey) + const limiter = new RateLimiter(5) for (const subscription of userSubscriptions) { - void sendPushToSubscription(control, target, subscription, data) + await limiter.exec(async () => { + await sendPushToSubscription(control, target, subscription, data) + }) } } @@ -621,7 +625,7 @@ async function sendPushToSubscription ( } catch (err) { control.ctx.info('Cannot send push notification to', { user: targetUser, err }) if (err instanceof WebPushError) { - if (errorMessages.some((p) => (err as WebPushError).body.includes(p))) { + if (errorMessages.some((p) => JSON.stringify((err as WebPushError).body).includes(p))) { const tx = control.txFactory.createTxRemoveDoc(subscription._class, subscription.space, subscription._id) await control.apply(control.ctx, [tx]) } diff --git a/server/backup/package.json b/server/backup/package.json index e5339813e6..bdc4bcd992 100644 --- a/server/backup/package.json +++ b/server/backup/package.json @@ -51,7 +51,6 @@ "@hcengineering/server-tool": "^0.6.0", "@hcengineering/server-client": "^0.6.0", "@hcengineering/server-token": "^0.6.11", - "@hcengineering/server-core": "^0.6.1", - "@hcengineering/server-indexer": "^0.6.0" + "@hcengineering/server-core": "^0.6.1" } } diff --git a/server/backup/src/backup.ts b/server/backup/src/backup.ts index 4e67fad498..3c905afb3c 100644 --- a/server/backup/src/backup.ts +++ b/server/backup/src/backup.ts @@ -23,7 +23,6 @@ import core, { Domain, DOMAIN_BLOB, DOMAIN_DOC_INDEX_STATE, - DOMAIN_FULLTEXT_BLOB, DOMAIN_MODEL, DOMAIN_TRANSIENT, DOMAIN_TX, @@ -42,7 +41,6 @@ import core, { } from '@hcengineering/core' import { BlobClient, createClient } from '@hcengineering/server-client' import { type StorageAdapter } from '@hcengineering/server-core' -import { fullTextPushStagePrefix } from '@hcengineering/server-indexer' import { generateToken } from '@hcengineering/server-token' import { connect } from '@hcengineering/server-tool' import { createReadStream, createWriteStream, existsSync, mkdirSync, statSync } from 'node:fs' @@ -617,12 +615,7 @@ function prepareClonedDocuments (docs: Doc[], sourceConnection: CoreClient & Bac // if full text is skipped, we need to clean stages for indexes. if (p._class === core.class.DocIndexState) { - for (const k of Object.keys((p as DocIndexState).stages)) { - if (k.startsWith(fullTextPushStagePrefix)) { - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete (p as DocIndexState).stages[k] - } - } + ;(p as DocIndexState).needIndex = true } if (collectionCud) { @@ -1930,15 +1923,34 @@ export async function restore ( } } } + async function performCleanDocIndexState (docsToRemove: Ref[]): Promise { + ctx.info('cleanup', { toRemove: docsToRemove.length, workspace: workspaceId.name, domain: c }) + while (docsToRemove.length > 0) { + const part = docsToRemove.splice(0, 1000) + try { + const docs = (await connection.loadDocs(DOMAIN_DOC_INDEX_STATE, part)) as DocIndexState[] + docs.forEach((it) => { + it.needIndex = true + it.removed = true + it.modifiedOn = Date.now() + }) + await connection.upload(DOMAIN_DOC_INDEX_STATE, docs) + } catch (err: any) { + ctx.error('failed to clean, will retry', { error: err, workspaceId: workspaceId.name }) + docsToRemove.push(...part) + } + } + } if (c !== DOMAIN_BLOB) { // Clean domain documents if not blob if (docsToRemove.length > 0 && opt.merge !== true) { if (c === DOMAIN_DOC_INDEX_STATE) { + // We need to create a request to clean fulltext for selected docuemnts. // We need o clean a FULLTEXT domain as well - await performCleanOfDomain([...docsToRemove], DOMAIN_FULLTEXT_BLOB) + await performCleanDocIndexState([...docsToRemove]) + } else { + await performCleanOfDomain(docsToRemove, c) } - - await performCleanOfDomain(docsToRemove, c) } } } diff --git a/server/core/src/base.ts b/server/core/src/base.ts index e25c947a45..571764cba4 100644 --- a/server/core/src/base.ts +++ b/server/core/src/base.ts @@ -65,17 +65,27 @@ export abstract class BaseMiddleware implements Middleware { return this.next?.loadModel(ctx, lastModelTx, hash) ?? emptyModelResult } - provideGroupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { + provideGroupBy( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery

+ ): Promise> { if (this.next !== undefined) { - return this.next.groupBy(ctx, domain, field) + return this.next.groupBy(ctx, domain, field, query) } - return Promise.resolve(new Set()) + return Promise.resolve(new Map()) } async close (): Promise {} - groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { - return this.provideGroupBy(ctx, domain, field) + groupBy( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery

+ ): Promise> { + return this.provideGroupBy(ctx, domain, field, query) } searchFulltext (ctx: MeasureContext, query: SearchQuery, options: SearchOptions): Promise { diff --git a/server/core/src/configuration.ts b/server/core/src/configuration.ts index 77803c60e0..71ee014568 100644 --- a/server/core/src/configuration.ts +++ b/server/core/src/configuration.ts @@ -43,7 +43,5 @@ export interface DbConfiguration { domains: Record defaultAdapter: string metrics: MeasureContext - contentAdapters: Record serviceAdapters: Record - defaultContentAdapter: string } diff --git a/server/core/src/content.ts b/server/core/src/content.ts index 2d829cab4b..7df37e0722 100644 --- a/server/core/src/content.ts +++ b/server/core/src/content.ts @@ -21,32 +21,25 @@ import { type ContentTextAdapter } from './types' class ContentAdapter implements ContentTextAdapter { constructor ( private readonly adapters: Map, - private readonly defaultAdapter: ContentTextAdapter, - private readonly context: MeasureContext + private readonly defaultAdapter: ContentTextAdapter ) {} - async content (name: string, type: string, doc: Readable): Promise { + content (ctx: MeasureContext, workspace: WorkspaceId, name: string, type: string, doc: Readable): Promise { const adapter = this.adapters.get(type) ?? this.defaultAdapter - return await adapter.content(name, type, doc) - } - - metrics (): MeasureContext { - return this.context + return adapter.content(ctx, workspace, name, type, doc) } } export async function createContentAdapter ( contentAdapters: Record, - defaultContentAdapter: string, - workspace: WorkspaceId, - context: MeasureContext + defaultContentAdapter: string ): Promise { const adapters = new Map() let defaultAdapter: ContentTextAdapter | undefined for (const key in contentAdapters) { const adapterConf = contentAdapters[key] - const adapter = await adapterConf.factory(adapterConf.url, workspace, context.newChild(key, {})) + const adapter = await adapterConf.factory(adapterConf.url) adapters.set(adapterConf.contentType, adapter) if (key === defaultContentAdapter) { @@ -56,5 +49,5 @@ export async function createContentAdapter ( if (defaultAdapter === undefined) { throw new Error('No default content adapter') } - return new ContentAdapter(adapters, defaultAdapter, context) + return new ContentAdapter(adapters, defaultAdapter) } diff --git a/server/core/src/mem.ts b/server/core/src/mem.ts index 14c33d8ded..91ff22dd4d 100644 --- a/server/core/src/mem.ts +++ b/server/core/src/mem.ts @@ -103,8 +103,13 @@ export class DummyDbAdapter implements DbAdapter { async update (ctx: MeasureContext, domain: Domain, operations: Map, DocumentUpdate>): Promise {} - async groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { - return new Set() + async groupBy( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery

+ ): Promise> { + return new Map() } async rawFindAll(domain: Domain, query: DocumentQuery, options?: FindOptions): Promise { diff --git a/server/core/src/pipeline.ts b/server/core/src/pipeline.ts index 17d4b95af1..5b286b445b 100644 --- a/server/core/src/pipeline.ts +++ b/server/core/src/pipeline.ts @@ -105,8 +105,13 @@ class PipelineImpl implements Pipeline { return this.head !== undefined ? await this.head.loadModel(ctx, lastModelTx, hash) : [] } - async groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { - return this.head !== undefined ? await this.head.groupBy(ctx, domain, field) : new Set() + async groupBy( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery

+ ): Promise> { + return this.head !== undefined ? await this.head.groupBy(ctx, domain, field, query) : new Map() } async searchFulltext (ctx: MeasureContext, query: SearchQuery, options: SearchOptions): Promise { diff --git a/server/core/src/types.ts b/server/core/src/types.ts index bf80c0fbdf..3549ad6ee1 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -14,7 +14,6 @@ // import { - MeasureMetricsContext, type Account, type Branding, type Class, @@ -58,6 +57,11 @@ export interface ServerFindOptions extends FindOptions { skipClass?: boolean skipSpace?: boolean + domainLookup?: { + field: string + domain: Domain + } + // Optional measure context, for server side operations ctx?: MeasureContext } @@ -81,7 +85,12 @@ export interface Middleware { ) => Promise> tx: (ctx: MeasureContext, tx: Tx[]) => Promise - groupBy: (ctx: MeasureContext, domain: Domain, field: string) => Promise> + groupBy: ( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery

+ ) => Promise> searchFulltext: ( ctx: MeasureContext, query: SearchQuery, @@ -141,6 +150,8 @@ export interface DBAdapterManager { initAdapters: (ctx: MeasureContext) => Promise closeContext: (ctx: MeasureContext) => Promise + + domainHelper?: DomainHelper } export interface PipelineContext { @@ -215,6 +226,7 @@ export interface TriggerControl { options?: FindOptions ) => Promise> hierarchy: Hierarchy + lowLevel: LowLevelStorage modelDb: ModelDb removedMap: Map, Doc> @@ -282,14 +294,16 @@ export interface EmbeddingSearchOption { export interface IndexedDoc { id: Ref _class: Ref>[] - space: Ref[] + space: Ref modifiedOn: Timestamp modifiedBy: Ref attachedTo?: Ref attachedToClass?: Ref> searchTitle?: string + searchTitle_fields?: any[] searchShortTitle?: string - searchIcon?: any + searchShortTitle_fields?: any[] + searchIcon_fields?: any[] fulltextSummary?: string [key: string]: any } @@ -306,114 +320,96 @@ export interface SearchStringResult { * @public */ export interface FullTextAdapter { - index: (doc: IndexedDoc) => Promise - update: (id: Ref, update: Record) => Promise - remove: (id: Ref[]) => Promise - updateMany: (docs: IndexedDoc[]) => Promise - + index: (ctx: MeasureContext, workspace: WorkspaceId, doc: IndexedDoc) => Promise + update: (ctx: MeasureContext, workspace: WorkspaceId, id: Ref, update: Record) => Promise + remove: (ctx: MeasureContext, workspace: WorkspaceId, id: Ref[]) => Promise + updateMany: (ctx: MeasureContext, workspace: WorkspaceId, docs: IndexedDoc[]) => Promise + load: (ctx: MeasureContext, workspace: WorkspaceId, docs: Ref[]) => Promise searchString: ( + ctx: MeasureContext, + workspace: WorkspaceId, query: SearchQuery, options: SearchOptions & { scoring?: SearchScoring[] } ) => Promise search: ( + ctx: MeasureContext, + workspace: WorkspaceId, _classes: Ref>[], search: DocumentQuery, size: number | undefined, from?: number ) => Promise - searchEmbedding: ( - _classes: Ref>[], - search: DocumentQuery, - embedding: number[], - options: EmbeddingSearchOption - ) => Promise - close: () => Promise - metrics: () => MeasureContext // If no field is provided, will return existing mapping of all dimms. - initMapping: (field?: { key: string, dims: number }) => Promise> - - load: (docs: Ref[]) => Promise + initMapping: (ctx: MeasureContext, field?: { key: string, dims: number }) => Promise } /** * @public */ export class DummyFullTextAdapter implements FullTextAdapter { - async initMapping (field?: { key: string, dims: number }): Promise> { + async initMapping (ctx: MeasureContext): Promise { + return true + } + + async index (ctx: MeasureContext, workspace: WorkspaceId, doc: IndexedDoc): Promise { return {} } - async index (doc: IndexedDoc): Promise { - return {} - } - - async load (docs: Ref[]): Promise { + async load (ctx: MeasureContext, workspace: WorkspaceId, docs: Ref[]): Promise { return [] } - async update (id: Ref, update: Record): Promise { + async update ( + ctx: MeasureContext, + workspace: WorkspaceId, + id: Ref, + update: Record + ): Promise { return {} } - async updateMany (docs: IndexedDoc[]): Promise { + async updateMany (ctx: MeasureContext, workspace: WorkspaceId, docs: IndexedDoc[]): Promise { return [] } - async searchString (query: SearchQuery, options: SearchOptions): Promise { + async searchString ( + ctx: MeasureContext, + workspace: WorkspaceId, + query: SearchQuery, + options: SearchOptions + ): Promise { return { docs: [] } } - async search (query: any): Promise { + async search (ctx: MeasureContext, workspace: WorkspaceId, query: any): Promise { return [] } - async searchEmbedding ( - _classes: Ref>[], - search: DocumentQuery, - embedding: number[], - options: EmbeddingSearchOption - ): Promise { - return [] - } - - async remove (id: Ref[]): Promise {} + async remove (ctx: MeasureContext, workspace: WorkspaceId, id: Ref[]): Promise {} async close (): Promise {} - - metrics (): MeasureContext { - return new MeasureMetricsContext('', {}, {}) - } } /** * @public */ -export type FullTextAdapterFactory = ( - url: string, - workspace: WorkspaceId, - context: MeasureContext -) => Promise +export type FullTextAdapterFactory = (url: string) => Promise /** * @public */ export interface ContentTextAdapter { - content: (name: string, type: string, doc: Readable) => Promise - metrics: () => MeasureContext + content: (ctx: MeasureContext, workspace: WorkspaceId, name: string, type: string, doc: Readable) => Promise } /** * @public */ -export type ContentTextAdapterFactory = ( - url: string, - workspace: WorkspaceId, - context: MeasureContext -) => Promise +export type ContentTextAdapterFactory = (url: string) => Promise /** * @public @@ -449,27 +445,32 @@ export type ObjectDDParticipantFunc = ( /** * @public */ -export type SearchProps = Record +export type SearchPresenterProvider = ( + doc: Doc, + parent: Doc | undefined, + space: Space | undefined, + hierarchy: Hierarchy, + mode: string +) => string +export type FieldParamKind = 'space' | 'parent' + +export type FieldTemplateParam = + | /* field */ [string] + | [/* kind */ 'func', /* Presenter function */ Resource, /* mode */ string] + | [/* kind */ FieldParamKind, /* field */ string] /** + * A concationation template, if string just put string, if obj, use source and field name. * @public */ -export type SearchPresenterProvider = (hierarchy: Hierarchy, props: SearchProps) => string -/** - * @public - */ -export type SearchPresenterFunc = Record> - -/** - * @public - */ -export type ClassSearchConfigProps = string | Record - -/** - * @public - */ -export type ClassSearchConfigProperty = string | { tmpl?: string, props: ClassSearchConfigProps[] } +export type FieldTemplate = /* text */ (string | /* field inject */ FieldTemplateParam)[] +export interface FieldTemplateComponent { + component?: Resource + fields?: FieldTemplateParam[] // Extra fields + template?: FieldTemplate // Primary value in index + extraFields?: FieldTemplate[] // Additional values in index. +} /** * @public */ @@ -479,24 +480,15 @@ export interface SearchScoring { boost: number } -/** - * @public - */ -export interface ClassSearchConfig { - icon?: Asset - iconConfig?: { component: any, props: ClassSearchConfigProps[] } - title: ClassSearchConfigProperty - shortTitle?: ClassSearchConfigProperty - scoring?: SearchScoring[] -} - /** * @public */ export interface SearchPresenter extends Class { - searchConfig: ClassSearchConfig - getSearchShortTitle?: SearchPresenterFunc - getSearchTitle?: SearchPresenterFunc + searchIcon?: Asset + iconConfig?: FieldTemplateComponent + title: FieldTemplateComponent | FieldTemplate + shortTitle?: FieldTemplateComponent | FieldTemplate + scoring?: SearchScoring[] } export interface ServiceAdapter { @@ -628,7 +620,9 @@ export function disableLogging (): void { export interface Workspace { context: MeasureContext id: string + token: string // Account workspace update token. pipeline: Promise + tickHash: number sessions: Map upgrade: boolean @@ -668,8 +662,7 @@ export interface SessionManager { token: Token, rawToken: string, pipelineFactory: PipelineFactory, - sessionId: string | undefined, - accountsUrl: string + sessionId: string | undefined ) => Promise broadcastAll: (workspace: Workspace, tx: Tx[], targets?: string[]) => void diff --git a/server/elastic/package.json b/server/elastic/package.json index dc357b8031..d44ed06ff6 100644 --- a/server/elastic/package.json +++ b/server/elastic/package.json @@ -42,7 +42,7 @@ "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", "@hcengineering/server-core": "^0.6.1", - "@elastic/elasticsearch": "^7.14.0", + "@elastic/elasticsearch": "^7.17.14", "@hcengineering/analytics": "^0.6.0" }, "repository": "https://github.com/hcengineering/platform", diff --git a/server/elastic/src/__init.ts b/server/elastic/src/__init.ts deleted file mode 100644 index 83a49eb196..0000000000 --- a/server/elastic/src/__init.ts +++ /dev/null @@ -1,60 +0,0 @@ -// -// Copyright © 2020, 2021 Anticrm Platform Contributors. -// Copyright © 2021 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Client } from '@elastic/elasticsearch' - -const url = process.env.ELASTIC_URL -if (url === undefined) { - console.error('please provide elastic url.') - process.exit(1) -} - -const client = new Client({ node: url }) - -client.ingest - .putPipeline({ - id: 'anticrm-pipeline', - body: { - processors: [{ attachment: { field: 'data' } }, { remove: { field: 'data' } }] - } - }) - .then(function () { - console.log('putPipeline Resolved') - }) - .catch(function (error) { - console.log('putPipeline error: ', error) - }) - -// // Create index -// client.create({index: 'pdfs', type: 'pdf', id: 'my-index-id', -// body: {description: 'Test pdf indexing'} -// }) -// .then(function () {console.log("Index created");}) -// .catch(function (error) {console.log(error);}) - -// var body = { -// anticrm_doc: { -// properties:{ -// title : {"type" : "keyword", "index" : "false"}, -// type : {"type" : "keyword", "index" : "false"}, -// "attachment.pdf" : {"type" : "keyword"} -// } -// } -// } - -// client.indices.putMapping({index:"pdfs", type:"pdf", body:body}) -// .then((response) => {addPipeline()}) -// .catch((error) => {console.log("putMapping error: " + error)}) diff --git a/server/elastic/src/__tests__/adapter.test.ts b/server/elastic/src/__tests__/adapter.test.ts index 2c54fc93ea..474c8acb2b 100644 --- a/server/elastic/src/__tests__/adapter.test.ts +++ b/server/elastic/src/__tests__/adapter.test.ts @@ -21,13 +21,10 @@ import { createElasticAdapter } from '../adapter' describe('Elastic Adapter', () => { let adapter: FullTextAdapter - + const ctx = new MeasureMetricsContext('-', {}) + const ws1 = getWorkspaceId('ws1') beforeEach(async () => { - adapter = await createElasticAdapter( - process.env.ELASTIC_URL ?? 'http://localhost:9200/', - getWorkspaceId('ws1'), - new MeasureMetricsContext('-', {}) - ) + adapter = await createElasticAdapter(process.env.ELASTIC_URL ?? 'http://localhost:9200/') }) afterEach(async () => { @@ -44,16 +41,18 @@ describe('Elastic Adapter', () => { _class: ['class1' as Ref>], modifiedBy: 'andrey' as Ref, modifiedOn: 0, - space: ['space1' as Ref], + space: 'space1' as Ref, content0: 'hey there!' } - await adapter.index(doc) - const hits = await adapter.search(['class1' as Ref>], {}, 1) + await adapter.index(ctx, ws1, doc) + const hits = await adapter.search(ctx, ws1, ['class1' as Ref>], {}, 1) console.log(hits) }) it('should find document with raw search', async () => { const result = await adapter.searchString( + ctx, + ws1, { query: 'hey' }, diff --git a/server/elastic/src/adapter.ts b/server/elastic/src/adapter.ts index 7b782abf14..c263f7ac34 100644 --- a/server/elastic/src/adapter.ts +++ b/server/elastic/src/adapter.ts @@ -14,11 +14,11 @@ // limitations under the License. // +import { Analytics } from '@hcengineering/analytics' import { Class, Doc, DocumentQuery, - IndexingConfiguration, MeasureContext, Ref, SearchOptions, @@ -27,19 +27,11 @@ import { TxResult, WorkspaceId } from '@hcengineering/core' -import type { - EmbeddingSearchOption, - FullTextAdapter, - IndexedDoc, - SearchScoring, - SearchStringResult -} from '@hcengineering/server-core' +import type { FullTextAdapter, IndexedDoc, SearchScoring, SearchStringResult } from '@hcengineering/server-core' import serverCore from '@hcengineering/server-core' -import { Analytics } from '@hcengineering/analytics' import { Client, errors as esErr } from '@elastic/elasticsearch' import { getMetadata } from '@hcengineering/platform' -import { Domain } from 'node:domain' const DEFAULT_LIMIT = 200 @@ -52,163 +44,104 @@ function getIndexVersion (): string { } class ElasticAdapter implements FullTextAdapter { - private readonly workspaceString: string - private readonly getFulltextDocId: (doc: Ref) => Ref - private readonly getDocId: (fulltext: Ref) => Ref + private readonly getFulltextDocId: (workspaceId: WorkspaceId, doc: Ref) => Ref + private readonly getDocId: (workspaceId: WorkspaceId, fulltext: Ref) => Ref private readonly indexName: string constructor ( private readonly client: Client, - readonly workspaceId: WorkspaceId, private readonly indexBaseName: string, - readonly indexVersion: string, - private readonly _metrics: MeasureContext + readonly indexVersion: string ) { this.indexName = `${indexBaseName}_${indexVersion}` - this.workspaceString = toWorkspaceString(workspaceId) - this.getFulltextDocId = (doc) => `${doc}@${this.workspaceString}` as Ref - this.getDocId = (fulltext) => fulltext.slice(0, -1 * (this.workspaceString.length + 1)) as Ref + this.getFulltextDocId = (workspaceId, doc) => `${doc}@${toWorkspaceString(workspaceId)}` as Ref + this.getDocId = (workspaceId, fulltext) => + fulltext.slice(0, -1 * (toWorkspaceString(workspaceId).length + 1)) as Ref } - async createIndexes (domain: Domain, config: Pick, 'indexes'>): Promise {} - - async initMapping (field?: { key: string, dims: number }): Promise> { - // const current = await this.client.indices.getMapping({}) - // console.log('Mappings', current) - // const mappings = current.body[this.workspaceString] + async initMapping (ctx: MeasureContext): Promise { const indexName = this.indexName - const result: Record = {} try { - const baseIndexExists = await this.client.indices.exists({ - index: this.indexBaseName - }) - const existingVersions = await this.client.indices.get({ - index: [`${this.indexBaseName}_*`] - }) - const existingOldVersionIndices = Object.keys(existingVersions.body).filter((name) => name !== indexName) - if (baseIndexExists.body) { - existingOldVersionIndices.push(this.indexBaseName) - } - + const existingVersions = await ctx.with('get-indexes', {}, () => + this.client.indices.get({ + index: [`${this.indexBaseName}_*`] + }) + ) + const allIndexes = Object.keys(existingVersions.body) + const existingOldVersionIndices = allIndexes.filter((name) => name !== indexName) if (existingOldVersionIndices.length > 0) { - await this.client.indices.delete({ - index: existingOldVersionIndices - }) + await ctx.with('delete-old-index', {}, () => + this.client.indices.delete({ + index: existingOldVersionIndices + }) + ) } - - const existsOldIndex = await this.client.indices.exists({ - index: this.workspaceString - }) - if (existsOldIndex.body) { - await this.client.indices.delete({ - index: this.workspaceString - }) - } - const existsIndex = await this.client.indices.exists({ - index: indexName - }) - if (!existsIndex.body) { - const createIndex = await this.client.indices.create({ - index: indexName, - body: { - settings: { - analysis: { - filter: { - english_stemmer: { - type: 'stemmer', - language: 'english' - }, - english_possessive_stemmer: { - type: 'stemmer', - language: 'possessive_english' - } - }, - analyzer: { - rebuilt_english: { - type: 'custom', - tokenizer: 'standard', - filter: ['english_possessive_stemmer', 'lowercase', 'english_stemmer'] - } - } - } - } - } - }) - console.log(createIndex) - } - - const mappings = await this.client.indices.getMapping({ - index: indexName - }) - if (field !== undefined) { - console.log('Mapping', mappings.body) - } - const wsMappings = mappings.body[indexName] - - // Collect old values. - for (const [k, v] of Object.entries(wsMappings?.mappings?.properties ?? {})) { - const va = v as any - if (va?.type === 'dense_vector') { - result[k] = va?.dims as number - } - if (k === 'workspaceId') { - if (va?.type !== 'keyword') { - this.metrics().info('Force index-recreate, since wrong index type was used') - await this.client.indices.delete({ - index: indexName - }) - return await this.initMapping(field) - } - } - } - await this.client.indices.putMapping({ - index: indexName, - body: { - properties: { - fulltextSummary: { - type: 'text', - analyzer: 'rebuilt_english' - }, - workspaceId: { - type: 'keyword', - index: true - } - } - } - }) - if (field?.key !== undefined) { - if (!(wsMappings?.mappings?.properties?.[field.key]?.type === 'dense_vector')) { - result[field.key] = field.dims - await this.client.indices.putMapping({ + const existsIndex = allIndexes.find((it) => it === indexName) !== undefined + if (!existsIndex) { + await ctx.with('create-index', { indexName }, () => + this.client.indices.create({ index: indexName, - allow_no_indices: true, body: { - properties: { - [field.key]: { - type: 'dense_vector', - dims: field.dims + settings: { + analysis: { + filter: { + english_stemmer: { + type: 'stemmer', + language: 'english' + }, + english_possessive_stemmer: { + type: 'stemmer', + language: 'possessive_english' + } + }, + analyzer: { + rebuilt_english: { + type: 'custom', + tokenizer: 'standard', + filter: ['english_possessive_stemmer', 'lowercase', 'english_stemmer'] + } + } } } } }) - } + ) } + + await ctx.with('put-mapping', {}, () => + this.client.indices.putMapping({ + index: indexName, + body: { + properties: { + fulltextSummary: { + type: 'text', + analyzer: 'rebuilt_english' + }, + workspaceId: { + type: 'keyword', + index: true + } + } + } + }) + ) } catch (err: any) { - Analytics.handleError(err) - console.error(err) + if (err.name !== 'ConnectionError') { + Analytics.handleError(err) + ctx.error(err) + } + return false } - return result + return true } async close (): Promise { await this.client.close() } - metrics (): MeasureContext { - return this._metrics - } - async searchString ( + ctx: MeasureContext, + workspaceId: WorkspaceId, query: SearchQuery, options: SearchOptions & { scoring?: SearchScoring[] } ): Promise { @@ -234,30 +167,12 @@ class ElasticAdapter implements FullTextAdapter { }, { match: { - workspaceId: { query: this.workspaceString, operator: 'and' } + workspaceId: { query: toWorkspaceString(workspaceId), operator: 'and' } } } ] } }, - functions: [ - { - script_score: { - script: { - source: "Math.max(0, ((doc['modifiedOn'].value / 1000 - 1672531200) / 2592000))" - /* - Give more score for more recent objects. 1672531200 is the start of 2023 - 2592000 is a month. The idea is go give 1 point for each month. For objects - older than Jan 2023 it will give just zero. - Better approach is to use gauss function, need to investigate futher how be - map modifiedOn, need to tell elastic that this is a date. - - But linear function is perfect to conduct an experiment - */ - } - } - } - ], boost_mode: 'sum' } }, @@ -313,13 +228,20 @@ class ElasticAdapter implements FullTextAdapter { } return resp - } catch (err) { - console.error('elastic error', JSON.stringify(err, null, 2)) + } catch (err: any) { + if (err.name === 'ConnectionError') { + ctx.warn('Elastic DB is not available') + return { docs: [] } + } + Analytics.handleError(err) + ctx.error('Elastic error', { error: err }) return { docs: [] } } } async search ( + ctx: MeasureContext, + workspaceId: WorkspaceId, _classes: Ref>[], query: DocumentQuery, size: number | undefined, @@ -339,7 +261,7 @@ class ElasticAdapter implements FullTextAdapter { }, { match: { - workspaceId: { query: this.workspaceString, operator: 'and' } + workspaceId: { query: toWorkspaceString(workspaceId), operator: 'and' } } } ], @@ -383,18 +305,34 @@ class ElasticAdapter implements FullTextAdapter { } try { - const result = await this.client.search({ - index: this.indexName, - body: { - query: request, - size: size ?? 200, - from: from ?? 0 + const result = await ctx.with( + 'search', + {}, + () => + this.client.search({ + index: this.indexName, + body: { + query: request, + size: size ?? 200, + from: from ?? 0 + } + }), + { + _classes, + size, + from, + query: request } - }) + ) const hits = result.body.hits.hits as any[] return hits.map((hit) => ({ ...hit._source, _score: hit._score })) - } catch (err) { - console.error(JSON.stringify(err, null, 2)) + } catch (err: any) { + if (err.name === 'ConnectionError') { + ctx.warn('Elastic DB is not available') + return [] + } + ctx.error('Elastic error', { error: err }) + Analytics.handleError(err) return [] } } @@ -406,88 +344,12 @@ class ElasticAdapter implements FullTextAdapter { } } - async searchEmbedding ( - _classes: Ref>[], - search: DocumentQuery, - embedding: number[], - options: EmbeddingSearchOption - ): Promise { - if (embedding.length === 0) return [] - const request: any = { - bool: { - should: [ - { - script_score: { - query: { - bool: { - filter: { - term: { - [options.field_enable]: true - } - } - } - }, - script: { - source: `cosineSimilarity(params.queryVector, '${options.field}') + 1`, - params: { - queryVector: embedding - } - }, - boost: options.embeddingBoost ?? 10.0 - } - }, - { - simple_query_string: { - query: search.$search, - flags: 'OR|PREFIX|PHRASE', - default_operator: 'and', - boost: options.fulltextBoost ?? 1 - } - } - ], - must: { - match: { - workspaceId: { query: this.workspaceString, operator: 'and' } - } - }, - filter: [ - { - bool: { - must: [{ terms: this.getTerms(_classes, '_class') }] - } - } - ] - } - } - - try { - const result = await this.client.search({ - index: this.indexName, - body: { - query: request, - size: options?.size ?? 200, - from: options?.from ?? 0 - } - }) - const sourceHits = result.body.hits.hits - - const min = options?.minScore ?? 75 - const embBoost = options.embeddingBoost ?? 10.0 - - const hits: any[] = sourceHits.filter((it: any) => it._score - embBoost > min) - return hits.map((hit) => ({ ...hit._source, _score: hit._score - embBoost })) - } catch (err) { - console.error(JSON.stringify(err, null, 2)) - return [] - } - } - - async index (doc: IndexedDoc): Promise { + async index (ctx: MeasureContext, workspaceId: WorkspaceId, doc: IndexedDoc): Promise { const wsDoc = { - workspaceId: this.workspaceString, + workspaceId: toWorkspaceString(workspaceId), ...doc } - const fulltextId = this.getFulltextDocId(doc.id) + const fulltextId = this.getFulltextDocId(workspaceId, doc.id) if (doc.data === undefined) { await this.client.index({ index: this.indexName, @@ -507,10 +369,15 @@ class ElasticAdapter implements FullTextAdapter { return {} } - async update (id: Ref, update: Record): Promise { + async update ( + ctx: MeasureContext, + workspaceId: WorkspaceId, + id: Ref, + update: Record + ): Promise { await this.client.update({ index: this.indexName, - id: this.getFulltextDocId(id), + id: this.getFulltextDocId(workspaceId, id), body: { doc: update } @@ -519,14 +386,17 @@ class ElasticAdapter implements FullTextAdapter { return {} } - async updateMany (docs: IndexedDoc[]): Promise { + async updateMany (ctx: MeasureContext, workspaceId: WorkspaceId, docs: IndexedDoc[]): Promise { const parts = Array.from(docs) while (parts.length > 0) { const part = parts.splice(0, 500) const operations = part.flatMap((doc) => { - const wsDoc = { workspaceId: this.workspaceString, ...doc } - return [{ index: { _index: this.indexName, _id: this.getFulltextDocId(doc.id) } }, { ...wsDoc, type: '_doc' }] + const wsDoc = { workspaceId: toWorkspaceString(workspaceId), ...doc } + return [ + { index: { _index: this.indexName, _id: this.getFulltextDocId(workspaceId, doc.id) } }, + { ...wsDoc, type: '_doc' } + ] }) const response = await this.client.bulk({ refresh: true, body: operations }) @@ -547,10 +417,10 @@ class ElasticAdapter implements FullTextAdapter { return [] } - async remove (docs: Ref[]): Promise { + async remove (ctx: MeasureContext, workspaceId: WorkspaceId, docs: Ref[]): Promise { try { while (docs.length > 0) { - const part = docs.splice(0, 10000) + const part = docs.splice(0, 5000) await this.client.deleteByQuery( { type: '_doc', @@ -561,13 +431,13 @@ class ElasticAdapter implements FullTextAdapter { must: [ { terms: { - _id: part.map(this.getFulltextDocId), + _id: part.map((it) => this.getFulltextDocId(workspaceId, it)), boost: 1.0 } }, { match: { - workspaceId: { query: this.workspaceString, operator: 'and' } + workspaceId: { query: toWorkspaceString(workspaceId), operator: 'and' } } } ] @@ -587,7 +457,7 @@ class ElasticAdapter implements FullTextAdapter { } } - async load (docs: Ref[]): Promise { + async load (ctx: MeasureContext, workspaceId: WorkspaceId, docs: Ref[]): Promise { const resp = await this.client.search({ index: this.indexName, type: '_doc', @@ -597,13 +467,13 @@ class ElasticAdapter implements FullTextAdapter { must: [ { terms: { - _id: docs.map(this.getFulltextDocId), + _id: docs.map((it) => this.getFulltextDocId(workspaceId, it)), boost: 1.0 } }, { match: { - workspaceId: { query: this.workspaceString, operator: 'and' } + workspaceId: { query: toWorkspaceString(workspaceId), operator: 'and' } } } ] @@ -612,23 +482,21 @@ class ElasticAdapter implements FullTextAdapter { size: docs.length } }) - return Array.from(resp.body.hits.hits.map((hit: any) => ({ ...hit._source, id: this.getDocId(hit._id) }))) + return Array.from( + resp.body.hits.hits.map((hit: any) => ({ ...hit._source, id: this.getDocId(workspaceId, hit._id) })) + ) } } /** * @public */ -export async function createElasticAdapter ( - url: string, - workspaceId: WorkspaceId, - metrics: MeasureContext -): Promise { +export async function createElasticAdapter (url: string): Promise { const client = new Client({ node: url }) const indexBaseName = getIndexName() const indexVersion = getIndexVersion() - return new ElasticAdapter(client, workspaceId, indexBaseName, indexVersion, metrics) + return new ElasticAdapter(client, indexBaseName, indexVersion) } diff --git a/server/elastic/src/backup.ts b/server/elastic/src/backup.ts deleted file mode 100644 index 72093cc57a..0000000000 --- a/server/elastic/src/backup.ts +++ /dev/null @@ -1,206 +0,0 @@ -// -// Copyright © 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Client } from '@elastic/elasticsearch' -import { - Class, - Doc, - DocumentQuery, - DocumentUpdate, - Domain, - FindOptions, - FindResult, - Hierarchy, - IndexingConfiguration, - Iterator, - MeasureContext, - Ref, - StorageIterator, - toWorkspaceString, - Tx, - TxResult, - WorkspaceId -} from '@hcengineering/core' -import { getMetadata } from '@hcengineering/platform' -import serverCore, { DbAdapter, DbAdapterHandler, type DomainHelperOperations } from '@hcengineering/server-core' - -function getIndexName (): string { - return getMetadata(serverCore.metadata.ElasticIndexName) ?? 'storage_index' -} - -function getIndexVersion (): string { - return getMetadata(serverCore.metadata.ElasticIndexVersion) ?? 'v1' -} - -class ElasticDataAdapter implements DbAdapter { - private readonly workspaceString: string - private readonly getFulltextDocId: (doc: Ref) => Ref - private readonly getDocId: (fulltext: Ref) => Ref - private readonly indexName: string - - constructor ( - readonly workspaceId: WorkspaceId, - private readonly client: Client, - readonly indexBaseName: string, - readonly indexVersion: string - ) { - this.indexName = `${indexBaseName}_${indexVersion}` - this.workspaceString = toWorkspaceString(workspaceId) - this.getFulltextDocId = (doc) => `${doc}@${this.workspaceString}` as Ref - this.getDocId = (fulltext) => fulltext.slice(0, -1 * (this.workspaceString.length + 1)) as Ref - } - - init?: ((domains?: string[], excludeDomains?: string[]) => Promise) | undefined - on?: ((handler: DbAdapterHandler) => void) | undefined - - async traverse( - domain: Domain, - query: DocumentQuery, - options?: Pick, 'sort' | 'limit' | 'projection'> - ): Promise> { - throw new Error('Method not implemented.') - } - - helper (): DomainHelperOperations { - return { - create: async () => {}, - exists: async () => true, - listDomains: async () => new Set(), - createIndex: async () => {}, - dropIndex: async () => {}, - listIndexes: async () => [], - estimatedCount: async () => 0 - } - } - - async groupBy( - ctx: MeasureContext, - domain: Domain, - field: string, - query?: DocumentQuery - ): Promise> { - return new Set() - } - - async findAll( - ctx: MeasureContext, - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise> { - return Object.assign([], { total: 0 }) - } - - async tx (ctx: MeasureContext, ...tx: Tx[]): Promise { - return [] - } - - async createIndexes (domain: Domain, config: Pick, 'indexes'>): Promise {} - async removeOldIndex (domain: Domain, deletePattern: RegExp[], keepPattern: RegExp[]): Promise {} - - async close (): Promise { - await this.client.close() - } - - find (ctx: MeasureContext, domain: Domain): StorageIterator { - throw new Error('Method not implemented.') - } - - async load (ctx: MeasureContext, domain: Domain, docs: Ref[]): Promise { - throw new Error('Method not implemented.') - } - - async upload (ctx: MeasureContext, domain: Domain, docs: Doc[]): Promise { - throw new Error('Method not implemented.') - } - - async update (ctx: MeasureContext, domain: Domain, operations: Map, DocumentUpdate>): Promise { - throw new Error('Method not implemented.') - } - - async rawFindAll(domain: Domain, query: DocumentQuery, options?: FindOptions): Promise { - throw new Error('Method not implemented.') - } - - async rawUpdate( - domain: Domain, - query: DocumentQuery, - operations: DocumentUpdate - ): Promise { - throw new Error('Method not implemented.') - } - - async rawDeleteMany(domain: Domain, query: DocumentQuery): Promise { - throw new Error('Method not implemented') - } - - async clean (ctx: MeasureContext, domain: Domain, docs: Ref[]): Promise { - const indexExists = await this.client.indices.exists({ - index: this.indexName - }) - if (!indexExists.body) { - // No need to clean, no index exists. - return - } - - while (docs.length > 0) { - const part = docs.splice(0, 10000) - await this.client.deleteByQuery( - { - type: '_doc', - index: this.indexName, - body: { - query: { - bool: { - must: [ - { - terms: { - _id: part.map(this.getFulltextDocId), - boost: 1.0 - } - }, - { - match: { - workspaceId: { query: this.workspaceString, operator: 'and' } - } - } - ] - } - }, - size: part.length - } - }, - undefined - ) - } - } -} - -/** - * @public - */ -export async function createElasticBackupDataAdapter ( - ctx: MeasureContext, - hierarchy: Hierarchy, - url: string, - workspaceId: WorkspaceId -): Promise { - const client = new Client({ - node: url - }) - const indexBaseName = getIndexName() - const indexVersion = getIndexVersion() - return new ElasticDataAdapter(workspaceId, client, indexBaseName, indexVersion) -} diff --git a/server/elastic/src/index.ts b/server/elastic/src/index.ts index 185e4869f6..7da598e632 100644 --- a/server/elastic/src/index.ts +++ b/server/elastic/src/index.ts @@ -15,4 +15,3 @@ // export { createElasticAdapter } from './adapter' -export { createElasticBackupDataAdapter } from './backup' diff --git a/server/front/package.json b/server/front/package.json index 5f1963175d..9a4eed1c04 100644 --- a/server/front/package.json +++ b/server/front/package.json @@ -54,7 +54,6 @@ "express-static-gzip": "^2.1.7", "uuid": "^8.3.2", "cors": "^2.8.5", - "@hcengineering/elastic": "^0.6.0", "@hcengineering/server-core": "^0.6.1", "@hcengineering/storage": "^0.6.0", "@hcengineering/server-storage": "^0.6.0", diff --git a/server/indexer/package.json b/server/indexer/package.json index 3fe5507df3..c1742e0515 100644 --- a/server/indexer/package.json +++ b/server/indexer/package.json @@ -42,6 +42,9 @@ "@hcengineering/text": "^0.6.5", "@hcengineering/analytics": "^0.6.0", "@hcengineering/query": "^0.6.12", + "@hcengineering/contact": "^0.6.24", + "@hcengineering/attachment": "^0.6.14", + "@hcengineering/drive": "^0.6.0", "fast-equals": "^5.0.1", "@hcengineering/storage": "^0.6.0" } diff --git a/server/indexer/src/fulltext.ts b/server/indexer/src/fulltext.ts index 2eef85d939..6befa636e4 100644 --- a/server/indexer/src/fulltext.ts +++ b/server/indexer/src/fulltext.ts @@ -14,306 +14,35 @@ // limitations under the License. // -import { Analytics } from '@hcengineering/analytics' -import core, { - type AttachedDoc, - type Class, - type Collection, - type Doc, - type DocIndexState, - type DocumentQuery, - type FindOptions, - type FindResult, +import { type Hierarchy, type MeasureContext, - type ObjQueryType, - type Ref, type SearchOptions, type SearchQuery, type SearchResult, - type Tx, - type TxCUD, - type TxCollectionCUD, - TxFactory, - TxProcessor, - type TxResult, - type WorkspaceId, - docKey, - isClassIndexable, - isFullTextAttribute, - isIndexedAttribute, - toFindResult + type WorkspaceId } from '@hcengineering/core' -import type { FullTextAdapter, IndexedDoc, SessionFindAll, StorageAdapter, WithFind } from '@hcengineering/server-core' -import { type FullTextIndexPipeline } from './indexer' -import { createStateDoc } from './indexer/utils' +import type { FullTextAdapter } from '@hcengineering/server-core' import { getScoringConfig, mapSearchResultDoc } from './mapper' -/** - * @public - */ -export class FullTextIndex implements WithFind { - txFactory = new TxFactory(core.account.System, true) +export async function searchFulltext ( + ctx: MeasureContext, + workspaceId: WorkspaceId, + hierarchy: Hierarchy, + adapter: FullTextAdapter, + query: SearchQuery, + options: SearchOptions +): Promise { + const resultRaw = (await adapter.searchString(ctx, workspaceId, query, { + ...options, + scoring: getScoringConfig(hierarchy, query.classes ?? []) + })) ?? { docs: [] } - constructor ( - private readonly hierarchy: Hierarchy, - private readonly adapter: FullTextAdapter | undefined, - private readonly storageFindAll: SessionFindAll, - readonly storageAdapter: StorageAdapter | undefined, - readonly workspace: WorkspaceId, - readonly indexer: FullTextIndexPipeline, - private readonly upgrade: boolean - ) { - if (!this.upgrade) { - // Schedule indexing after consistency check - void this.indexer.startIndexing() - } - } - - async close (): Promise { - this.indexer.triggerIndexing() - if (!this.upgrade) { - await this.indexer.cancel() - } else { - await this.indexer.processUpload(this.indexer.metrics) - } - } - - async tx (ctx: MeasureContext, txes: Tx[]): Promise { - const stDocs = new Map, { create?: DocIndexState, updated: boolean, removed: boolean }>() - for (let tx of txes) { - let attachedTo: Ref | undefined - let attachedToClass: Ref> | undefined - if (tx._class === core.class.TxCollectionCUD) { - const txcol = tx as TxCollectionCUD - attachedTo = txcol.objectId as Ref - attachedToClass = txcol.objectClass - tx = txcol.tx - } - if (TxProcessor.isExtendsCUD(tx._class)) { - const cud = tx as TxCUD - - if (!isClassIndexable(this.hierarchy, cud.objectClass, this.indexer.contexts)) { - // No need, since no indixable fields or attachments. - continue - } - - let stDoc: DocIndexState | undefined - if (cud._class === core.class.TxCreateDoc) { - // Add doc for indexing - stDoc = createStateDoc(cud.objectId, cud.objectClass, { - attributes: {}, - stages: {}, - attachedTo, - attachedToClass, - space: tx.objectSpace, - removed: false, - needIndex: true - }) - stDocs.set(cud.objectId as Ref, { create: stDoc, updated: false, removed: false }) - } else { - const old = stDocs.get(cud.objectId as Ref) - if (cud._class === core.class.TxRemoveDoc && old?.create !== undefined) { - // Object created and deleted, skip index - stDocs.delete(cud.objectId as Ref) - continue - } else { - // Create and update - if (old?.removed === true) continue - else { - stDocs.set(cud.objectId as Ref, { - ...old, - create: cud._class !== core.class.TxRemoveDoc ? old?.create : undefined, - updated: cud._class !== core.class.TxRemoveDoc && old?.create === undefined, - removed: cud._class === core.class.TxRemoveDoc - }) - } - } - } - } - } - await ctx.with('queue', {}, async (ctx) => { - await this.indexer.queue(ctx, stDocs) + const result: SearchResult = { + ...resultRaw, + docs: resultRaw.docs.map((raw) => { + return mapSearchResultDoc(hierarchy, raw) }) - return {} - } - - async findAll( - ctx: MeasureContext, - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise> { - const { _id, $search, ...mainQuery } = query - if ($search === undefined) return toFindResult([]) - - const ids: Set> = new Set>() - const baseClass = this.hierarchy.getBaseClass(_class) - let classes = this.hierarchy.getDescendants(baseClass).filter((it) => !this.hierarchy.isMixin(it)) - - const attrs = this.hierarchy.getAllAttributes(_class) - - // We need to filter all non indexed fields from query to make it work properly - const findQuery: DocumentQuery = { - $search: query.$search - } - try { - for (const [k, attr] of attrs) { - if (isFullTextAttribute(attr) || isIndexedAttribute(attr)) { - const vv = (query as any)[k] - if (vv != null) { - if ( - k === '_class' || - k === 'modifiedBy' || - k === 'modifiedOn' || - k === 'space' || - k === 'attachedTo' || - k === 'attachedToClass' - ) { - findQuery[k] = vv - } else { - const docKeyValue = docKey(attr.name, { _class: attr.attributeOf }) - findQuery[docKeyValue] = vv - } - } - } - if (attr.type._class === core.class.Collection) { - // we need attached documents to be in classes - const coll = attr.type as Collection - const dsc = this.hierarchy.getDescendants(coll.of).filter((it) => !this.hierarchy.isMixin(it)) - classes = classes.concat(dsc) - } - } - } catch (err: any) { - Analytics.handleError(err) - } - - classes = classes.filter((it, idx, arr) => arr.indexOf(it) === idx) - - classes = classes.filter((it) => { - if (typeof query._class === 'object') { - if (query._class?.$in !== undefined) { - return query._class.$in.includes(it) - } - if (query._class?.$nin !== undefined) { - return !query._class.$nin.includes(it) - } - } - return true - }) - - const fullTextLimit = Math.min(5000, (options?.limit ?? 200) * 100) - - let { docs, pass } = await this.indexer.search(classes, findQuery, fullTextLimit) - - if (docs.length === 0 && pass) { - docs = (await this.adapter?.search(classes, findQuery, fullTextLimit)) ?? [] - } - const indexedDocMap = new Map, IndexedDoc>() - - for (const doc of docs) { - if ( - doc._class != null && - Array.isArray(doc._class) && - doc._class.some((cl) => this.hierarchy.isDerived(cl, baseClass)) - ) { - ids.add(doc.id) - indexedDocMap.set(doc.id, doc) - } - if (doc._class !== null && !Array.isArray(doc._class) && this.hierarchy.isDerived(doc._class, baseClass)) { - ids.add(doc.id) - indexedDocMap.set(doc.id, doc) - } - - if (doc.attachedTo != null) { - if (doc.attachedToClass != null && this.hierarchy.isDerived(doc.attachedToClass, baseClass)) { - if (this.hierarchy.isDerived(doc.attachedToClass, baseClass)) { - ids.add(doc.attachedTo) - indexedDocMap.set(doc.attachedTo, doc) - } - } else { - ids.add(doc.attachedTo) - indexedDocMap.set(doc.attachedTo, doc) - } - } - } - if (docs.length === 0) { - return toFindResult([], 0) - } - const scoreSearch: number | undefined = (options?.sort as any)?.['#score'] - - const resultIds = Array.from(getResultIds(ids, _id)) - let result = await this.storageFindAll( - ctx, - _class, - { _id: { $in: resultIds }, ...mainQuery }, - { - ...options, - limit: scoreSearch !== undefined ? docs.length : options?.limit - } - ) - - // Just assign scores based on idex - result.forEach((it) => { - const idDoc = indexedDocMap.get(it._id) - const { _score } = idDoc as any - it.$source = { - $score: _score - } - }) - if (scoreSearch !== undefined) { - result.sort((a, b) => scoreSearch * ((a.$source?.$score ?? 0) - (b.$source?.$score ?? 0))) - if (options?.limit !== undefined && options?.limit < result.length) { - result = toFindResult(result.slice(0, options?.limit), result.total) - } - } - return result - } - - async searchFulltext (ctx: MeasureContext, query: SearchQuery, options: SearchOptions): Promise { - const resultRaw = (await this.adapter?.searchString(query, { - ...options, - scoring: getScoringConfig(this.hierarchy, query.classes ?? []) - })) ?? { docs: [] } - - const result: SearchResult = { - ...resultRaw, - docs: resultRaw.docs.map((raw) => { - return mapSearchResultDoc(this.hierarchy, raw) - }) - } - return result - } - - submitting: Promise | undefined - - timeout: any -} - -function getResultIds (ids: Set>, _id: ObjQueryType> | undefined): Set> { - const result = new Set>() - if (_id !== undefined) { - if (typeof _id === 'string') { - if (ids.has(_id)) { - result.add(_id) - } - } else if (_id.$in !== undefined) { - for (const id of _id.$in) { - if (ids.has(id)) { - result.add(id) - } - } - } else if (_id.$nin !== undefined) { - for (const id of _id.$nin) { - ids.delete(id) - } - return ids - } else if (_id.$ne !== undefined) { - ids.delete(_id.$ne) - return ids - } - } else { - return ids } return result } diff --git a/server/indexer/src/index.ts b/server/indexer/src/index.ts index 7fcb9ec2d3..f1ae8ddc0d 100644 --- a/server/indexer/src/index.ts +++ b/server/indexer/src/index.ts @@ -13,34 +13,18 @@ // limitations under the License. // -import type { - ContentTextAdapter, - FullTextAdapter, - FullTextAdapterFactory, - SessionFindAll -} from '@hcengineering/server-core' -import type { StorageAdapter } from '@hcengineering/storage' -import type { FullTextPipelineStage } from './indexer' +import type { ContentTextAdapterConfiguration, FullTextAdapterFactory } from '@hcengineering/server-core' export * from './fulltext' export * from './indexer' export * from './rekoni' export * from './ydoc' -/** - * @public - */ -export type FullTextPipelineStageFactory = ( - adapter: FullTextAdapter, - storageFindAll: SessionFindAll, - storageAdapter: StorageAdapter, - contentAdapter: ContentTextAdapter -) => FullTextPipelineStage[] - export interface FulltextDBConfiguration { - fulltextAdapter?: { + fulltextAdapter: { factory: FullTextAdapterFactory url: string - stages: FullTextPipelineStageFactory } + contentAdapters: Record + defaultContentAdapter: string } diff --git a/server/indexer/src/indexer/content.ts b/server/indexer/src/indexer/content.ts deleted file mode 100644 index f6e7b720c7..0000000000 --- a/server/indexer/src/indexer/content.ts +++ /dev/null @@ -1,182 +0,0 @@ -// -// Copyright © 2022, 2023 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Analytics } from '@hcengineering/analytics' -import core, { - getFullTextIndexableAttributes, - type Blob, - type Class, - type Doc, - type DocIndexState, - type DocumentQuery, - type DocumentUpdate, - type MeasureContext, - type Ref, - type WorkspaceId -} from '@hcengineering/core' -import { - type ContentTextAdapter, - type DbAdapter, - type IndexedDoc, - type StorageAdapter -} from '@hcengineering/server-core' -import { - contentStageId, - fieldStateId, - type DocUpdateHandler, - type FullTextPipeline, - type FullTextPipelineStage -} from './types' -import { docKey, docUpdKey } from './utils' -/** - * @public - */ -export class ContentRetrievalStage implements FullTextPipelineStage { - require = [] - stageId = contentStageId - - extra = ['content', 'base64'] - - enabled = true - - // Clear all except following. - clearExcept: string[] = [fieldStateId, contentStageId] - - updateFields: DocUpdateHandler[] = [] - - textLimit = 100 * 1024 - - constructor ( - readonly storageAdapter: StorageAdapter | undefined, - readonly workspace: WorkspaceId, - readonly metrics: MeasureContext, - private readonly contentAdapter: ContentTextAdapter - ) {} - - async initialize (ctx: MeasureContext, storage: DbAdapter, pipeline: FullTextPipeline): Promise { - // Just do nothing - } - - async search ( - _classes: Ref>[], - search: DocumentQuery, - size?: number, - from?: number - ): Promise<{ docs: IndexedDoc[], pass: boolean }> { - return { docs: [], pass: true } - } - - async collect (toIndex: DocIndexState[], pipeline: FullTextPipeline): Promise { - for (const doc of toIndex) { - if (pipeline.cancelling) { - return - } - await this.updateContent(doc, pipeline) - } - } - - async updateContent (doc: DocIndexState, pipeline: FullTextPipeline): Promise { - const attributes = getFullTextIndexableAttributes(pipeline.hierarchy, doc.objectClass) - // Copy content attributes as well. - const update: DocumentUpdate = {} - - if (pipeline.cancelling) { - return - } - - for (const [, val] of Object.entries(attributes)) { - if (val.type._class === core.class.TypeBlob) { - // We need retrieve value of attached document content. - const ref = doc.attributes[docKey(val.name, { _class: val.attributeOf })] as Ref - if (ref !== undefined && ref !== '') { - try { - const docInfo: Blob | undefined = await this.storageAdapter?.stat(this.metrics, this.workspace, ref) - if (docInfo !== undefined && docInfo.size < 30 * 1024 * 1024) { - // We have blob, we need to decode it to string. - const contentType = (docInfo.contentType ?? '').split(';')[0] - - if (!contentType.includes('image')) { - const digest = docInfo.etag - const digestKey = docKey(val.name, { _class: val.attributeOf, digest: true }) - if (doc.attributes[digestKey] !== digest) { - ;(update as any)[docUpdKey(digestKey)] = digest - - const readable = await this.storageAdapter?.get(this.metrics, this.workspace, ref) - - if (readable !== undefined) { - let textContent = await this.metrics.with('fetch', {}, () => - this.contentAdapter.content(ref, contentType, readable) - ) - readable?.destroy() - - textContent = textContent - .split(/ +|\t+|\f+/) - .filter((it) => it) - .join(' ') - .split(/\n\n+/) - .join('\n') - - // trim to large content - if (textContent.length > this.textLimit) { - textContent = textContent.slice(0, this.textLimit) - } - textContent = Buffer.from(textContent).toString('base64') - ;(update as any)[docUpdKey(val.name, { _class: val.attributeOf, extra: this.extra })] = textContent - - if (doc.attachedTo != null) { - const parentUpdate: DocumentUpdate = {} - - ;(parentUpdate as any)[ - docUpdKey(val.name, { _class: val.attributeOf, docId: doc._id, extra: this.extra }) - ] = textContent - - // We do not need to pull stage, just update elastic with document. - await pipeline.update(doc.attachedTo as Ref, true, parentUpdate) - } - } - } - } - } - } catch (err: any) { - if (((err?.message ?? '') as string).includes('NoSuchKeyError')) { - // Ignore error - } else { - const wasError = (doc as any).error !== undefined - - if (wasError) { - continue - } - Analytics.handleError(err) - await pipeline.update(doc._id, false, { - [docKey('error')]: JSON.stringify({ message: err.message, err }) - }) - // Print error only first time, and update it in doc index - this.metrics.error('failed to get content for', { error: err }) - } - } - } - } - } - - await pipeline.update(doc._id, true, update) - } - - async remove (docs: DocIndexState[], pipeline: FullTextPipeline): Promise { - // will be handled by field processor - for (const doc of docs) { - await pipeline.update(doc._id, true, {}) - } - } -} diff --git a/server/indexer/src/indexer/field.ts b/server/indexer/src/indexer/field.ts deleted file mode 100644 index c2e4914cfc..0000000000 --- a/server/indexer/src/indexer/field.ts +++ /dev/null @@ -1,227 +0,0 @@ -// -// Copyright © 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Analytics } from '@hcengineering/analytics' -import core, { - extractDocKey, - getFullTextContext, - getFullTextIndexableAttributes, - type Class, - type Doc, - type DocIndexState, - type DocumentQuery, - type DocumentUpdate, - type MeasureContext, - type Ref -} from '@hcengineering/core' -import { type DbAdapter, type IndexedDoc, type SessionFindAll } from '@hcengineering/server-core' -import { deepEqual } from 'fast-equals' -import { - contentStageId, - fieldStateId, - type DocUpdateHandler, - type FullTextPipeline, - type FullTextPipelineStage -} from './types' -import { collectPropagate, docKey, docUpdKey, getContent, getCustomAttrKeys, isFullTextAttribute } from './utils' -/** - * @public - */ -export class IndexedFieldStage implements FullTextPipelineStage { - require = [] - stageId = fieldStateId - // Do not clear downloaded content - clearExcept: string[] = [contentStageId] - - clearField: string[] = [] - - updateFields: DocUpdateHandler[] = [] - - enabled = true - - constructor (private readonly dbStorageFindAll: SessionFindAll) {} - - async initialize (ctx: MeasureContext, storage: DbAdapter, pipeline: FullTextPipeline): Promise {} - - async search ( - _classes: Ref>[], - search: DocumentQuery, - size?: number, - from?: number - ): Promise<{ docs: IndexedDoc[], pass: boolean }> { - return { docs: [], pass: true } - } - - async collect (toIndex: DocIndexState[], pipeline: FullTextPipeline, metrics: MeasureContext): Promise { - const byClass = toIndex.reduce>, DocIndexState[]>>((p, c) => { - p[c.objectClass] = [...(p[c.objectClass] ?? []), c] - return p - }, {}) - - const processed = new Set>() - - for (const [v, values] of Object.entries(byClass)) { - // Obtain real documents - const valueIds = new Map(values.map((it) => [it._id, it])) - const objClass = v as Ref> - const kids = Array.from(valueIds.keys()) - const docs = await this.dbStorageFindAll( - metrics, - objClass, - { - _id: kids.length === 1 ? kids[0] : { $in: kids } - }, - { limit: kids.length } - ) - const attributes = getFullTextIndexableAttributes(pipeline.hierarchy, objClass) - - // Child docs. - - let allChildDocs: DocIndexState[] | undefined - - for (const doc of docs) { - if (pipeline.cancelling) { - return - } - processed.add(doc._id as Ref) - const docState = valueIds.get(doc._id as Ref) as DocIndexState - - try { - // Copy content attributes as well. - const content = getContent(pipeline.hierarchy, attributes, doc) - - const docUpdate: DocumentUpdate = {} - - let changes = 0 - docUpdate.mixins = pipeline.hierarchy.findAllMixins(doc as Doc) - // Convert previous child fields to just - for (const [k] of Object.entries(docState.attributes)) { - const { attr, docId, _class } = extractDocKey(k) - if ( - (docId !== undefined && attr !== '') || - (_class !== undefined && !pipeline.hierarchy.isDerived(docState.objectClass, _class)) - ) { - // If it some previous indexed field. - // ;(docUpdate as any)[docUpdKey(k)] = null - ;(docUpdate as any).$unset = { ...((docUpdate as any).$unset ?? {}), [docUpdKey(k)]: '' } - } - } - - const customAttrValues: any = [] - for (const [, v] of Object.entries(content)) { - if (v.attr.isCustom === true && v.value !== '' && v.value !== undefined) { - // No need to put localized text as attributes. We do not use it at all - // Just put all content for custom attribute inside one custom field - customAttrValues.push({ label: v.attr.label, value: v.value }) - continue - } - - // Check for content changes and collect update - const dKey = docKey(v.attr.name, { _class: v.attr.attributeOf }) - const dUKey = docUpdKey(v.attr.name, { _class: v.attr.attributeOf }) - - // Full re-index in case stage value is changed - if (!deepEqual(docState.attributes[dKey], v.value)) { - changes++ - if (typeof v.value !== 'object') { - ;(docUpdate as any)[dUKey] = v.value - } - } - } - - const { customAttrKey, customAttrUKey } = getCustomAttrKeys() - if ( - (docState.attributes[customAttrKey] !== undefined || customAttrValues.length > 0) && - !deepEqual(docState.attributes[customAttrKey], customAttrValues) - ) { - changes++ - ;(docUpdate as any)[customAttrUKey] = customAttrValues - } - - if (docState.attachedTo != null && changes > 0) { - const ctx = getFullTextContext(pipeline.hierarchy, objClass, pipeline.contexts) - if (ctx.parentPropagate ?? true) { - // We need to clear field stage from parent, so it will be re indexed. - await pipeline.update(docState.attachedTo as Ref, false, {}) - } - } - - const propagate: Ref>[] = collectPropagate(pipeline, docState.objectClass) - if (propagate.length > 0) { - // We need to propagate all changes to all child's of following classes. - if (allChildDocs === undefined) { - const ids = docs.map((it) => it._id) - - allChildDocs = await metrics.with( - 'propagate', - {}, - async (ctx) => - await this.dbStorageFindAll( - ctx, - core.class.DocIndexState, - { - attachedTo: ids.length === 1 ? ids[0] : { $in: ids } - }, - { limit: ids.length, skipSpace: true, skipClass: true } - ) - ) - } - const childs = allChildDocs.filter((it) => it.attachedTo === docState._id) - // Marck childs to be indexed on next step - await pipeline.queue(metrics, new Map(childs.map((it) => [it._id, { updated: true, removed: false }]))) - } - - await pipeline.update(docState._id, true, docUpdate) - } catch (err: any) { - Analytics.handleError(err) - continue - } - } - } - if (!pipeline.cancelling) { - for (const d of toIndex) { - if (!processed.has(d._id)) { - await pipeline.markRemove(d) - } - } - } - } - - // Remove should be safe to missing class - async remove (docs: DocIndexState[], pipeline: FullTextPipeline): Promise { - for (const doc of docs) { - if (doc.attachedTo !== undefined) { - const attachedTo = doc.attachedTo as Ref - const parentDocUpdate: DocumentUpdate = {} - - for (const [k] of Object.entries(doc.attributes)) { - const { _class, attr, extra, docId } = extractDocKey(k) - - if (_class !== undefined && docId === undefined) { - const keyAttr = pipeline.hierarchy.findAttribute(_class, attr) - if (keyAttr !== undefined && isFullTextAttribute(keyAttr)) { - ;(parentDocUpdate as any)[docUpdKey(attr, { _class, docId: doc._id, extra })] = null - } - } - } - - if (Object.keys(parentDocUpdate).length > 0) { - await pipeline.update(attachedTo, false, parentDocUpdate) - } - } - await pipeline.update(doc._id, true, {}) - } - } -} diff --git a/server/indexer/src/indexer/fulltext.ts b/server/indexer/src/indexer/fulltext.ts deleted file mode 100644 index b49f8765e9..0000000000 --- a/server/indexer/src/indexer/fulltext.ts +++ /dev/null @@ -1,172 +0,0 @@ -// -// Copyright © 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import core, { - type Class, - type Doc, - type DocumentQuery, - type FindOptions, - type FindResult, - type IndexingUpdateEvent, - type MeasureContext, - type Ref, - type SearchOptions, - type SearchQuery, - type SearchResult, - type Tx, - type TxWorkspaceEvent, - generateId, - WorkspaceEvent -} from '@hcengineering/core' -import { PlatformError, unknownError } from '@hcengineering/platform' -import type { - DbConfiguration, - FullTextAdapter, - Middleware, - MiddlewareCreator, - PipelineContext, - SessionFindAll, - TxMiddlewareResult -} from '@hcengineering/server-core' -import { BaseMiddleware, createContentAdapter } from '@hcengineering/server-core' -import type { FulltextDBConfiguration } from '..' -import { FullTextIndex } from '../fulltext' -import { FullTextIndexPipeline } from '../indexer' - -/** - * @public - */ -export class FullTextMiddleware extends BaseMiddleware implements Middleware { - private fulltext!: FullTextIndex - private fulltextAdapter?: FullTextAdapter - - constructor ( - context: PipelineContext, - next: Middleware | undefined, - readonly conf: DbConfiguration & FulltextDBConfiguration, - readonly upgrade?: boolean - ) { - super(context, next) - } - - static create (conf: DbConfiguration & FulltextDBConfiguration, upgrade: boolean): MiddlewareCreator { - return async (ctx, context, next): Promise => { - const middleware = new FullTextMiddleware(context, next, conf, upgrade) - await middleware.init(ctx) - return middleware - } - } - - async init (ctx: MeasureContext): Promise { - if (this.context.adapterManager == null) { - throw new PlatformError(unknownError('Adapter manager should be specified')) - } - if (this.context.storageAdapter == null) { - throw new PlatformError(unknownError('Storage adapter should be specified')) - } - const fullTextCtx = ctx.newChild('🗒️ fulltext', {}) - if (this.conf.fulltextAdapter !== undefined) { - const fta = this.conf.fulltextAdapter - this.fulltextAdapter = await fullTextCtx.with( - 'init', - {}, - async (ctx) => await fta.factory(fta.url, this.context.workspace, fullTextCtx) - ) - } - - const contentAdapter = await ctx.with('create content adapter', {}, (ctx) => - createContentAdapter( - this.conf.contentAdapters, - this.conf.defaultContentAdapter, - this.context.workspace, - fullTextCtx.newChild('content', {}) - ) - ) - const defaultAdapter = this.context.adapterManager.getDefaultAdapter() - const findAll: SessionFindAll = (ctx, _class, query, options) => { - return this.provideFindAll(ctx, _class, query, options) - } - - // TODO: Extract storage adapter to context - const stages = - this.conf.fulltextAdapter !== undefined && this.fulltextAdapter !== undefined - ? this.conf.fulltextAdapter.stages(this.fulltextAdapter, findAll, this.context.storageAdapter, contentAdapter) - : [] - - const indexer = new FullTextIndexPipeline( - defaultAdapter, - stages, - this.context.hierarchy, - this.context.workspace, - fullTextCtx, - this.context.modelDb, - (ctx: MeasureContext, classes: Ref>[]) => { - ctx.info('broadcast indexing update', { classes, workspace: this.context.workspace }) - const evt: IndexingUpdateEvent = { - _class: classes - } - const tx: TxWorkspaceEvent = { - _class: core.class.TxWorkspaceEvent, - _id: generateId(), - event: WorkspaceEvent.IndexingUpdate, - modifiedBy: core.account.System, - modifiedOn: Date.now(), - objectSpace: core.space.DerivedTx, - space: core.space.DerivedTx, - params: evt - } - // Send tx to pipeline - void this.context.broadcastEvent?.(ctx, [tx]) - } - ) - this.fulltext = new FullTextIndex( - this.context.hierarchy, - this.fulltextAdapter, - findAll, - this.context.storageAdapter, - this.context.workspace, - indexer, - this.upgrade ?? false - ) - } - - findAll( - ctx: MeasureContext, - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise> { - if (query?.$search !== undefined) { - return ctx.with('fulltext-find-all', {}, (ctx) => this.fulltext.findAll(ctx, _class, query, options)) - } - return this.provideFindAll(ctx, _class, query, options) - } - - searchFulltext (ctx: MeasureContext, query: SearchQuery, options: SearchOptions): Promise { - return ctx.with('full-text-search', {}, (ctx) => { - return this.fulltext.searchFulltext(ctx, query, options) - }) - } - - async tx (ctx: MeasureContext, tx: Tx[]): Promise { - await this.fulltext.tx(ctx, tx) - return await this.provideTx(ctx, tx) - } - - async close (): Promise { - await this.fulltext.close() - await this.fulltextAdapter?.close() - } -} diff --git a/server/indexer/src/indexer/fulltextPush.ts b/server/indexer/src/indexer/fulltextPush.ts deleted file mode 100644 index 410eec6ed3..0000000000 --- a/server/indexer/src/indexer/fulltextPush.ts +++ /dev/null @@ -1,463 +0,0 @@ -// -// Copyright © 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Analytics } from '@hcengineering/analytics' -import core, { - type AnyAttribute, - type ArrOf, - type Branding, - type Class, - type Doc, - type DocIndexState, - type DocumentQuery, - type DocumentUpdate, - extractDocKey, - getFullTextContext, - type Hierarchy, - isFullTextAttribute, - type MeasureContext, - RateLimiter, - type Ref, - SortingOrder, - toIdMap, - type WorkspaceId -} from '@hcengineering/core' -import { type DbAdapter, type FullTextAdapter, type IndexedDoc, type SessionFindAll } from '@hcengineering/server-core' -import { jsonToText, markupToJSON } from '@hcengineering/text' -import { updateDocWithPresenter } from '../mapper' -import { - contentStageId, - type DocUpdateHandler, - fieldStateId, - type FullTextPipeline, - type FullTextPipelineStage, - fullTextPushStageId, - summaryStageId -} from './types' -import { collectPropagate, collectPropagateClasses, docKey, isCustomAttr } from './utils' - -/** - * @public - */ -export class FullTextPushStage implements FullTextPipelineStage { - require = [fieldStateId, contentStageId, summaryStageId] - stageId = fullTextPushStageId - - enabled = true - - updateFields: DocUpdateHandler[] = [] - - limit = 10 - - dimmVectors: Record = {} - - field_enabled = '_use' - - constructor ( - private readonly dbStorageFindAll: SessionFindAll, - readonly fulltextAdapter: FullTextAdapter, - readonly workspace: WorkspaceId, - readonly branding: Branding | null - ) {} - - async initialize (ctx: MeasureContext, storage: DbAdapter, pipeline: FullTextPipeline): Promise { - // Just do nothing - try { - const r = await this.fulltextAdapter.initMapping() - for (const [k, v] of Object.entries(r)) { - this.dimmVectors[k] = Array.from(Array(v).keys()).map((it) => 0) - } - } catch (err: any) { - Analytics.handleError(err) - } - } - - async update (doc: DocIndexState, update: DocumentUpdate): Promise {} - - checkIntegrity (indexedDoc: IndexedDoc): void { - for (const [k, dimms] of Object.entries(this.dimmVectors)) { - if (indexedDoc[k] === undefined || indexedDoc[k].length !== dimms.length) { - indexedDoc[k] = dimms - indexedDoc[`${k}${this.field_enabled}`] = false - } - } - } - - async search ( - _classes: Ref>[], - search: DocumentQuery, - size?: number, - from?: number - ): Promise<{ docs: IndexedDoc[], pass: boolean }> { - return { docs: [], pass: true } - } - - allAttrs = new WeakMap, Map>() - - async collect (toIndex: DocIndexState[], pipeline: FullTextPipeline, ctx: MeasureContext): Promise { - const bulk: IndexedDoc[] = [] - - const part = [...toIndex] - - const parentsMap = new Map, DocIndexState>() - - const pushQueue = new RateLimiter(5) - while (part.length > 0) { - const toIndexPart = part.splice(0, 50) - - const childIds = toIndexPart - .filter((it) => { - const fctx = getFullTextContext(pipeline.hierarchy, it.objectClass, pipeline.contexts) - return fctx.childProcessingAllowed ?? true - }) - .map((it) => it._id) - - const allChildDocs = await ctx.with( - 'find-child', - {}, - async (ctx) => - await this.dbStorageFindAll( - ctx, - core.class.DocIndexState, - { - attachedTo: childIds.length === 1 ? childIds[0] : { $in: childIds } - }, - { - skipClass: true, - skipSpace: true - } - ) - ) - - // spaces - const spaceDocs = toIdMap( - await ctx.with( - 'find-spaces', - {}, - async (ctx) => - await this.dbStorageFindAll( - ctx, - core.class.DocIndexState, - { - _id: { - $in: toIndexPart.map( - (doc) => - (doc.attributes[docKey('space', { _class: doc.objectClass })] ?? doc.space) as Ref - ) - } - }, - { - skipClass: true, - skipSpace: true - } - ) - ) - ) - - for (const doc of toIndexPart) { - if (pipeline.cancelling) { - return - } - const elasticDoc = createElasticDoc(doc) - try { - ctx.withSync('updateDoc2Elastic', {}, (ctx) => { - updateDoc2Elastic(this.allAttrs, ctx, doc.attributes, elasticDoc, undefined, undefined, pipeline.hierarchy) - }) - - // Include all child attributes - const childDocs = allChildDocs.filter((it) => it.attachedTo === doc._id) - if (childDocs.length > 0) { - for (const c of childDocs) { - const fctx = getFullTextContext(pipeline.hierarchy, c.objectClass, pipeline.contexts) - if (fctx.parentPropagate ?? true) { - ctx.withSync('updateDoc2Elastic', {}, (ctx) => { - updateDoc2Elastic( - this.allAttrs, - ctx, - c.attributes, - elasticDoc, - c._id, - undefined, - pipeline.hierarchy, - true - ) - }) - } - } - } - let parentDoc: DocIndexState | undefined - if (doc.attachedToClass != null && doc.attachedTo != null) { - const propagate: Ref>[] = collectPropagate(pipeline, doc.attachedToClass) - if (propagate.some((it) => pipeline.hierarchy.isDerived(doc.objectClass, it))) { - // We need to include all parent content into this one. - parentDoc = - parentsMap.get(doc.attachedTo as Ref) ?? - (await ctx.with('find-parent', {}, async (ctx) => - ( - await this.dbStorageFindAll( - ctx, - core.class.DocIndexState, - { - _id: doc.attachedTo as Ref - }, - { - limit: 1, - skipClass: true, - skipSpace: true - } - ) - ).shift() - )) - if (parentDoc !== undefined) { - parentsMap.set(parentDoc._id, parentDoc) - const ppdoc = parentDoc - ctx.withSync('updateDoc2Elastic', {}, (ctx) => { - updateDoc2Elastic( - this.allAttrs, - ctx, - ppdoc.attributes, - elasticDoc, - ppdoc._id, - undefined, - pipeline.hierarchy, - true - ) - }) - - const collectClasses = collectPropagateClasses(pipeline, parentDoc.objectClass) - if (collectClasses.length > 0) { - let last: number = 0 - while (true) { - const collections = await this.dbStorageFindAll( - ctx, - core.class.DocIndexState, - { - attachedTo: parentDoc._id, - objectClass: { $in: collectClasses }, - modifiedOn: { $gt: last } - }, - { - sort: { - modifiedOn: SortingOrder.Ascending - }, - skipClass: true, - skipSpace: true, - limit: 500 - } - ) - if (collections.length === 0) { - break - } - last = collections[collections.length - 1].modifiedOn - for (const c of collections) { - ctx.withSync('updateDoc2Elastic', {}, (ctx) => { - updateDoc2Elastic( - this.allAttrs, - ctx, - c.attributes, - elasticDoc, - c._id, - undefined, - pipeline.hierarchy, - true - ) - }) - } - } - } - } - } - } - const spaceDoc = spaceDocs.get( - (doc.attributes[docKey('space', { _class: doc.objectClass })] ?? doc.space) as Ref - ) - - await updateDocWithPresenter(pipeline.hierarchy, doc, elasticDoc, { parentDoc, spaceDoc }, this.branding) - - this.checkIntegrity(elasticDoc) - bulk.push(elasticDoc) - } catch (err: any) { - Analytics.handleError(err) - const wasError = (doc as any).error !== undefined - - await pipeline.update(doc._id, false, { [docKey('error')]: JSON.stringify({ message: err.message, err }) }) - if (wasError) { - continue - } - // Print error only first time, and update it in doc index - console.error(err) - continue - } - } - // Perform bulk update to elastic - - await pushQueue.exec(async () => { - try { - try { - await ctx.with('push-elastic', {}, async () => { - await this.fulltextAdapter.updateMany(bulk) - }) - } catch (err: any) { - Analytics.handleError(err) - // Try to push one by one - await ctx.with('push-elastic-by-one', {}, async () => { - for (const d of bulk) { - try { - await this.fulltextAdapter.update(d.id, d) - } catch (err2: any) { - Analytics.handleError(err2) - } - } - }) - } - if (!pipeline.cancelling) { - for (const doc of toIndexPart) { - await pipeline.update(doc._id, true, {}) - } - } - } catch (err: any) { - Analytics.handleError(err) - } - }) - } - await pushQueue.waitProcessing() - } - - async remove (docs: DocIndexState[], pipeline: FullTextPipeline): Promise { - await this.fulltextAdapter.remove(docs.map((it) => it._id)) - // will be handled by field processor - for (const doc of docs) { - await pipeline.update(doc._id, true, {}) - } - } -} - -/** - * @public - */ -export function createElasticDoc (upd: DocIndexState): IndexedDoc { - const doc = { - id: upd._id, - _class: [upd.objectClass, ...(upd.mixins ?? [])], - modifiedBy: upd.modifiedBy, - modifiedOn: upd.modifiedOn, - space: [upd.space], - attachedTo: upd.attachedTo, - attachedToClass: upd.attachedToClass - } - return doc -} -function updateDoc2Elastic ( - allAttrs: WeakMap, Map>, - ctx: MeasureContext, - attributes: Record, - doc: IndexedDoc, - docIdOverride?: Ref, - refAttribute?: string, - hierarchy?: Hierarchy, - isChildOrParentDoc?: boolean -): void { - for (const [k, v] of Object.entries(attributes)) { - if (v == null) { - continue - } - let { _class, attr, docId, extra, digest } = extractDocKey(k) - if (attr.length === 0) { - continue - } - - let vv: any = v - if (vv != null && extra.includes('base64')) { - ctx.withSync('buffer-from', {}, () => { - vv = Buffer.from(v, 'base64').toString() - }) - } - try { - if (vv != null) { - const cachedClass = _class ?? doc._class[0] - if (hierarchy?.hasClass(cachedClass) ?? false) { - const cl = hierarchy?.getClass(cachedClass) as Class - let attrs = cl !== undefined ? allAttrs.get(cl) : undefined - - if (attrs === undefined && cachedClass != null) { - attrs = new Map() - if (attrs !== undefined) { - allAttrs.set(cl, attrs) - } - } - const attribute = attrs?.get(attr) ?? hierarchy?.findAttribute(cachedClass, attr) - if (attribute !== undefined && attrs !== undefined) { - attrs.set(attr, attribute) - allAttrs.set(cl, attrs) - } - if (attribute !== undefined) { - if ( - isFullTextAttribute(attribute) || - (isChildOrParentDoc === true && - !( - attribute.type._class === core.class.RefTo || - (attribute.type._class === core.class.ArrOf && - (attribute.type as ArrOf).of._class === core.class.RefTo) - )) - ) { - let vvv = vv - if (attribute.type._class === core.class.TypeMarkup) { - ctx.withSync('markup-to-json-text', {}, () => { - vvv = jsonToText(markupToJSON(vv)) - }) - } - if (!(doc.fulltextSummary ?? '').includes(vvv)) { - doc.fulltextSummary = (doc.fulltextSummary ?? '') + vvv + '\n' - continue - } - } - } - } - } - } catch (err: any) { - Analytics.handleError(err) - } - - docId = docIdOverride ?? docId - if (docId === undefined) { - if (typeof vv !== 'object' || isCustomAttr(k)) { - doc[k] = vv - } - continue - } - const docIdAttr = docKey(attr, { _class, extra: extra.filter((it) => it !== 'base64'), digest }) - if (vv !== null) { - // Since we replace array of values, we could ignore null - doc[docIdAttr] = - doc[docIdAttr] == null - ? [] - : typeof doc[docIdAttr] === 'string' || !Array.isArray(doc[docIdAttr]) - ? [doc[docIdAttr]] - : doc[docIdAttr] - if (vv !== '') { - if (typeof vv !== 'object') { - doc[docIdAttr] = Array.from(new Set([...doc[docIdAttr], vv])) - } - } - } - } - - const spaceKey = docKey('space', { _class: core.class.Doc }) - if (doc[spaceKey] !== undefined) { - const existingSpaces = Array.isArray(doc.space) ? doc.space : [doc.space] - const newSpaces = Array.isArray(doc[spaceKey]) ? doc[spaceKey] : [doc[spaceKey]] - doc.space = [...existingSpaces, ...newSpaces].filter((it, idx, arr) => arr.indexOf(it) === idx) - } -} diff --git a/server/indexer/src/indexer/index.ts b/server/indexer/src/indexer/index.ts index e2ff19d0fc..998ee59f30 100644 --- a/server/indexer/src/indexer/index.ts +++ b/server/indexer/src/indexer/index.ts @@ -13,11 +13,6 @@ // limitations under the License. // -export * from './content' -export * from './field' -export * from './fulltext' -export * from './fulltextPush' export * from './indexer' -export * from './summary' export * from './types' export * from './utils' diff --git a/server/indexer/src/indexer/indexer.ts b/server/indexer/src/indexer/indexer.ts index 9c82cc32e5..63bc00787d 100644 --- a/server/indexer/src/indexer/indexer.ts +++ b/server/indexer/src/indexer/indexer.ts @@ -14,99 +14,132 @@ // import { Analytics } from '@hcengineering/analytics' +import attachmentPlugin, { type Attachment } from '@hcengineering/attachment' +import contactPlugin from '@hcengineering/contact' import core, { + type AnyAttribute, + type AttachedDoc, + type Blob, type Class, + type CollaborativeDoc, DOMAIN_DOC_INDEX_STATE, + DOMAIN_MIGRATION, type Doc, type DocIndexState, type DocumentQuery, type DocumentUpdate, + type Domain, + type FindResult, type FullTextSearchContext, type Hierarchy, + type IdMap, type MeasureContext, + type MigrationState, type ModelDb, type Ref, + type Space, TxFactory, + type WithLookup, type WorkspaceIdWithUrl, - _getOperator, + collaborativeDocParse, + coreId, docKey, + generateId, + getFullTextIndexableAttributes, groupByArray, - setObjectValue, - systemAccountEmail + isClassIndexable, + isFullTextAttribute, + isIndexedAttribute, + systemAccountEmail, + toIdMap, + withContext } from '@hcengineering/core' -import type { DbAdapter, IndexedDoc } from '@hcengineering/server-core' +import drivePlugin, { type FileVersion } from '@hcengineering/drive' +import type { + ContentTextAdapter, + DbAdapter, + FullTextAdapter, + IndexedDoc, + StorageAdapter +} from '@hcengineering/server-core' import { RateLimiter, SessionDataImpl } from '@hcengineering/server-core' -import { type FullTextPipeline, type FullTextPipelineStage } from './types' +import { jsonToText, markupToJSON } from '@hcengineering/text' +import { findSearchPresenter, updateDocWithPresenter } from '../mapper' +import { type FullTextPipeline } from './types' +import { createIndexedDoc, createStateDoc, getContent } from './utils' -export * from './content' -export * from './field' export * from './types' export * from './utils' +const textLimit = 500 * 1024 + // Global Memory management configuration /** * @public */ export const globalIndexer = { - allowParallel: 2, - processingSize: 25 + allowParallel: 10, + processingSize: 100 } const rateLimiter = new RateLimiter(globalIndexer.allowParallel) let indexCounter = 0 + +function extractValues (obj: any): string { + let res = '' + + const o = [obj] + while (o.length > 0) { + const oo = o.shift() + if (typeof oo === 'object') { + o.push(Object.values(oo)) + continue + } + if (Array.isArray(oo)) { + o.push(...oo) + continue + } + res += `${oo} ` + } + return res +} /** * @public */ export class FullTextIndexPipeline implements FullTextPipeline { - pending = new Map, DocumentUpdate>() - toIndex = new Map, DocIndexState>() - extraIndex = new Map, DocIndexState>() - stageChanged = 0 - cancelling: boolean = false - - currentStage: FullTextPipelineStage | undefined - - readyStages: string[] - indexing: Promise | undefined + verify: Promise | undefined indexId = indexCounter++ - updateTriggerTimer: any - updateOps = new Map, DocumentUpdate>() - - uploadOps: DocIndexState[] = [] - contexts: Map>, FullTextSearchContext> - propogage = new Map>, Ref>[]>() - propogageClasses = new Map>, Ref>[]>() + + byDomain = new Map>[]>() constructor ( + readonly fulltextAdapter: FullTextAdapter, private readonly storage: DbAdapter, - private readonly stages: FullTextPipelineStage[], readonly hierarchy: Hierarchy, readonly workspace: WorkspaceIdWithUrl, readonly metrics: MeasureContext, readonly model: ModelDb, - readonly broadcastUpdate: (ctx: MeasureContext, classes: Ref>[]) => void + readonly storageAdapter: StorageAdapter, + readonly contentAdapter: ContentTextAdapter, + readonly broadcastUpdate: (ctx: MeasureContext, classes: Ref>[]) => void, + readonly checkIndexes: () => Promise, + readonly closeContext: (ctx: MeasureContext) => Promise ) { - this.readyStages = stages.map((it) => it.stageId) - this.readyStages.sort() this.contexts = new Map(model.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it])) } async cancel (): Promise { this.cancelling = true - clearTimeout(this.updateBroadcast) - clearInterval(this.updateTriggerTimer) - // We need to upload all bulk changes. - await this.processUpload(this.metrics) + await this.verify this.triggerIndexing() await this.indexing - await this.flush(true) this.metrics.warn('Cancel indexing', { workspace: this.workspace.name, indexId: this.indexId }) } @@ -121,274 +154,156 @@ export class FullTextIndexPipeline implements FullTextPipeline { ) } - async search ( - _classes: Ref>[], - search: DocumentQuery, - size: number | undefined, - from?: number - ): Promise<{ docs: IndexedDoc[], pass: boolean }> { - const result: IndexedDoc[] = [] - for (const st of this.stages) { - await st.initialize(this.metrics, this.storage, this) - const docs = await st.search(_classes, search, size, from) - result.push(...docs.docs) - if (!docs.pass) { - return { docs: result, pass: false } - } - } - return { docs: result, pass: true } - } - - async flush (force = false): Promise { - if (this.pending.size > 0 && (this.pending.size >= 50 || force)) { - // Push all pending changes to storage. - try { - await this.storage.update(this.metrics, DOMAIN_DOC_INDEX_STATE, this.pending) - } catch (err: any) { - Analytics.handleError(err) - // Go one by one. - for (const o of this.pending) { - await this.storage.update(this.metrics, DOMAIN_DOC_INDEX_STATE, new Map([o])) - } - } - this.pending.clear() - } - } - updateDoc (doc: DocIndexState, tx: DocumentUpdate, finish: boolean): DocIndexState { - for (const key in tx) { - if (key.startsWith('$')) { - const operator = _getOperator(key) - operator(doc, (tx as any)[key]) - } else { - setObjectValue(key, doc, (tx as any)[key]) - } - } - - const spaceKey = docKey('space', { _class: core.class.Doc }) - if (doc.attributes !== undefined && doc.attributes[spaceKey] !== undefined) { - doc.space = doc.attributes[spaceKey] - } - if (finish) { + doc.needIndex = false doc.modifiedBy = core.account.System doc.modifiedOn = Date.now() } return doc } - async processUpload (ctx: MeasureContext): Promise { - const ops = this.updateOps - this.updateOps = new Map() - const toUpload = this.uploadOps - this.uploadOps = [] - if (toUpload.length > 0) { - await ctx.with('upload', {}, async () => { - await this.storage.upload(this.metrics, DOMAIN_DOC_INDEX_STATE, toUpload) - }) - } - if (ops.size > 0) { - await ctx.with('update', {}, async () => { - await this.storage.update(this.metrics, DOMAIN_DOC_INDEX_STATE, ops) - }) - } - if (toUpload.length > 0 || ops.size > 0) { - this.triggerIndexing() - } - } - - async queue ( - ctx: MeasureContext, - updates: Map, { create?: DocIndexState, updated: boolean, removed: boolean }> - ): Promise { - const entries = Array.from(updates.entries()) - const uploads = entries.filter((it) => it[1].create !== undefined).map((it) => it[1].create) as DocIndexState[] - if (uploads.length > 0) { - this.uploadOps.push(...uploads) - } - - const onlyUpdates = entries.filter((it) => it[1].create === undefined) - - if (onlyUpdates.length > 0) { - for (const u of onlyUpdates) { - const upd: DocumentUpdate = { removed: u[1].removed } - - // We need to clear only first state, to prevent multiple index operations to happen. - ;(upd as any)['stages.' + this.stages[0].stageId] = false - upd.needIndex = true - this.updateOps.set(u[0], upd) - } - } - } - - add (doc: DocIndexState): void { - this.extraIndex.set(doc._id, doc) - } - - // Update are commulative - async update ( - docId: Ref, - mark: boolean, - update: DocumentUpdate, - flush?: boolean - ): Promise { - let udoc = this.toIndex.get(docId) - if (udoc !== undefined) { - await this.stageUpdate(udoc, update) - - udoc = this.updateDoc(udoc, update, mark) - this.toIndex.set(docId, udoc) - } - - if (udoc === undefined) { - udoc = this.extraIndex.get(docId) - if (udoc !== undefined) { - await this.stageUpdate(udoc, update) - udoc = this.updateDoc(udoc, update, mark) - this.extraIndex.set(docId, udoc) - } - } - - if (udoc === undefined) { - // Some updated, document, let's load it. - udoc = (await this.storage.load(this.metrics, DOMAIN_DOC_INDEX_STATE, [docId])).shift() as DocIndexState - } - - if (udoc !== undefined && this.currentStage !== undefined) { - const stageId = this.currentStage.stageId - // Update current stage, value - update.stages = this.filterCurrentStages(udoc) - update.stages[stageId] = mark - - if (this.currentStage.clearExcept !== undefined) { - for (const [k] of Object.entries(update.stages)) { - if (k !== this.currentStage.stageId && !this.currentStage.clearExcept.includes(k)) { - update.stages[k] = false - } - } - } - - // Filter unsupported stages - udoc.stages = update.stages - - const stg = Object.values(udoc.stages) - if (!stg.includes(false) && stg.length === this.stages.length) { - // Check if all marks are true, we need to clear needIndex. - udoc.needIndex = false - } - - if (Object.keys(update).length > 0) { - this.currentStages[stageId] = (this.currentStages[stageId] ?? 0) + 1 - this.stageChanged++ - } - } - - const current = this.pending.get(docId) - if (current === undefined) { - this.pending.set(docId, update) - } else { - this.pending.set(docId, { ...current, ...update }) - } - - await this.flush(flush ?? false) - } - - // Update are commulative - async updateNeedIndex (docId: Ref, value: boolean, flush?: boolean): Promise { - const update = { needIndex: value } - let udoc = this.toIndex.get(docId) - if (udoc !== undefined) { - await this.stageUpdate(udoc, update) - - udoc = this.updateDoc(udoc, update, true) - this.toIndex.set(docId, udoc) - } - - if (udoc === undefined) { - udoc = this.extraIndex.get(docId) - if (udoc !== undefined) { - await this.stageUpdate(udoc, update) - udoc = this.updateDoc(udoc, update, true) - this.extraIndex.set(docId, udoc) - } - } - - if (udoc === undefined) { - // Some updated, document, let's load it. - udoc = (await this.storage.load(this.metrics, DOMAIN_DOC_INDEX_STATE, [docId])).shift() as DocIndexState - } - - const current = this.pending.get(docId) - if (current === undefined) { - this.pending.set(docId, update) - } else { - this.pending.set(docId, { ...current, ...update }) - } - - await this.flush(flush ?? false) - } - triggerCounts = 0 triggerIndexing = (): void => {} - currentStages: Record = {} - private filterCurrentStages (udoc: DocIndexState): Record { - const result: Record = {} - for (const [k, v] of Object.entries(udoc.stages ?? {})) { - if (this.currentStages[k] !== undefined) { - result[k] = v - } + async startIndexing (indexing: () => void): Promise { + this.verify = this.verifyWorkspace(this.metrics, indexing) + void this.verify.then(() => { + this.indexing = this.doIndexing(indexing) + }) + } + + @withContext('verify-workspace') + async verifyWorkspace (ctx: MeasureContext, indexing: () => void): Promise { + const fullReindex = 'full-text-indexer-v2' + const docStructure = 'full-text-structure-v2' + const indexes = 'verify-indexes-v2' + // We need to apply migrations if required. + const migrations = await this.storage.findAll(ctx, core.class.MigrationState, { + plugin: coreId + }) + + if (migrations.find((it) => it.state === indexes) === undefined) { + ctx.warn('Rebuild DB index', { workspace: this.workspace.name }) + // Clean all existing docs, they will be re-created on verify stage + await this.checkIndexes() + + await this.addMigration(ctx, indexes) } - return result - } - private async stageUpdate (udoc: DocIndexState, update: DocumentUpdate): Promise { - for (const u of this.currentStage?.updateFields ?? []) { - await u(udoc, update) + if (migrations.find((it) => it.state === fullReindex) === undefined) { + ctx.warn('rebuilding index to v2', { workspace: this.workspace.name }) + // Clean all existing docs, they will be re-created on verify stage + await this.storage.rawUpdate( + DOMAIN_DOC_INDEX_STATE, + {}, + { needIndex: true, attributes: null, stages: null } + ) + + await this.addMigration(ctx, fullReindex) } - } - async startIndexing (): Promise { - this.indexing = this.doIndexing() + // Verify class integrity if required + const allClasses = this.hierarchy.getDescendants(core.class.Doc) - clearTimeout(this.updateTriggerTimer) - this.updateTriggerTimer = setInterval(() => { - void this.processUpload(this.metrics) - }, 250) - } + const contexts = new Map(this.model.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it])) - async initializeStages (): Promise { - for (const st of this.stages) { - if (this.cancelling) { - return + const allIndexed = allClasses.filter((it) => isClassIndexable(this.hierarchy, it, contexts)) + + const domainPriorities: Record = { + space: 1500, + contact: 1000, + task: 500, + recruit: 400, + document: 300, + attachment: -100 + } + + this.byDomain = groupByArray(allIndexed, (it) => this.hierarchy.getDomain(it)) + this.byDomain = new Map( + Array.from(this.byDomain.entries()).sort((a, b) => { + const ap = domainPriorities[a[0]] ?? 0 + const bp = domainPriorities[b[0]] ?? 0 + return bp - ap + }) + ) + + if (migrations.find((it) => it.state === docStructure) === undefined) { + ctx.warn('verify document structure', { version: docStructure, workspace: this.workspace.name }) + + for (const [domain, classes] of this.byDomain.entries()) { + await ctx.with('verify-domain', { domain }, async () => { + // Iterate over all domain documents and add appropriate entries + const iterator = await this.storage.traverse( + domain, + { _class: { $in: classes } }, + { projection: { _class: 1, _id: 1 } } + ) + try { + while (true) { + indexing() + const docs = await iterator.next(1000) + if (docs == null || docs.length === 0) { + break + } + const states = toIdMap( + await this.storage.rawFindAll(DOMAIN_DOC_INDEX_STATE, { _id: { $in: docs.map((it) => it._id) } }) + ) + // Find missing documents + const missingDocs = docs + .filter((it) => !states.has(it._id)) + .map((it) => createStateDoc(it._id, it._class, { needIndex: true, removed: false })) + + if (missingDocs.length > 0) { + await this.storage.upload(ctx, DOMAIN_DOC_INDEX_STATE, missingDocs) + } + } + } catch (err: any) { + ctx.error('failed to restore index state', { err }) + } finally { + await iterator.close() + } + }) } - await st.initialize(this.metrics, this.storage, this) + // Mark missing classes as deleted ones. + await this.storage.rawUpdate( + DOMAIN_DOC_INDEX_STATE, + { objectClass: { $nin: allIndexed } }, + { + removed: true, + needIndex: true + } + ) + await this.addMigration(ctx, docStructure) } } broadcastClasses = new Set>>() - updateBroadcast: any = undefined + broadcasts: number = 0 - async doIndexing (): Promise { - // Check model is upgraded to support indexer. - - await this.metrics.with('init-states', {}, async () => { - await this.initStates() - }) + private async addMigration (ctx: MeasureContext, state: string): Promise { + const mstate: MigrationState = { + _class: core.class.MigrationState, + _id: generateId(), + modifiedBy: core.account.System, + modifiedOn: Date.now(), + space: core.space.Configuration, + plugin: coreId, + state + } + await this.storage.upload(ctx, DOMAIN_MIGRATION, [mstate]) + } + async doIndexing (indexing: () => void): Promise { while (!this.cancelling) { // Clear triggers this.triggerCounts = 0 - this.stageChanged = 0 - await this.metrics.with('initialize-stages', { workspace: this.workspace.name }, async () => { - await this.initializeStages() - }) - const _classes = await this.metrics.with( + const { classUpdate: _classes, processed } = await this.metrics.with( 'processIndex', { workspace: this.workspace.name }, - async (ctx) => await this.processIndex(ctx) + async (ctx) => await this.processIndex(ctx, indexing) ) // Also update doc index state queries. @@ -398,32 +313,29 @@ export class FullTextIndexPipeline implements FullTextPipeline { this.metrics.info('No wait, trigger counts', { triggerCount: this.triggerCounts }) } - if (this.toIndex.size === 0 && this.stageChanged === 0 && this.triggerCounts === 0) { - if (this.toIndex.size === 0) { - this.metrics.warn('Indexing complete', { indexId: this.indexId, workspace: this.workspace.name }) + if (this.triggerCounts === 0) { + if (processed > 0) { + this.metrics.warn('Indexing complete', { indexId: this.indexId, workspace: this.workspace.name, processed }) } if (!this.cancelling) { // We need to send index update event - clearTimeout(this.updateBroadcast) - this.updateBroadcast = setTimeout(() => { - this.broadcastClasses.delete(core.class.DocIndexState) - if (this.broadcastClasses.size > 0) { - const toSend = Array.from(this.broadcastClasses.values()) - this.broadcastClasses.clear() - this.broadcastUpdate(this.metrics, toSend) - } - }, 5000) + if (this.broadcasts === 0) { + this.broadcasts++ + setTimeout(() => { + this.broadcasts = 0 + this.broadcastClasses.delete(core.class.DocIndexState) + if (this.broadcastClasses.size > 0) { + const toSend = Array.from(this.broadcastClasses.values()) + this.broadcastClasses.clear() + this.broadcastUpdate(this.metrics, toSend) + } + }, 5000) + } - let notified = false await new Promise((resolve) => { this.triggerIndexing = () => { this.triggerCounts++ - if (!notified) { - notified = true - setTimeout(() => { - resolve(null) - }, 500) // Start indexing only after cooldown - } + resolve(null) } }) } @@ -432,40 +344,141 @@ export class FullTextIndexPipeline implements FullTextPipeline { this.metrics.warn('Exit indexer', { indexId: this.indexId, workspace: this.workspace.name }) } - private async processIndex (ctx: MeasureContext): Promise>[]> { + async indexDocuments ( + ctx: MeasureContext, + docs: { + _class: Ref> + _id: Ref + }[] + ): Promise { + const parts = [...docs] + while (parts.length > 0) { + const part = parts.splice(0, 50) + + const states = toIdMap( + await this.storage.findAll( + ctx, + core.class.DocIndexState, + { _id: { $in: part.map((it) => it._id as Ref) } }, + { projection: { _id: 1 } } + ) + ) + + const toInsert: DocIndexState[] = [] + const toUpdate: Ref[] = [] + for (const p of part) { + if (states.has(p._id as Ref)) { + toUpdate.push(p._id as Ref) + } else { + // Space will be updated on indexing + toInsert.push( + createStateDoc(p._id, p._class, { + needIndex: true, + removed: false + }) + ) + } + } + if (toInsert.length > 0) { + await this.storage.upload(ctx, DOMAIN_DOC_INDEX_STATE, toInsert) + } + if (toUpdate.length > 0) { + await this.storage.rawUpdate( + DOMAIN_DOC_INDEX_STATE, + { _id: { $in: toUpdate } }, + { needIndex: true } + ) + } + } + this.triggerIndexing() + } + + private async processIndex ( + ctx: MeasureContext, + indexing: () => void + ): Promise<{ classUpdate: Ref>[], processed: number }> { const _classUpdate = new Set>>() + let processed = 0 await rateLimiter.exec(async () => { + let st = Date.now() + + ctx.id = generateId() + let groupBy = await this.storage.groupBy(ctx, DOMAIN_DOC_INDEX_STATE, 'objectClass', { needIndex: true }) + const total = Array.from(groupBy.values()).reduce((a, b) => a + b, 0) while (true) { try { if (this.cancelling) { return Array.from(_classUpdate.values()) } - await ctx.with('flush', {}, async () => { - await this.flush(true) - }) + const q: DocumentQuery = { + needIndex: true + } + let domainLookup: Domain | undefined + // set a class for more priority domains if they have documents + for (const [domain, classes] of this.byDomain.entries()) { + // Calc if we have classes pending for this domain. + const pending = classes.filter((it) => (groupBy.get(it) ?? 0) > 0) + if (pending.length > 0) { + // We have some classes pending indexing in this domain. + q.objectClass = { $in: pending } + domainLookup = domain + break + } + } + if (domainLookup === undefined) { + // Nothing to index, we should remove pending needIndex requests, since they are not required anymore + if (Array.from(groupBy.values()).reduce((a, b) => a + b, 0) > 0) { + await this.storage.rawDeleteMany(DOMAIN_DOC_INDEX_STATE, { + needIndex: true, + objectClass: { $in: Array.from(groupBy.keys()) as Ref>[] } + }) + } + break + } - let result: DocIndexState[] | undefined = await ctx.with('get-indexable', {}, async () => { - return await this.storage.findAll( - ctx, - core.class.DocIndexState, - { - needIndex: true - }, - { + let result: FindResult | WithLookup[] = await ctx.with( + 'get-indexable', + {}, + async (ctx) => { + return await this.storage.findAll(ctx, core.class.DocIndexState, q, { limit: globalIndexer.processingSize, skipClass: true, - skipSpace: true - } - ) - }) - if (result === undefined) { + skipSpace: true, + domainLookup: { + field: '_id', + domain: domainLookup + } + }) + } + ) + if (result.length === 0) { + if (q.objectClass !== undefined) { + // If we searched with objectClass, we need to recalculate groupBy + groupBy = await this.storage.groupBy(ctx, DOMAIN_DOC_INDEX_STATE, 'objectClass', { needIndex: true }) + continue + } // No more results break } - await this.processRemove(result) + indexing() // Update no sleep + + for (const r of result) { + if (typeof r._id === 'object') { + r._id = (r._id as any).toString() + } + const p = groupBy.get(r.objectClass) + if (p !== undefined) { + if (p - 1 === 0) { + groupBy.delete(r.objectClass) + } else { + groupBy.set(r.objectClass, p - 1) + } + } + } + + const toRemove: DocIndexState[] = await this.processRemove(ctx, result) result = result.filter((it) => !it.removed) - const toRemove: DocIndexState[] = [] // Check and remove missing class documents. result = result.filter((doc) => { const _class = this.model.findObject(doc.objectClass) @@ -490,60 +503,260 @@ export class FullTextIndexPipeline implements FullTextPipeline { } } - if (result.length > 0) { + await this.processDocuments(result, ctx, _classUpdate) + processed += result.length + if (Date.now() - st > 5000) { + st = Date.now() this.metrics.info('Full text: Indexing', { indexId: this.indexId, workspace: this.workspace.name, - ...this.currentStages + domain: domainLookup, + processed, + total }) - } else { - // Nothing to index, check on next cycle. - break - } - const retry: DocIndexState[] = [] - - await this.processStages(result, ctx, _classUpdate) - - // Force clear needIndex, it will be re trigger if some propogate will happen next. - if (!this.cancelling) { - for (const u of result) { - const stg = Object.values(u.stages) - if (!stg.includes(false) && stg.length === this.stages.length) { - // Check if all marks are true, we need to clear needIndex. - u.needIndex = false - await this.updateNeedIndex(u._id, false) - } else { - // Mark as retry on - retry.push(u) - } - } - } - if (retry.length > 0) { - await this.processStages(retry, ctx, _classUpdate) - if (!this.cancelling) { - for (const u of retry) { - // Since retry is happen, it shoudl be marked already. - u.needIndex = false - await this.updateNeedIndex(u._id, false) - } - } } } catch (err: any) { Analytics.handleError(err) this.metrics.error('error during index', { error: err }) } } + await this.closeContext(ctx) }) - return Array.from(_classUpdate.values()) + return { classUpdate: Array.from(_classUpdate.values()), processed } } - private async processStages ( - result: DocIndexState[], + private async findParents (ctx: MeasureContext, docs: AttachedDoc[]): Promise> { + const result: Doc[] = [] + const groups = groupByArray(docs, (doc) => doc.attachedToClass) + for (const [_class, _docs] of groups) { + const pids = Array.from(new Set(_docs.map((it) => it.attachedTo))) + try { + const gdocs = await ctx.with('find-spaces', {}, (ctx) => + this.storage.findAll( + ctx, + _class, + { + _id: { + $in: pids + } + }, + { + skipSpace: true + } + ) + ) + result.push(...gdocs) + } catch (err: any) { + ctx.error('failed to find parents for', { + _class, + pids, + docCl: Array.from(new Set(_docs.map((it) => it._class))) + }) + continue + } + } + return toIdMap(result) + } + + private async processDocuments ( + result: WithLookup[], ctx: MeasureContext, _classUpdate: Set>> ): Promise { - this.toIndex = new Map(result.map((it) => [it._id, it])) - const contextData = new SessionDataImpl( + const contextData = this.createContextData() + ctx.contextData = contextData + let indexedDocs: IndexedDoc[] = [] + // Find documents matching query + + const byClass = groupByArray, Ref>>(result, (it) => it.objectClass) + + const docUpdates = new Map, DocumentUpdate>() + + const pushQueue = new RateLimiter(5) + + const pushToIndex = async (): Promise => { + const docs = [...indexedDocs] + indexedDocs = [] + if (docs.length === 0) { + return + } + await pushQueue.exec(async () => { + try { + try { + await ctx.with('push-elastic', {}, async () => { + await this.fulltextAdapter.updateMany(ctx, this.workspace, docs) + }) + } catch (err: any) { + Analytics.handleError(err) + // Try to push one by one + await ctx.with('push-elastic-by-one', {}, async () => { + for (const d of docs) { + try { + await this.fulltextAdapter.update(ctx, this.workspace, d.id, d) + } catch (err2: any) { + Analytics.handleError(err2) + } + } + }) + } + } catch (err: any) { + Analytics.handleError(err) + } + }) + } + + let spaceDocs: IdMap | undefined + const updateSpaces = async (): Promise => { + spaceDocs = toIdMap( + await ctx.with('find-spaces', {}, (ctx) => + this.storage.findAll( + ctx, + core.class.Space, + { + _id: { + $in: Array.from(new Set(result.map((doc) => doc.space))) + } + }, + { + skipClass: true, + skipSpace: true + } + ) + ) + ) + } + + const rateLimit = new RateLimiter(10) + + for (const [v, values] of byClass.entries()) { + const searchPresenter = findSearchPresenter(this.hierarchy, v) + // Obtain real documents + const valueIds = new Map(values.map((it) => [it._id, it])) + const docs = values.map((it) => it.$lookup?._id as Doc).filter((it) => it !== undefined) + let parentDocs: IdMap | undefined + + // We need to add all to broadcast update + for (const doc of docs) { + if (typeof doc._id === 'object') { + doc._id = (doc._id as any).toString() + } + _classUpdate.add(doc._class) + if (this.cancelling) { + return + } + const docState = valueIds.get(doc._id as Ref) as WithLookup + const indexedDoc = createIndexedDoc(doc, this.hierarchy.findAllMixins(doc), doc.space) + + await rateLimit.exec(async () => { + await ctx.with('process-document', { _class: doc._class }, async (ctx) => { + try { + // Copy content attributes as well. + const docUpdate: DocumentUpdate = { + needIndex: false + } + if (docState.space !== doc.space) { + docUpdate.space = doc.space + } + + // TODO: Check if need to include parent values here as well + + // Collect all indexable values + const attributes = getFullTextIndexableAttributes(this.hierarchy, doc._class) + const content = getContent(this.hierarchy, attributes, doc) + + indexedDoc.fulltextSummary = '' + + for (const [, v] of Object.entries(content)) { + if (v.attr.type._class === core.class.TypeBlob) { + await this.processBlob(ctx, v, doc, indexedDoc) + continue + } + + if (v.attr.type._class === core.class.TypeCollaborativeDoc) { + await this.processCollaborativeDoc(ctx, v, indexedDoc) + continue + } + if ((isFullTextAttribute(v.attr) || v.attr.isCustom === true) && v.value !== undefined) { + if (v.attr.type._class === core.class.TypeMarkup) { + ctx.withSync('markup-to-json-text', {}, () => { + indexedDoc.fulltextSummary += '\n' + jsonToText(markupToJSON(v.value)) + }) + } else { + indexedDoc.fulltextSummary += '\n' + v.value + } + + continue + } + + if (isIndexedAttribute(v.attr)) { + // We need to put indexed attr in place + + // Check for content changes and collect update + const dKey = docKey(v.attr.name, v.attr.attributeOf) + if (dKey !== '_class') { + if (typeof v.value !== 'object') { + indexedDoc[dKey] = v.value + } else { + // We need to extract only values + indexedDoc[dKey] = extractValues(v.value) + } + } + continue + } + } + + // trim to large content + if (indexedDoc.fulltextSummary.length > textLimit) { + indexedDoc.fulltextSummary = indexedDoc.fulltextSummary.slice(0, textLimit) + } + + if (searchPresenter !== undefined) { + await ctx.with('update-search-presenter', { _class: doc._class }, async () => { + if (parentDocs === undefined) { + parentDocs = this.hierarchy.isDerived(v, core.class.AttachedDoc) + ? await this.findParents(ctx, docs as unknown as AttachedDoc[]) + : undefined + } + const parentDoc = parentDocs?.get((doc as AttachedDoc).attachedTo as Ref) + if (spaceDocs === undefined) { + await updateSpaces() + } + const spaceDoc = spaceDocs?.get(doc.space) // docState.$lookup?.space + await updateDocWithPresenter(this.hierarchy, doc, indexedDoc, parentDoc, spaceDoc, searchPresenter) + }) + } + + indexedDoc.id = doc._id + indexedDoc.space = doc.space + indexedDocs.push(indexedDoc) + + if (indexedDocs.length > 25) { + void pushToIndex() + } + + docUpdates.set(docState._id, docUpdate) + } catch (err: any) { + ctx.error('failed to process document', { + id: doc._id, + class: doc._class, + workspace: this.workspace.name + }) + Analytics.handleError(err) + } + }) + }) + } + } + await rateLimit.waitProcessing() + + await pushToIndex() + await pushQueue.waitProcessing() + + await ctx.with('update-index-state', {}, (ctx) => this.storage.update(ctx, DOMAIN_DOC_INDEX_STATE, docUpdates)) + } + + private createContextData (): SessionDataImpl { + return new SessionDataImpl( systemAccountEmail, '', true, @@ -555,87 +768,148 @@ export class FullTextIndexPipeline implements FullTextPipeline { new Map(), this.model ) - ctx.contextData = contextData - for (const st of this.stages) { - this.extraIndex.clear() - this.stageChanged = 0 - // Find documents matching query - const toIndex = this.matchStates(st) + } - if (toIndex.length > 0) { - // Do Indexing - this.currentStage = st + @withContext('process-collaborative-doc') + private async processCollaborativeDoc ( + ctx: MeasureContext, + v: { value: any, attr: AnyAttribute }, + indexedDoc: IndexedDoc + ): Promise { + const collaborativeDoc = v.value as CollaborativeDoc + if (collaborativeDoc !== undefined && collaborativeDoc !== '') { + const { documentId } = collaborativeDocParse(collaborativeDoc) - await ctx.with('collect-' + st.stageId, {}, async (ctx) => { - await st.collect(toIndex, this, ctx) - }) - if (this.cancelling) { - break - } - - toIndex.forEach((it) => _classUpdate.add(it.objectClass)) - } else { - continue + const docInfo: Blob | undefined = await this.storageAdapter?.stat(ctx, this.workspace, documentId) + try { + await this.handleBlob(ctx, docInfo, indexedDoc) + } catch (err: any) { + Analytics.handleError(err) + ctx.error('failed to handle blob', { _id: docInfo?._id, workspace: this.workspace.name }) } } } - private async processRemove (docs: DocIndexState[]): Promise { - let total = 0 - this.toIndex = new Map(docs.map((it) => [it._id, it])) - - this.extraIndex.clear() - - const toIndex = Array.from(this.toIndex.values()).filter((it) => it.removed) - if (toIndex.length === 0) { - return - } - const toRemoveIds = [] - for (const st of this.stages) { - if (toIndex.length > 0) { - // Do Indexing - this.currentStage = st - await st.remove(toIndex, this) - } else { - break + @withContext('process-blob') + private async processBlob ( + ctx: MeasureContext, + v: { value: any, attr: AnyAttribute }, + doc: Doc, + indexedDoc: IndexedDoc + ): Promise { + // We need retrieve value of attached document content. + try { + const ref = v.value as Ref + if (ref === '' || ref.startsWith('http://') || ref.startsWith('https://')) { + return } - } - // If all stages are complete, remove document - const allStageIds = this.stages.map((it) => it.stageId) - for (const doc of toIndex) { - if (allStageIds.every((it) => doc.stages[it])) { - toRemoveIds.push(doc._id) + if (v.attr.name === 'avatar' || v.attr.attributeOf === contactPlugin.class.Contact) { + return } - } + if (v.attr.attributeOf === attachmentPlugin.class.Attachment && v.attr.name === 'file') { + const file = doc as Attachment + if (!isBlobAllowed(file.type)) { + // Skip blob with invalid, or diallowed content type + return + } + } + if (v.attr.attributeOf === drivePlugin.class.FileVersion && v.attr.name === 'file') { + const file = doc as FileVersion + if (!isBlobAllowed(file.type)) { + // Skip blob with invalid, or diallowed content type + return + } + } + const docInfo: Blob | undefined = await this.storageAdapter.stat(ctx, this.workspace, ref) + if (docInfo !== undefined && docInfo.size < 30 * 1024 * 1024) { + // We have blob, we need to decode it to string. + const contentType = (docInfo.contentType ?? '').split(';')[0] - await this.flush(true) - if (toRemoveIds.length > 0) { - await this.storage.clean(this.metrics, DOMAIN_DOC_INDEX_STATE, toRemoveIds) - total += toRemoveIds.length - this.metrics.info('indexer', { - _classes: Array.from(groupByArray(toIndex, (it) => it.objectClass).keys()), - total, - count: toRemoveIds.length + if (contentType.includes('text/') || contentType.includes('application/vnd.github.VERSION.diff')) { + await this.handleTextBlob(ctx, docInfo, indexedDoc) + } else if (isBlobAllowed(contentType)) { + await this.handleBlob(ctx, docInfo, indexedDoc) + } + } + } catch (err: any) { + ctx.warn('faild to process text content', { + id: doc._id, + _class: doc._class, + field: v.attr.name, + err, + workspace: this.workspace.name }) } } - private async initStates (): Promise { - this.currentStages = {} - for (const st of this.stages) { - this.currentStages[st.stageId] = 0 + private async handleBlob (ctx: MeasureContext, docInfo: Blob | undefined, indexedDoc: IndexedDoc): Promise { + if (docInfo !== undefined) { + const contentType = (docInfo.contentType ?? '').split(';')[0] + const readable = await this.storageAdapter?.get(ctx, this.workspace, docInfo._id) + + if (readable !== undefined) { + try { + let textContent = await ctx.with('fetch', {}, () => + this.contentAdapter.content(ctx, this.workspace, docInfo._id, contentType, readable) + ) + textContent = textContent + .split(/ +|\t+|\f+/) + .filter((it) => it) + .join(' ') + .split(/\n\n+/) + .join('\n') + + indexedDoc.fulltextSummary += '\n' + textContent + } finally { + readable?.destroy() + } + } } } - private matchStates (st: FullTextPipelineStage): DocIndexState[] { - const toIndex: DocIndexState[] = [] - const require = [...st.require].filter((it) => this.stages.find((q) => q.stageId === it && q.enabled)) - for (const o of this.toIndex.values()) { - // We need to contain all state values - if (require.every((it) => o.stages?.[it]) && !(o.stages?.[st.stageId] ?? false)) { - toIndex.push(o) - } + private async handleTextBlob ( + ctx: MeasureContext, + docInfo: Blob | undefined, + indexedDoc: IndexedDoc + ): Promise { + if (docInfo !== undefined) { + let textContent = Buffer.concat( + (await this.storageAdapter?.read(ctx, this.workspace, docInfo._id)) as any + ).toString() + + textContent = textContent + .split(/ +|\t+|\f+/) + .filter((it) => it) + .join(' ') + .split(/\n\n+/) + .join('\n') + + indexedDoc.fulltextSummary += '\n' + textContent } - return toIndex + } + + private async processRemove (ctx: MeasureContext, docs: DocIndexState[]): Promise { + try { + const toRemove = docs.filter((it) => it.removed) + if (toRemove.length !== 0) { + await this.fulltextAdapter.remove( + ctx, + this.workspace, + toRemove.map((it) => it._id) + ) + } + return toRemove + } catch (err: any) { + Analytics.handleError(err) + } + return [] } } +function isBlobAllowed (contentType: string): boolean { + return ( + !contentType.includes('image/') && + !contentType.includes('video/') && + !contentType.includes('binary/octet-stream') && + !contentType.includes('application/octet-stream') + ) +} diff --git a/server/indexer/src/indexer/summary.ts b/server/indexer/src/indexer/summary.ts deleted file mode 100644 index 56096ab8b9..0000000000 --- a/server/indexer/src/indexer/summary.ts +++ /dev/null @@ -1,335 +0,0 @@ -// -// Copyright © 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Analytics } from '@hcengineering/analytics' -import core, { - type AnyAttribute, - type Class, - type Doc, - type DocIndexState, - type DocumentQuery, - type DocumentUpdate, - extractDocKey, - getFullTextContext, - type Hierarchy, - isFullTextAttribute, - type MeasureContext, - type Ref -} from '@hcengineering/core' -import { translate } from '@hcengineering/platform' -import { type DbAdapter, type IndexedDoc, type SessionFindAll } from '@hcengineering/server-core' -import { jsonToText, markupToJSON } from '@hcengineering/text' -import { - contentStageId, - type DocUpdateHandler, - fieldStateId, - type FullTextPipeline, - type FullTextPipelineStage, - summaryStageId -} from './types' -import { collectPropagate, collectPropagateClasses, isCustomAttr } from './utils' - -/** - * @public - */ -export class FullSummaryStage implements FullTextPipelineStage { - require = [fieldStateId, contentStageId] - stageId = summaryStageId - - enabled = true - - clearExcept?: string[] = undefined - - updateFields: DocUpdateHandler[] = [] - - // If specified, index only fields with content specified. - matchExtra: string[] = [] // 'content', 'base64'] // '#en' - - fieldFilter: ((attr: AnyAttribute, value: string) => boolean)[] = [] - - // Summary should be not a bigger what 1mb of data. - summaryLimit = 1024 * 1024 - - constructor (private readonly dbStorageFindAll: SessionFindAll) {} - - async initialize (ctx: MeasureContext, storage: DbAdapter, pipeline: FullTextPipeline): Promise {} - - async search ( - _classes: Ref>[], - search: DocumentQuery, - size?: number, - from?: number - ): Promise<{ docs: IndexedDoc[], pass: boolean }> { - return { docs: [], pass: true } - } - - async collect (toIndex: DocIndexState[], pipeline: FullTextPipeline, metrics: MeasureContext): Promise { - const part = [...toIndex] - while (part.length > 0) { - const toIndexPart = part.splice(0, 100) - - const kids = toIndexPart.map((it) => it._id) - const allChildDocs = await metrics.with( - 'find-child', - {}, - async (ctx) => - await this.dbStorageFindAll(ctx, core.class.DocIndexState, { - attachedTo: kids.length === 1 ? kids[0] : { $in: kids } - }) - ) - - for (const doc of toIndexPart) { - if (pipeline.cancelling) { - return - } - - const needIndex = isIndexingRequired(pipeline, doc) - - // No need to index this class, mark embeddings as empty ones. - if (!needIndex) { - await pipeline.update(doc._id, true, {}) - continue - } - - const update: DocumentUpdate = {} - - let embeddingText = await extractIndexedValues(doc, pipeline.hierarchy, { - matchExtra: this.matchExtra, - fieldFilter: this.fieldFilter - }) - - // Include all child attributes - const childDocs = allChildDocs.filter((it) => it.attachedTo === doc._id) - if (childDocs.length > 0) { - for (const c of childDocs) { - const ctx = getFullTextContext(pipeline.hierarchy, c.objectClass, pipeline.contexts) - if (ctx.parentPropagate ?? true) { - if (embeddingText.length > this.summaryLimit) { - break - } - embeddingText += - '\n' + - (await extractIndexedValues(c, pipeline.hierarchy, { - matchExtra: this.matchExtra, - fieldFilter: this.fieldFilter - })) - } - } - } - - if (doc.attachedToClass != null && doc.attachedTo != null) { - const propagate: Ref>[] = collectPropagate(pipeline, doc.attachedToClass) - if (propagate.some((it) => pipeline.hierarchy.isDerived(doc.objectClass, it))) { - // We need to include all parent content into this one. - const [parentDoc] = await this.dbStorageFindAll( - metrics, - core.class.DocIndexState, - { _id: doc.attachedTo as Ref }, - { - limit: 1, - skipSpace: true, - skipClass: true - } - ) - if (parentDoc !== undefined) { - const ctx = collectPropagateClasses(pipeline, parentDoc.objectClass) - if (ctx.length > 0) { - let last = 0 - while (true) { - const collections = await this.dbStorageFindAll( - metrics, - core.class.DocIndexState, - { - attachedTo: parentDoc._id, - objectClass: ctx.length === 1 ? ctx[0] : { $in: ctx }, - modifiedOn: { $gt: last } - }, - { - limit: 250, - skipClass: true, - skipSpace: true - } - ) - if (collections.length === 0) { - break - } - last = collections[collections.length - 1].modifiedOn - for (const c of collections) { - embeddingText += - '\n' + - (await extractIndexedValues(c, pipeline.hierarchy, { - matchExtra: this.matchExtra, - fieldFilter: this.fieldFilter - })) - } - } - } - - if (embeddingText.length > this.summaryLimit) { - break - } - embeddingText += - '\n' + - (await extractIndexedValues(parentDoc, pipeline.hierarchy, { - matchExtra: this.matchExtra, - fieldFilter: this.fieldFilter - })) - } - } - } - - update.fullSummary = embeddingText - - await pipeline.update(doc._id, true, update) - } - } - } - - async remove (docs: DocIndexState[], pipeline: FullTextPipeline): Promise { - // will be handled by field processor - for (const doc of docs) { - await pipeline.update(doc._id, true, {}) - } - } -} - -/** - * @public - */ -export function isIndexingRequired (pipeline: FullTextPipeline, doc: DocIndexState): boolean { - return getFullTextContext(pipeline.hierarchy, doc.objectClass, pipeline.contexts).fullTextSummary ?? false -} - -/** - * @public - */ -export async function extractIndexedValues ( - doc: DocIndexState, - hierarchy: Hierarchy, - opt: { - matchExtra: string[] - fieldFilter: ((attr: AnyAttribute, value: string) => boolean)[] - } -): Promise { - const attributes: Record>, Record> = {} - const childAttributes: Record>, Record> = {} - const currentReplacement: Record = {} - - for (const [k, v] of Object.entries(doc.attributes)) { - if (v == null) { - continue - } - try { - const { _class, attr, extra, docId } = extractDocKey(k) - if (docId !== undefined) { - continue - } - - let sourceContent = `${v as string}`.trim() - if (extra.includes('base64')) { - sourceContent = Buffer.from(sourceContent, 'base64').toString().trim() - } - if (sourceContent.length === 0) { - continue - } - - if (isCustomAttr(attr)) { - const str = v - .map((pair: { label: string, value: string }) => { - return `${pair.label} is ${pair.value}` - }) - .join(' ') - const cl = doc.objectClass - attributes[cl] = { ...attributes[cl], [k]: str } - } - - if (_class === undefined) { - // Skip all helper fields. - continue - } - - if (!opt.matchExtra.every((it) => extra.includes(it))) { - continue - } - // Check if attribute is indexable - const keyAttr: AnyAttribute | undefined = hierarchy.findAttribute(_class, attr) - if (keyAttr === undefined) { - // Skip if there is no attribute. - continue - } - - if (keyAttr.type._class === core.class.TypeMarkup) { - sourceContent = jsonToText(markupToJSON(sourceContent)) - } - - if (!opt.fieldFilter.every((it) => it(keyAttr, sourceContent))) { - // Some of filters not pass value - continue - } - - if (!isFullTextAttribute(keyAttr)) { - continue - } - if (keyAttr.type._class === core.class.TypeBlob && extra.length === 0) { - // Skip attachment id values. - continue - } - - const repl = extra.join('#') - - if ((currentReplacement[attr] ?? '').length <= repl.length) { - const label = await translate(keyAttr.label, {}) - const cl = _class ?? doc.objectClass - - if (docId === undefined) { - attributes[cl] = { ...attributes[cl], [k]: `${label} is ${sourceContent}\n` } - } else { - childAttributes[cl] = { ...childAttributes[cl], [k]: sourceContent } - } - currentReplacement[attr] = repl - } - } catch (err: any) { - Analytics.handleError(err) - } - } - let embeddingText = '' - - for (const [, vv] of Object.entries(attributes)) { - embeddingText += '\n' - for (const [, v] of Object.entries(vv)) { - // Check if attribute is text one. - embeddingText += ' ' + v + '\n' - } - } - - // Extra child attributes - for (const [, vv] of Object.entries(childAttributes)) { - for (const [, v] of Object.entries(vv)) { - // Check if attribute is text one. - embeddingText += ' ' + v + '\n' - } - } - - // Trim empty inner space. - embeddingText = (embeddingText ?? '') - .split(/ +|\t+/) - .filter((it) => it) - .join(' ') - embeddingText = (embeddingText ?? '') - .split(/\n\n+/) - .filter((it) => it) - .join('\n\n') - return embeddingText.trim() -} diff --git a/server/indexer/src/indexer/types.ts b/server/indexer/src/indexer/types.ts index 91bec382a4..7d2d199884 100644 --- a/server/indexer/src/indexer/types.ts +++ b/server/indexer/src/indexer/types.ts @@ -17,15 +17,12 @@ import { type Class, type Doc, type DocIndexState, - type DocumentQuery, type DocumentUpdate, type FullTextSearchContext, type Hierarchy, - type MeasureContext, type ModelDb, type Ref } from '@hcengineering/core' -import type { DbAdapter, IndexedDoc } from '@hcengineering/server-core' /** * @public @@ -33,34 +30,8 @@ import type { DbAdapter, IndexedDoc } from '@hcengineering/server-core' export interface FullTextPipeline { hierarchy: Hierarchy model: ModelDb - contexts: Map>, FullTextSearchContext> - propogage: Map>, Ref>[]> - propogageClasses: Map>, Ref>[]> - - update: ( - docId: Ref, - mark: boolean, - update: DocumentUpdate, - flush?: boolean - ) => Promise - - add: (doc: DocIndexState) => void - markRemove: (doc: DocIndexState) => Promise - - search: ( - _classes: Ref>[], - search: DocumentQuery, - size: number | undefined, - from?: number - ) => Promise<{ docs: IndexedDoc[], pass: boolean }> - - queue: ( - ctx: MeasureContext, - updates: Map, { create?: DocIndexState, updated: boolean, removed: boolean }> - ) => Promise - cancelling: boolean } @@ -68,66 +39,3 @@ export interface FullTextPipeline { * @public */ export type DocUpdateHandler = (doc: DocIndexState, update: DocumentUpdate) => Promise - -/** - * @public - */ -export interface FullTextPipelineStage { - // States required to be complete - require: string[] - - // State to be updated - stageId: string - - // If specified, will clear all stages except specified + current - clearExcept?: string[] - - // Will propagate some changes for both mark values. - updateFields: DocUpdateHandler[] - - enabled: boolean - initialize: (ctx: MeasureContext, storage: DbAdapter, pipeline: FullTextPipeline) => Promise - - // Collect all changes related to bulk of document states - collect: (docs: DocIndexState[], pipeline: FullTextPipeline, ctx: MeasureContext) => Promise - - // Handle remove of items. - remove: (docs: DocIndexState[], pipeline: FullTextPipeline) => Promise - - // Search helper - search: ( - _classes: Ref>[], - search: DocumentQuery, - size: number | undefined, - from?: number - ) => Promise<{ docs: IndexedDoc[], pass: boolean }> -} - -/** - * @public - */ -export const contentStageId = 'cnt-v3' -/** - * @public - */ -export const fieldStateId = 'fld-v15' - -/** - * @public - */ -export const fullTextPushStageId = 'fts-v17' - -/** - * @public - */ -export const summaryStageId = 'sum-v5' - -/** - * @public - */ -export const collabStageId = 'collab-v1' - -/** - * @public - */ -export const fullTextPushStagePrefix = 'fts-' diff --git a/server/indexer/src/indexer/utils.ts b/server/indexer/src/indexer/utils.ts index 2f1e57bf25..a4d634633f 100644 --- a/server/indexer/src/indexer/utils.ts +++ b/server/indexer/src/indexer/utils.ts @@ -26,11 +26,10 @@ import core, { type Ref, type Space } from '@hcengineering/core' -import plugin from '@hcengineering/server-core' +import plugin, { type IndexedDoc } from '@hcengineering/server-core' import { type FullTextPipeline } from './types' -export { docKey, docUpdKey, extractDocKey, isFullTextAttribute } from '@hcengineering/core' -export type { IndexKeyOptions } from '@hcengineering/core' +export { docKey, isFullTextAttribute } from '@hcengineering/core' /** * @public @@ -97,42 +96,6 @@ export function traverseFullTextContexts ( } } -/** - * @public - */ -export function collectPropagate (pipeline: FullTextPipeline, objectClass: Ref>): Ref>[] { - let propagate = pipeline.propogage.get(objectClass) - if (propagate !== undefined) { - return propagate - } - const set = new Set>>() - traverseFullTextContexts(pipeline, objectClass, (fts) => { - fts?.propagate?.forEach((it) => { - set.add(it) - }) - }) - - propagate = Array.from(set.values()) - pipeline.propogage.set(objectClass, propagate) - return propagate -} - -/** - * @public - */ -export function collectPropagateClasses (pipeline: FullTextPipeline, objectClass: Ref>): Ref>[] { - let propagate = pipeline.propogageClasses.get(objectClass) - if (propagate !== undefined) { - return propagate - } - const set = new Set>>() - traverseFullTextContexts(pipeline, objectClass, (fts) => fts?.propagateClasses?.forEach((it) => set.add(it))) - - propagate = Array.from(set.values()) - pipeline.propogageClasses.set(objectClass, propagate) - return propagate -} - const CUSTOM_ATTR_KEY = 'customAttributes' const CUSTOM_ATTR_UPDATE_KEY = 'attributes.customAttributes' @@ -149,3 +112,16 @@ export function getCustomAttrKeys (): { customAttrKey: string, customAttrUKey: s export function isCustomAttr (attr: string): boolean { return attr === CUSTOM_ATTR_KEY } +/** + * @public + */ +export function createIndexedDoc (doc: Doc, mixins: Ref>[] | undefined, space: Ref): IndexedDoc { + const indexedDoc = { + id: doc._id, + _class: [doc._class, ...(mixins ?? [])], + modifiedBy: doc.modifiedBy, + modifiedOn: doc.modifiedOn, + space + } + return indexedDoc +} diff --git a/server/indexer/src/mapper.ts b/server/indexer/src/mapper.ts index 12b4070428..25aa5665e5 100644 --- a/server/indexer/src/mapper.ts +++ b/server/indexer/src/mapper.ts @@ -1,95 +1,24 @@ import { - docKey, - type Branding, + getObjectValue, type Class, type Doc, - type DocIndexState, type Hierarchy, type Ref, - type RefTo, - type SearchResultDoc + type SearchResultDoc, + type Space } from '@hcengineering/core' -import { getResource, type Resource } from '@hcengineering/platform' +import { getResource } from '@hcengineering/platform' import plugin, { - type ClassSearchConfigProps, + type FieldTemplate, + type FieldTemplateComponent, + type FieldTemplateParam, type IndexedDoc, type SearchPresenter, - type SearchPresenterFunc, type SearchScoring } from '@hcengineering/server-core' -interface IndexedReader { - get: (attribute: string) => any - getDoc: (attribute: string) => IndexedReader | undefined -} - -// TODO: Rework to use mongo -function createIndexedReader ( - _class: Ref>, - hierarchy: Hierarchy, - doc: DocIndexState, - otherDocs?: Record, - refAttribute?: string -): IndexedReader { - return { - get: (attr: string) => { - const realAttr = hierarchy.findAttribute(_class, attr) - if (realAttr !== undefined) { - return doc.attributes[docKey(attr, { _class: realAttr.attributeOf })] ?? (doc as any)[attr] - } - return undefined - }, - getDoc: (attr: string) => { - const realAttr = hierarchy.findAttribute(_class, attr) - if (realAttr !== undefined) { - const anotherDoc = otherDocs?.[attr] - if (anotherDoc !== undefined) { - const refAtrr = realAttr.type as RefTo - return createIndexedReader(refAtrr.to, hierarchy, anotherDoc, otherDocs, docKey(attr, { _class })) - } - } - return undefined - } - } -} - -async function readAndMapProps ( - reader: IndexedReader, - props: ClassSearchConfigProps[], - searchProvider?: { - hierarchy: Hierarchy - providers: SearchPresenterFunc - } -): Promise> { - const res: Record = {} - for (const prop of props) { - if (typeof prop === 'string') { - res[prop] = reader.get(prop) - } else { - for (const [propName, rest] of Object.entries(prop)) { - if (rest.length > 1) { - const val = reader.getDoc(rest[0])?.get(rest[1]) - const v = Array.isArray(val) ? val[0] : val - if (searchProvider !== undefined) { - const func = - searchProvider.providers !== undefined && Object.keys(searchProvider.providers).includes(propName) - ? ((await getResource(searchProvider.providers[propName])) as any) - : undefined - if (func !== undefined) { - res[propName] = func(searchProvider.hierarchy, { _class: res?._class, [propName]: v }) - continue - } - } - res[propName] = v - } - } - } - } - return res -} - -function findSearchPresenter (hierarchy: Hierarchy, _class: Ref>): SearchPresenter | undefined { +export function findSearchPresenter (hierarchy: Hierarchy, _class: Ref>): SearchPresenter | undefined { const ancestors = hierarchy.getAncestors(_class).reverse() for (const _class of ancestors) { const searchMixin = hierarchy.classHierarchyMixin(_class, plugin.mixin.SearchPresenter) @@ -105,64 +34,81 @@ function findSearchPresenter (hierarchy: Hierarchy, _class: Ref>): Se */ export async function updateDocWithPresenter ( hierarchy: Hierarchy, - doc: DocIndexState, + doc: Doc, elasticDoc: IndexedDoc, - refDocs: { - parentDoc: DocIndexState | undefined - spaceDoc: DocIndexState | undefined - }, - branding: Branding | null + parentDoc: Doc | undefined, + spaceDoc: Space | undefined, + searchPresenter: SearchPresenter ): Promise { - const searchPresenter = findSearchPresenter(hierarchy, doc.objectClass) - if (searchPresenter === undefined) { - return - } - - const reader = createIndexedReader(doc.objectClass, hierarchy, doc, { - space: refDocs.spaceDoc, - attachedTo: refDocs.parentDoc - }) - - const props = [ + const props: { name: string, config: FieldTemplateComponent | FieldTemplate }[] = [ { name: 'searchTitle', - config: searchPresenter.searchConfig.title, - provider: searchPresenter.getSearchTitle + config: searchPresenter.title } - ] as any[] + ] - if (searchPresenter.searchConfig.shortTitle !== undefined) { + if (searchPresenter.shortTitle !== undefined) { props.push({ name: 'searchShortTitle', - config: searchPresenter.searchConfig.shortTitle, - provider: searchPresenter.getSearchShortTitle, - lastNameFirst: branding?.lastNameFirst + config: searchPresenter.shortTitle }) } - if (searchPresenter.searchConfig.iconConfig !== undefined) { + if (searchPresenter.iconConfig !== undefined) { props.push({ name: 'searchIcon', - config: searchPresenter.searchConfig.iconConfig + config: searchPresenter.iconConfig }) } + async function extractParam (f: FieldTemplateParam): Promise { + if (f.length === 1) { + return getObjectValue(f[0], doc) + } + switch (f[0]) { + case 'func': { + const rf = await getResource(f[1]) + return rf(doc, parentDoc, spaceDoc, hierarchy, f[2]) + } + case 'space': + return spaceDoc !== undefined ? getObjectValue(f[1], spaceDoc) : '' + case 'parent': + return parentDoc !== undefined ? getObjectValue(f[1], parentDoc) : '' + } + } + async function formatTemplate (template: FieldTemplate): Promise { + let tValue = '' + for (const t of template) { + if (typeof t === 'string') { + tValue += t + } else { + tValue += `${await extractParam(t)}` + } + } + return tValue + } + for (const prop of props) { - let value - if (prop.config.tmpl !== undefined) { - const tmpl = prop.config.tmpl - const renderProps = await readAndMapProps(reader, prop.config.props, { hierarchy, providers: prop.provider }) - value = fillTemplate(tmpl, renderProps) - } else if (typeof prop.config === 'string') { - value = reader.get(prop.config) - } else if (prop.provider !== undefined) { - const func = await getResource(Object.values(prop.provider)[0] as Resource) - const renderProps = await readAndMapProps(reader, prop.config.props) - value = func(hierarchy, { _class: doc.objectClass, ...renderProps }) - } else if (prop.name === 'searchIcon') { - value = await readAndMapProps(reader, prop.config.props) + if (!Array.isArray(prop.config)) { + if (prop.config.fields !== undefined) { + const params: string[] = [] + for (const f of prop.config.fields) { + params.push(await extractParam(f)) + } + elasticDoc[prop.name + '_fields'] = params + } + if (prop.config.template !== undefined) { + elasticDoc[prop.name] = await formatTemplate(prop.config.template) + } + if (prop.config.extraFields !== undefined) { + elasticDoc[prop.name + '_extra'] = [] + for (const t of prop.config.extraFields) { + elasticDoc[prop.name + '_extra'].push(await formatTemplate(t)) + } + } + } else { + elasticDoc[prop.name] = await formatTemplate(prop.config) } - elasticDoc[prop.name] = value } } @@ -170,8 +116,8 @@ export function getScoringConfig (hierarchy: Hierarchy, classes: Ref> let results: SearchScoring[] = [] for (const _class of classes) { const searchPresenter = findSearchPresenter(hierarchy, _class) - if (searchPresenter?.searchConfig.scoring !== undefined) { - results = results.concat(searchPresenter?.searchConfig.scoring) + if (searchPresenter?.scoring !== undefined) { + results = results.concat(searchPresenter?.scoring) } } return results @@ -185,7 +131,6 @@ export function mapSearchResultDoc (hierarchy: Hierarchy, raw: IndexedDoc): Sear id: raw.id, title: raw.searchTitle, shortTitle: raw.searchShortTitle, - iconProps: raw.searchIcon, doc: { _id: raw.id, _class: raw._class[0] @@ -193,17 +138,52 @@ export function mapSearchResultDoc (hierarchy: Hierarchy, raw: IndexedDoc): Sear score: raw._score } - const searchPresenter = findSearchPresenter(hierarchy, doc.doc._class) - if (searchPresenter?.searchConfig.icon !== undefined) { - doc.icon = searchPresenter.searchConfig.icon + function fUpper (name: string): string { + return name[0].toUpperCase() + name.slice(1) } - if (searchPresenter?.searchConfig.iconConfig !== undefined) { - doc.iconComponent = searchPresenter.searchConfig.iconConfig.component + + function toProps (comp: FieldTemplateComponent, values: any[]): Record { + const result: Record = {} + if (!Array.isArray(comp)) { + let pos = 0 + for (const f of comp.fields ?? []) { + if (f.length === 1) { + result[f[0]] = values[pos] + pos++ + } + if (f.length === 2) { + result[f[0] + fUpper(f[1])] = values[pos] + pos++ + } + } + } + return result + } + + const searchPresenter = findSearchPresenter(hierarchy, doc.doc._class) + if (searchPresenter !== undefined) { + if (searchPresenter.searchIcon !== undefined) { + doc.icon = searchPresenter.searchIcon + } + if (searchPresenter.iconConfig !== undefined) { + doc.iconComponent = { + component: searchPresenter.iconConfig.component, + props: toProps(searchPresenter.iconConfig, raw.searchIcon_fields ?? []) + } + } + if (!Array.isArray(searchPresenter.title)) { + doc.titleComponent = { + component: searchPresenter.title.component, + props: toProps(searchPresenter.title, raw.searchTitle_fields ?? []) + } + } + if (searchPresenter.shortTitle !== undefined && !Array.isArray(searchPresenter.shortTitle)) { + doc.shortTitleComponent = { + component: searchPresenter.shortTitle.component, + props: toProps(searchPresenter.shortTitle, raw.searchShortTitle_fields ?? []) + } + } } return doc } - -function fillTemplate (tmpl: string, props: Record): string { - return tmpl.replace(/{(.*?)}/g, (_, key: string) => props[key]) -} diff --git a/server/indexer/src/rekoni.ts b/server/indexer/src/rekoni.ts index cd4a9e2850..9eea7d9990 100644 --- a/server/indexer/src/rekoni.ts +++ b/server/indexer/src/rekoni.ts @@ -5,14 +5,10 @@ import { generateToken } from '@hcengineering/server-token' /** * @public */ -export async function createRekoniAdapter ( - url: string, - workspace: WorkspaceId, - _metrics: MeasureContext -): Promise { - const token = generateToken('anticrm-hcenginnering', workspace) +export async function createRekoniAdapter (url: string): Promise { return { - content: async (name: string, type: string, doc): Promise => { + content: async (ctx: MeasureContext, workspace: WorkspaceId, name: string, type: string, doc): Promise => { + const token = generateToken('anticrm-hcenginnering', workspace) try { // Node doesn't support Readable with fetch. const chunks: any[] = [] @@ -21,7 +17,7 @@ export async function createRekoniAdapter ( doc.on('data', (chunk) => { len += (chunk as Buffer).length chunks.push(chunk) - if (len > 10 * 1024 * 1024) { + if (len > 30 * 1024 * 1024) { reject(new Error('file to big for content processing')) } }) @@ -38,27 +34,22 @@ export async function createRekoniAdapter ( await fetch(`${url}/toText?name=${encodeURIComponent(name)}&type=${encodeURIComponent(type)}`, { method: 'POST', body, - // timeout: 15000, headers: { Authorization: 'Bearer ' + token, - 'Content-type': typeof doc === 'string' ? 'text/plain' : 'application/octet-stream' + 'Content-type': 'application/octet-stream' } } as any) ).json() if (r.error !== undefined) { - throw new Error(r.error) + throw new Error(JSON.stringify(r.error)) } return r.content } catch (err: any) { - console.info('Content Processing error', name, type, doc, err) if (err.message === 'Response code 400 (Bad Request)' || err.code === 400) { return '' } throw err } - }, - metrics (): MeasureContext { - return _metrics } } } diff --git a/server/indexer/src/ydoc.ts b/server/indexer/src/ydoc.ts index 4b6fbaa9d4..140a2bd519 100644 --- a/server/indexer/src/ydoc.ts +++ b/server/indexer/src/ydoc.ts @@ -1,19 +1,21 @@ import { type MeasureContext, type WorkspaceId } from '@hcengineering/core' import { type ContentTextAdapter } from '@hcengineering/server-core' import { pmNodeToText, yDocContentToNodes } from '@hcengineering/text' -import { Readable } from 'node:stream' import { Buffer } from 'node:buffer' +import { Readable } from 'node:stream' /** * @public */ -export async function createYDocAdapter ( - _url: string, - _workspace: WorkspaceId, - _metrics: MeasureContext -): Promise { +export async function createYDocAdapter (_url: string): Promise { return { - content: async (_name: string, _type: string, data: Readable | Buffer | string): Promise => { + content: async ( + ctx: MeasureContext, + workspace: WorkspaceId, + _name: string, + _type: string, + data: Readable | Buffer | string + ): Promise => { const chunks: any[] = [] if (data instanceof Readable) { @@ -42,9 +44,6 @@ export async function createYDocAdapter ( } return '' - }, - metrics (): MeasureContext { - return _metrics } } } diff --git a/server/middleware/package.json b/server/middleware/package.json index 9ef285c7b6..3b04b8a8f0 100644 --- a/server/middleware/package.json +++ b/server/middleware/package.json @@ -43,6 +43,7 @@ "@hcengineering/server-preference": "^0.6.0", "@hcengineering/server-notification": "^0.6.1", "@hcengineering/query": "^0.6.12", + "@hcengineering/analytics": "^0.6.0", "fast-equals": "^5.0.1" } } diff --git a/server/middleware/src/domainFind.ts b/server/middleware/src/domainFind.ts index 1597bc339f..cf73fdf46b 100644 --- a/server/middleware/src/domainFind.ts +++ b/server/middleware/src/domainFind.ts @@ -69,7 +69,12 @@ export class DomainFindMiddleware extends BaseMiddleware implements Middleware { ) } - groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { - return this.adapterManager.getAdapter(domain, false).groupBy(ctx, domain, field) + groupBy( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery

+ ): Promise> { + return this.adapterManager.getAdapter(domain, false).groupBy(ctx, domain, field, query) } } diff --git a/server/middleware/src/fulltext.ts b/server/middleware/src/fulltext.ts new file mode 100644 index 0000000000..ee8a8cebb0 --- /dev/null +++ b/server/middleware/src/fulltext.ts @@ -0,0 +1,435 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Analytics } from '@hcengineering/analytics' +import core, { + docKey, + isFullTextAttribute, + isIndexedAttribute, + toFindResult, + type AttachedDoc, + type Class, + type Collection, + type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, + type FullTextSearchContext, + type MeasureContext, + type ObjQueryType, + type Ref, + type SearchOptions, + type SearchQuery, + type SearchResult, + type SessionData, + type Tx +} from '@hcengineering/core' +import { PlatformError, unknownError } from '@hcengineering/platform' +import type { + IndexedDoc, + Middleware, + MiddlewareCreator, + PipelineContext, + TxMiddlewareResult +} from '@hcengineering/server-core' +import { BaseMiddleware } from '@hcengineering/server-core' +/** + * @public + */ +export class FullTextMiddleware extends BaseMiddleware implements Middleware { + fulltextEndpoint: string + contexts = new Map>, FullTextSearchContext>() + + warmupTriggered: boolean = false + + constructor ( + context: PipelineContext, + next: Middleware | undefined, + fulltextUrl: string, + readonly token: string + ) { + super(context, next) + const fulltextEndpoints = fulltextUrl.split(';').map((it) => it.trim()) + + const hash = this.hashWorkspace(context.workspace.name) + this.fulltextEndpoint = fulltextEndpoints[Math.abs(hash % fulltextEndpoints.length)] + } + + hashWorkspace (dbWorkspaceName: string): number { + return [...dbWorkspaceName].reduce((hash, c) => (Math.imul(31, hash) + c.charCodeAt(0)) | 0, 0) + } + + static create (url: string, token: string): MiddlewareCreator { + return async (ctx, context, next): Promise => { + const middleware = new FullTextMiddleware(context, next, url, token) + await middleware.init(ctx) + return middleware + } + } + + async handleBroadcast (ctx: MeasureContext): Promise { + if (ctx.contextData.needWarmupFulltext === true) { + void this.sendWarmup() + } + await super.handleBroadcast(ctx) + } + + async init (ctx: MeasureContext): Promise { + if (this.context.adapterManager == null) { + throw new PlatformError(unknownError('Adapter manager should be specified')) + } + this.contexts = new Map( + this.context.modelDb.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it]) + ) + + // Send one warumup so reindex will + void this.sendWarmup() + } + + async sendWarmup (): Promise { + if (!this.warmupTriggered) { + this.warmupTriggered = true + setTimeout(() => { + this.warmupTriggered = false + void this._sendWarmup() + }, 25) + } + } + + async _sendWarmup (): Promise { + // Just send a warmup to indexer to start do something, or just reindex if some work are pending. + try { + await fetch(this.fulltextEndpoint + '/api/v1/warmup', { + method: 'PUT', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + token: this.token + }) + }) + } catch (err: any) { + if (err?.cause?.code === 'ECONNRESET' || err?.cause?.code === 'ECONNREFUSED') { + return + } + Analytics.handleError(err) + } + } + + async search( + _classes: Ref>[], + query: DocumentQuery, + fullTextLimit: number + ): Promise { + try { + return await ( + await fetch(this.fulltextEndpoint + '/api/v1/search', { + method: 'PUT', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + token: this.token, + workspace: this.context.workspace, + _classes, + query, + fullTextLimit + }) + }) + ).json() + } catch (err: any) { + if (err?.cause?.code === 'ECONNRESET' || err?.cause?.code === 'ECONNREFUSED') { + // TODO: We need to send event about indexing is complete after a while + return [] + } + Analytics.handleError(err) + return [] + } + } + + async findAll( + ctx: MeasureContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + if (query?.$search === undefined) { + return await this.provideFindAll(ctx, _class, query, options) + } + + const { _id, $search, ...mainQuery } = query + if ($search === undefined) { + return toFindResult([]) + } + + const ids: Set> = new Set>() + const childIds: Set> = new Set>() + const baseClass = this.context.hierarchy.getBaseClass(_class) + let classes = this.context.hierarchy.getDescendants(baseClass).filter((it) => !this.context.hierarchy.isMixin(it)) + + const attrs = this.context.hierarchy.getAllAttributes(_class) + + // We need to filter all non indexed fields from query to make it work properly + const findQuery: DocumentQuery = { + $search: query.$search + } + + const childClasses = new Set>>() + try { + for (const [k, attr] of attrs) { + if (isFullTextAttribute(attr) || isIndexedAttribute(attr)) { + const vv = (query as any)[k] + if (vv != null) { + if ( + k === '_class' || + k === 'modifiedBy' || + k === 'modifiedOn' || + k === 'space' || + k === 'attachedTo' || + k === 'attachedToClass' + ) { + findQuery[k] = vv + } else { + const docKeyValue = docKey(attr.name, attr.attributeOf) + findQuery[docKeyValue] = vv + } + } + } + if (attr.type._class === core.class.Collection) { + // we need attached documents to be in classes + const coll = attr.type as Collection + const dsc = this.context.hierarchy.getDescendants(coll.of).filter((it) => !this.context.hierarchy.isMixin(it)) + for (const d of dsc) { + childClasses.add(d) + } + } + } + } catch (err: any) { + Analytics.handleError(err) + } + + classes = classes.filter((it, idx, arr) => arr.indexOf(it) === idx) + + classes = classes.filter((it) => { + if (typeof query._class === 'object') { + if (query._class?.$in !== undefined) { + return query._class.$in.includes(it) + } + if (query._class?.$nin !== undefined) { + return !query._class.$nin.includes(it) + } + } + return true + }) + + const fullTextLimit = Math.min(5000, (options?.limit ?? 200) * 100) + + // Find main documents, not attached ones. + const { indexedDocMap } = await this.findDocuments(classes, findQuery, fullTextLimit, baseClass, ids) + + for (const c of classes) { + // We do not need to overlap + childClasses.delete(c) + } + + const childQuery: DocumentQuery = { + $search: findQuery.$search, + attachedToClass: { $in: classes } + } + if (findQuery.space !== undefined) { + childQuery.space = findQuery.space + } + const { childDocs, childIndexedDocMap } = + childClasses !== undefined && childClasses.size > 0 + ? await this.findChildDocuments(Array.from(childClasses), childQuery, fullTextLimit, baseClass, childIds) + : { + childDocs: [], + childIndexedDocMap: new Map() + } + + const scoreSearch: number | undefined = (options?.sort as any)?.['#score'] + + const resultIds = Array.from(this.getResultIds(ids, _id)) + const childResultIds = Array.from(this.getResultIds(childIds, _id)) + resultIds.push(...childResultIds) + + let result = + resultIds.length > 0 + ? await this.provideFindAll( + ctx, + _class, + { _id: { $in: Array.from(new Set(resultIds)) }, ...mainQuery }, + options + ) + : toFindResult([]) + + // Just assign scores based on idex + result.forEach((it) => { + const idDoc = indexedDocMap.get(it._id) ?? childIndexedDocMap.get(it._id) + const { _score } = idDoc + + const maxScore = childDocs.reduceRight((p, cur) => (p > cur.$score ? p : cur.$score), _score) + + it.$source = { + $score: maxScore + } + }) + if (scoreSearch !== undefined) { + result.sort((a, b) => scoreSearch * ((a.$source?.$score ?? 0) - (b.$source?.$score ?? 0))) + if (options?.limit !== undefined && options?.limit < result.length) { + result = toFindResult(result.slice(0, options?.limit), result.total) + } + } + return result + } + + private async findDocuments( + classes: Ref>[], + findQuery: DocumentQuery, + fullTextLimit: number, + baseClass: Ref>, + ids: Set> + ): Promise<{ docs: IndexedDoc[], indexedDocMap: Map, IndexedDoc> }> { + const docs = await this.search(classes, findQuery, fullTextLimit) + + const indexedDocMap = new Map, IndexedDoc>() + + for (const doc of docs) { + if ( + doc._class != null && + Array.isArray(doc._class) && + doc._class.some((cl) => this.context.hierarchy.isDerived(cl, baseClass)) + ) { + ids.add(doc.id) + indexedDocMap.set(doc.id, doc) + } + if ( + doc._class !== null && + !Array.isArray(doc._class) && + this.context.hierarchy.isDerived(doc._class, baseClass) + ) { + ids.add(doc.id) + indexedDocMap.set(doc.id, doc) + } + } + return { docs, indexedDocMap } + } + + private async findChildDocuments( + classes: Ref>[], + findQuery: DocumentQuery, + fullTextLimit: number, + baseClass: Ref>, + ids: Set> + ): Promise<{ childDocs: IndexedDoc[], childIndexedDocMap: Map, IndexedDoc> }> { + const childDocs = await this.search(classes, findQuery, fullTextLimit) + + const childIndexedDocMap = new Map, IndexedDoc>() + + for (const doc of childDocs) { + if (doc.attachedTo != null) { + if (doc.attachedToClass != null && this.context.hierarchy.isDerived(doc.attachedToClass, baseClass)) { + if (this.context.hierarchy.isDerived(doc.attachedToClass, baseClass)) { + ids.add(doc.attachedTo) + childIndexedDocMap.set(doc.attachedTo, doc) + } + } else { + ids.add(doc.attachedTo) + childIndexedDocMap.set(doc.attachedTo, doc) + } + } + } + return { childDocs, childIndexedDocMap } + } + + async searchFulltext (ctx: MeasureContext, query: SearchQuery, options: SearchOptions): Promise { + try { + return await ctx.with('full-text-search', {}, async (ctx) => { + return await ( + await fetch(this.fulltextEndpoint + '/api/v1/full-text-search', { + method: 'PUT', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + workspace: this.context.workspace, + token: this.token, + query, + options + }) + }) + ).json() + }) + } catch (err: any) { + if (err?.cause?.code === 'ECONNRESET' || err?.cause?.code === 'ECONNREFUSED') { + // TODO: We need to send event about indexing is complete after a while + return { docs: [] } + } + Analytics.handleError(err) + return { docs: [] } + } + } + + tx (ctx: MeasureContext, txes: Tx[]): Promise { + return this.provideTx(ctx, txes) + } + + async close (): Promise { + try { + await fetch(this.fulltextEndpoint + '/api/v1/close', { + method: 'PUT', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + token: this.token + }) + }) + } catch (err: any) { + if (err?.cause?.code === 'ECONNRESET' || err?.cause?.code === 'ECONNREFUSED') { + return + } + Analytics.handleError(err) + } + } + + getResultIds (ids: Set>, _id: ObjQueryType> | undefined): Set> { + const result = new Set>() + if (_id !== undefined) { + if (typeof _id === 'string') { + if (ids.has(_id)) { + result.add(_id) + } + } else if (_id.$in !== undefined) { + for (const id of _id.$in) { + if (ids.has(id)) { + result.add(id) + } + } + } else if (_id.$nin !== undefined) { + for (const id of _id.$nin) { + ids.delete(id) + } + return ids + } else if (_id.$ne !== undefined) { + ids.delete(_id.$ne) + return ids + } + } else { + return ids + } + return result + } +} diff --git a/server/middleware/src/index.ts b/server/middleware/src/index.ts index 67d4157877..908b412a29 100644 --- a/server/middleware/src/index.ts +++ b/server/middleware/src/index.ts @@ -23,6 +23,7 @@ export * from './dbAdapterHelper' export * from './derivedEntry' export * from './domainFind' export * from './domainTx' +export * from './fulltext' export * from './liveQuery' export * from './lookup' export * from './lowLevel' diff --git a/server/middleware/src/lowLevel.ts b/server/middleware/src/lowLevel.ts index 608aa11aeb..9d4d8b8004 100644 --- a/server/middleware/src/lowLevel.ts +++ b/server/middleware/src/lowLevel.ts @@ -58,8 +58,13 @@ export class LowLevelMiddleware extends BaseMiddleware implements Middleware { async clean (ctx: MeasureContext, domain: Domain, docs: Ref[]): Promise { await adapterManager.getAdapter(domain, true).clean(ctx, domain, docs) }, - groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { - return adapterManager.getAdapter(domain, false).groupBy(ctx, domain, field) + groupBy( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery

+ ): Promise> { + return adapterManager.getAdapter(domain, false).groupBy(ctx, domain, field, query) }, rawFindAll(domain: Domain, query: DocumentQuery, options?: FindOptions): Promise { return adapterManager.getAdapter(domain, false).rawFindAll(domain, query, options) diff --git a/server/middleware/src/model.ts b/server/middleware/src/model.ts index 100b472c88..9da40d896f 100644 --- a/server/middleware/src/model.ts +++ b/server/middleware/src/model.ts @@ -45,7 +45,8 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware { constructor ( context: PipelineContext, next: Middleware | undefined, - readonly systemTx: Tx[] + readonly systemTx: Tx[], + readonly holdModel: boolean = true ) { super(context, next) } @@ -54,16 +55,17 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware { ctx: MeasureContext, context: PipelineContext, next: Middleware | undefined, - systemTx: Tx[] + systemTx: Tx[], + holdModel: boolean = true ): Promise { - const middleware = new ModelMiddleware(context, next, systemTx) + const middleware = new ModelMiddleware(context, next, systemTx, holdModel) await middleware.init(ctx) return middleware } - static create (tx: Tx[]): MiddlewareCreator { + static create (tx: Tx[], holdModel: boolean = true): MiddlewareCreator { return (ctx, context, next) => { - return this.doCreate(ctx, context, next, tx) + return this.doCreate(ctx, context, next, tx, holdModel) } } @@ -90,20 +92,25 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware { } } this.context.modelDb.addTxes(ctx, model, true) - return model + if (this.holdModel) { + return model + } + return [] }) this.setModel(this.model) } private addModelTx (tx: Tx): void { - this.model.push(tx) - const h = crypto.createHash('sha1') - h.update(this.lastHash) - h.update(JSON.stringify(tx)) - const hash = h.digest('hex') - this.hashes.set(hash, this.model.length - 1) - this.setLastHash(hash) + if (this.holdModel) { + this.model.push(tx) + const h = crypto.createHash('sha1') + h.update(this.lastHash) + h.update(JSON.stringify(tx)) + const hash = h.digest('hex') + this.hashes.set(hash, this.model.length - 1) + this.setLastHash(hash) + } } private setLastHash (hash: string): void { @@ -117,15 +124,14 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware { private setModel (model: Tx[]): void { let last = '' - this.hashes = new Map( - model.map((it, index) => { - const h = crypto.createHash('sha1') - h.update(last) - h.update(JSON.stringify(it)) - last = h.digest('hex') - return [last, index] - }) - ) + const values: [string, number][] = model.map((it, index) => { + const h = crypto.createHash('sha1') + h.update(last) + h.update(JSON.stringify(it)) + last = h.digest('hex') + return [last, index] + }) + this.hashes = new Map(values) this.setLastHash(last) } diff --git a/server/middleware/src/spaceSecurity.ts b/server/middleware/src/spaceSecurity.ts index a7e76eeba1..257219e60c 100644 --- a/server/middleware/src/spaceSecurity.ts +++ b/server/middleware/src/spaceSecurity.ts @@ -426,7 +426,9 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar async getDomainSpaces (ctx: MeasureContext, domain: Domain): Promise>> { let domainSpaces = this._domainSpaces.get(domain) if (domainSpaces === undefined) { - const p = this.next?.groupBy>(ctx, domain, this.getKey(domain)) ?? Promise.resolve(new Set()) + const p = ( + this.next?.groupBy, Doc>(ctx, domain, this.getKey(domain)) ?? Promise.resolve(new Map()) + ).then((r) => new Set>(r.keys())) this._domainSpaces.set(domain, p) domainSpaces = await p this._domainSpaces.set(domain, domainSpaces) diff --git a/server/middleware/src/triggers.ts b/server/middleware/src/triggers.ts index 6ecce4c76f..175afc122d 100644 --- a/server/middleware/src/triggers.ts +++ b/server/middleware/src/triggers.ts @@ -22,6 +22,7 @@ import core, { DOMAIN_MODEL, type Doc, type DocumentUpdate, + type LowLevelStorage, type MeasureContext, type Mixin, type Ref, @@ -49,6 +50,7 @@ import type { TxMiddlewareResult } from '@hcengineering/server-core' import serverCore, { BaseMiddleware, SessionDataImpl, SessionFindAll, Triggers } from '@hcengineering/server-core' +import { filterBroadcastOnly } from './utils' /** * @public @@ -85,6 +87,9 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware { if (this.context.storageAdapter == null) { throw new PlatformError(unknownError('Storage adapter should be specified')) } + if (this.context.lowLevelStorage == null) { + throw new PlatformError(unknownError('Low level storage should be specified')) + } this.storageAdapter = this.context.storageAdapter this.triggers.init(this.context.modelDb) } @@ -92,7 +97,11 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware { async tx (ctx: MeasureContext, tx: Tx[]): Promise { await this.triggers.tx(tx) const result = await this.provideTx(ctx, tx) - await this.processDerived(ctx, tx) + + const ftx = filterBroadcastOnly(tx, this.context.hierarchy) + if (ftx.length > 0) { + await this.processDerived(ctx, ftx) + } return result } @@ -119,6 +128,7 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware { const triggerControl: Omit = { removedMap: ctx.contextData.removedMap, workspace: this.context.workspace, + lowLevel: this.context.lowLevelStorage as LowLevelStorage, branding: this.context.branding, storageAdapter: this.storageAdapter, serviceAdaptersManager: this.context.serviceAdapterManager as ServiceAdaptersManager, diff --git a/server/middleware/src/utils.ts b/server/middleware/src/utils.ts index 1ab7e9a17f..deeec45855 100644 --- a/server/middleware/src/utils.ts +++ b/server/middleware/src/utils.ts @@ -13,7 +13,17 @@ // limitations under the License. // -import core, { Account, AccountRole, type MeasureContext, type SessionData } from '@hcengineering/core' +import core, { + Account, + AccountRole, + TxProcessor, + type Doc, + type Hierarchy, + type MeasureContext, + type SessionData, + type Tx, + type TxCUD +} from '@hcengineering/core' export function isOwner (account: Account, ctx: MeasureContext): boolean { return account.role === AccountRole.Owner || account._id === core.account.System || ctx.contextData.admin === true @@ -22,3 +32,24 @@ export function isOwner (account: Account, ctx: MeasureContext): bo export function isSystem (account: Account, ctx: MeasureContext): boolean { return account._id === core.account.System || ctx.contextData.admin === true } + +export function filterBroadcastOnly (tx: Tx[], hierarchy: Hierarchy): Tx[] { + const ftx = tx.filter((it) => { + if (TxProcessor.isExtendsCUD(it._class)) { + const cud = it as TxCUD + const bonly = hierarchy.getClassifierProp(cud.objectClass, 'broadcastOnly') + if (bonly === true) { + return false + } + const objClass = hierarchy.getClass(cud.objectClass) + const mix = hierarchy.hasMixin(objClass, core.mixin.TransientConfiguration) + if (mix && hierarchy.as(objClass, core.mixin.TransientConfiguration).broadcastOnly) { + hierarchy.setClassifierProp(cud.objectClass, 'broadcastOnly', true) + // We do not need to store a broadcast only transactions into model. + return false + } + } + return true + }) + return ftx +} diff --git a/server/minio/src/index.ts b/server/minio/src/index.ts index 405b9b2a3d..094519b5ef 100644 --- a/server/minio/src/index.ts +++ b/server/minio/src/index.ts @@ -291,7 +291,8 @@ export class MinioService implements StorageAdapter { err?.code === 'NoSuchKey' || err?.code === 'NotFound' || err?.message === 'No such key' || - err?.Code === 'NoSuchKey' + err?.Code === 'NoSuchKey' || + err?.code === 'ECONNRESET' ) { // Do not print error in this case return diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index 37f63b6665..88706936b6 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -491,9 +491,13 @@ abstract class MongoAdapterBase implements DbAdapter { lookup: Lookup | undefined, object: any, parent?: string, - parentObject?: any + parentObject?: any, + domainLookup?: { + field: string + domain: Domain + } ): Promise { - if (lookup === undefined) return + if (lookup === undefined && domainLookup === undefined) return for (const key in lookup) { if (key === '_id') { await this.fillReverseLookup(clazz, lookup, object, parent, parentObject) @@ -511,6 +515,14 @@ abstract class MongoAdapterBase implements DbAdapter { await this.fillLookup(value, object, key, fullKey, targetObject) } } + if (domainLookup !== undefined) { + if (object.$lookup === undefined) { + object.$lookup = {} + } + object.$lookup._id = object['dl_' + domainLookup.field + '_lookup'][0] + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete object['dl_' + domainLookup.field + '_lookup'] + } } private async fillReverseLookup( @@ -590,8 +602,18 @@ abstract class MongoAdapterBase implements DbAdapter { const pipeline: any[] = [] const tquery = this.translateQuery(clazz, query, options) - const slowPipeline = isLookupQuery(query) || isLookupSort(options?.sort) + const slowPipeline = isLookupQuery(query) || isLookupSort(options?.sort) || options.domainLookup !== undefined const steps = this.getLookups(clazz, options?.lookup) + + if (options.domainLookup !== undefined) { + steps.push({ + from: options.domainLookup.domain, + localField: options.domainLookup.field, + foreignField: '_id', + as: 'dl_' + options.domainLookup.field + '_lookup' + }) + } + if (slowPipeline) { if (Object.keys(tquery.base).length > 0) { pipeline.push({ $match: tquery.base }) @@ -649,7 +671,7 @@ abstract class MongoAdapterBase implements DbAdapter { } for (const row of result) { await ctx.with('fill-lookup', {}, async (ctx) => { - await this.fillLookupValue(ctx, clazz, options?.lookup, row) + await this.fillLookupValue(ctx, clazz, options?.lookup, row, undefined, undefined, options.domainLookup) }) if (row.$lookup !== undefined) { for (const [, v] of Object.entries(row.$lookup)) { @@ -783,7 +805,7 @@ abstract class MongoAdapterBase implements DbAdapter { domain: Domain, field: string, query?: DocumentQuery - ): Promise> { + ): Promise> { const result = await ctx.with('groupBy', { domain }, async (ctx) => { const coll = this.collection(domain) const grResult = await coll @@ -791,12 +813,13 @@ abstract class MongoAdapterBase implements DbAdapter { ...(query !== undefined ? [{ $match: query }] : []), { $group: { - _id: '$' + field + _id: '$' + field, + count: { $sum: 1 } } } ]) .toArray() - return new Set(grResult.map((it) => it._id as unknown as T)) + return new Map(grResult.map((it) => [it._id as unknown as T, it.count])) }) return result } @@ -815,10 +838,12 @@ abstract class MongoAdapterBase implements DbAdapter { let result: FindResult const domain = options?.domain ?? this.hierarchy.getDomain(_class) if ( - options != null && - (options?.lookup != null || this.isEnumSort(_class, options) || this.isRulesSort(options)) + options?.lookup != null || + this.isEnumSort(_class, options) || + this.isRulesSort(options) || + options?.domainLookup !== undefined ) { - return await this.findWithPipeline(ctx, domain, _class, query, options, stTime) + return await this.findWithPipeline(ctx, domain, _class, query, options ?? {}, stTime) } const coll = this.collection(domain) diff --git a/server/postgres/src/storage.ts b/server/postgres/src/storage.ts index fa3f5b7d72..7722977156 100644 --- a/server/postgres/src/storage.ts +++ b/server/postgres/src/storage.ts @@ -66,6 +66,7 @@ import { } from '@hcengineering/server-core' import { createHash } from 'crypto' import type postgres from 'postgres' +import { getDocFieldsByDomains, translateDomain } from './schemas' import { type ValueType } from './types' import { convertDoc, @@ -83,7 +84,6 @@ import { parseUpdate, type PostgresClientReference } from './utils' -import { getDocFieldsByDomains, translateDomain } from './schemas' abstract class PostgresAdapterBase implements DbAdapter { protected readonly _helper: DBCollectionHelper @@ -364,6 +364,23 @@ abstract class PostgresAdapterBase implements DbAdapter { const domain = translateDomain(options?.domain ?? this.hierarchy.getDomain(_class)) const sqlChunks: string[] = [] const joins = this.buildJoin(_class, options?.lookup) + if (options?.domainLookup !== undefined) { + const baseDomain = translateDomain(this.hierarchy.getDomain(_class)) + + const domain = translateDomain(options.domainLookup.domain) + const key = options.domainLookup.field + const as = `dl_lookup_${domain}_${key}` + joins.push({ + isReverse: false, + table: domain, + path: options.domainLookup.field, + toAlias: as, + toField: '_id', + fromField: key, + fromAlias: baseDomain, + toClass: undefined + }) + } const select = `SELECT ${this.getProjection(_class, domain, options?.projection, joins)} FROM ${domain}` const secJoin = this.addSecurity(query, domain, ctx.contextData) if (secJoin !== undefined) { @@ -392,7 +409,7 @@ abstract class PostgresAdapterBase implements DbAdapter { const finalSql: string = [select, ...sqlChunks].join(' ') const result = await connection.unsafe(finalSql) - if (options?.lookup === undefined) { + if (options?.lookup === undefined && options?.domainLookup === undefined) { return toFindResult( result.map((p) => parseDocWithProjection(p as any, options?.projection)), total @@ -541,7 +558,7 @@ abstract class PostgresAdapterBase implements DbAdapter { if (res === undefined) continue const { obj, key } = res const val = this.getModelLookupValue(doc, modelJoin, simpleJoins) - if (val !== undefined) { + if (val !== undefined && modelJoin.toClass !== undefined) { const res = this.modelDb.findAllSync(modelJoin.toClass, { [modelJoin.toField]: (doc as any)[modelJoin.fromField] }) @@ -1211,16 +1228,20 @@ abstract class PostgresAdapterBase implements DbAdapter { await connection`DELETE FROM ${connection(translateDomain(domain))} WHERE _id = ANY(${docs}) AND "workspaceId" = ${this.workspaceId.name}` } - async groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { + async groupBy( + ctx: MeasureContext, + domain: Domain, + field: string, + query?: DocumentQuery

+ ): Promise> { const connection = (await this.getConnection(ctx)) ?? this.client const key = isDataField(domain, field) ? `data ->> '${field}'` : `"${field}"` const result = await ctx.with('groupBy', { domain }, async (ctx) => { try { const result = await connection.unsafe( - `SELECT DISTINCT ${key} as ${field} FROM ${translateDomain(domain)} WHERE "workspaceId" = $1`, - [this.workspaceId.name] + `SELECT DISTINCT ${key} as ${field}, Count(*) AS count FROM ${translateDomain(domain)} WHERE ${this.buildRawQuery(domain, query ?? {})} GROUP BY ${key}` ) - return new Set(result.map((r) => r[field])) + return new Map(result.map((r) => [r[field.toLocaleLowerCase()], parseInt(r.count)])) } catch (err) { ctx.error('Error while grouping by', { domain, field }) throw err diff --git a/server/postgres/src/utils.ts b/server/postgres/src/utils.ts index 675f04adda..77dedcad4c 100644 --- a/server/postgres/src/utils.ts +++ b/server/postgres/src/utils.ts @@ -468,6 +468,6 @@ export interface JoinProps { toAlias: string // alias for the table toField: string // field to join on isReverse: boolean - toClass: Ref> + toClass?: Ref> classes?: Ref>[] // filter by classes } diff --git a/server/server-pipeline/package.json b/server/server-pipeline/package.json index a74e3d3b0c..2a3275977c 100644 --- a/server/server-pipeline/package.json +++ b/server/server-pipeline/package.json @@ -40,7 +40,6 @@ "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", "@hcengineering/server-core": "^0.6.1", - "@hcengineering/server-indexer": "^0.6.0", "@hcengineering/server-attachment": "^0.6.1", "@hcengineering/server-attachment-resources": "^0.6.0", "@hcengineering/server-collaboration": "^0.6.0", @@ -49,7 +48,6 @@ "@hcengineering/server-storage": "^0.6.0", "@hcengineering/postgres": "^0.6.0", "@hcengineering/mongo": "^0.6.1", - "@hcengineering/elastic": "^0.6.0", "@hcengineering/server-contact": "^0.6.1", "@hcengineering/server-contact-resources": "^0.6.0", "@hcengineering/server-notification": "^0.6.1", @@ -173,6 +171,8 @@ "@hcengineering/server-ai-bot": "^0.6.0", "@hcengineering/server-ai-bot-resources": "^0.6.0", "@hcengineering/time": "^0.6.0", - "@hcengineering/time-assets": "^0.6.0" + "@hcengineering/time-assets": "^0.6.0", + "@hcengineering/server-fulltext": "^0.6.0", + "@hcengineering/server-fulltext-resources": "^0.6.0" } } diff --git a/server/server-pipeline/src/index.ts b/server/server-pipeline/src/index.ts index 21ef35d4c3..c7cefc2637 100644 --- a/server/server-pipeline/src/index.ts +++ b/server/server-pipeline/src/index.ts @@ -14,7 +14,6 @@ // limitations under the License. // -export * from './indexing' export * from './internationalization' export * from './pipeline' export * from './serverPlugins' diff --git a/server/server-pipeline/src/indexing.ts b/server/server-pipeline/src/indexing.ts deleted file mode 100644 index 784adfb12c..0000000000 --- a/server/server-pipeline/src/indexing.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* eslint-disable @typescript-eslint/unbound-method */ -import { type Branding, type MeasureContext, type WorkspaceId } from '@hcengineering/core' -import { CollaborativeContentRetrievalStage } from '@hcengineering/server-collaboration' -import { - type ContentTextAdapter, - type FullTextAdapter, - type SessionFindAll, - type StorageAdapter -} from '@hcengineering/server-core' - -import { - ContentRetrievalStage, - FullSummaryStage, - FullTextPushStage, - IndexedFieldStage, - globalIndexer, - type FullTextPipelineStage -} from '@hcengineering/server-indexer' - -export function createIndexStages ( - fullText: MeasureContext, - workspace: WorkspaceId, - branding: Branding | null, - adapter: FullTextAdapter, - storageFindAll: SessionFindAll, - storageAdapter: StorageAdapter, - contentAdapter: ContentTextAdapter, - indexParallel: number, - indexProcessing: number -): FullTextPipelineStage[] { - // Allow 2 workspaces to be indexed in parallel - globalIndexer.allowParallel = indexParallel - globalIndexer.processingSize = indexProcessing - - const stages: FullTextPipelineStage[] = [] - - // Add regular stage to for indexable fields change tracking. - stages.push(new IndexedFieldStage(storageFindAll)) - - // Obtain text content from storage(like minio) and use content adapter to convert files to text content. - stages.push(new ContentRetrievalStage(storageAdapter, workspace, fullText.newChild('content', {}), contentAdapter)) - - // Obtain collaborative content - stages.push( - new CollaborativeContentRetrievalStage( - storageAdapter, - workspace, - fullText.newChild('collaborative', {}), - contentAdapter - ) - ) - // Summary stage - const summaryStage = new FullSummaryStage(storageFindAll) - - stages.push(summaryStage) - - // Push all content to elastic search - const pushStage = new FullTextPushStage(storageFindAll, adapter, workspace, branding) - stages.push(pushStage) - - // // OpenAI prepare stage - // const openAIStage = new OpenAIEmbeddingsStage(adapter, workspace) - // // We depend on all available stages. - // openAIStage.require = stages.map((it) => it.stageId) - - // openAIStage.updateSummary(summaryStage) - - // stages.push(openAIStage) - - return stages -} diff --git a/server/server-pipeline/src/pipeline.ts b/server/server-pipeline/src/pipeline.ts index ee77c3da08..494031263c 100644 --- a/server/server-pipeline/src/pipeline.ts +++ b/server/server-pipeline/src/pipeline.ts @@ -2,18 +2,17 @@ import { DOMAIN_BENCHMARK, DOMAIN_BLOB, - DOMAIN_FULLTEXT_BLOB, DOMAIN_MODEL, DOMAIN_TRANSIENT, DOMAIN_TX, Hierarchy, ModelDb, + systemAccountEmail, type Branding, type MeasureContext, type Tx, type WorkspaceIdWithUrl } from '@hcengineering/core' -import { createElasticAdapter, createElasticBackupDataAdapter } from '@hcengineering/elastic' import { ApplyTxMiddleware, BroadcastMiddleware, @@ -24,6 +23,7 @@ import { DBAdapterMiddleware, DomainFindMiddleware, DomainTxMiddleware, + FullTextMiddleware, LiveQueryMiddleware, LookupMiddleware, LowLevelMiddleware, @@ -45,8 +45,6 @@ import { createInMemoryAdapter, createNullAdapter, createPipeline, - DummyDbAdapter, - DummyFullTextAdapter, type DbAdapterFactory, type DbConfiguration, type Middleware, @@ -57,14 +55,8 @@ import { type StorageAdapter, type StorageConfiguration } from '@hcengineering/server-core' -import { - createRekoniAdapter, - createYDocAdapter, - FullTextMiddleware, - type FulltextDBConfiguration -} from '@hcengineering/server-indexer' import { buildStorageFromConfig, createStorageDataAdapter, storageConfigFromEnv } from '@hcengineering/server-storage' -import { createIndexStages } from './indexing' +import { generateToken } from '@hcengineering/server-token' /** * @public @@ -87,7 +79,7 @@ export function getTxAdapterFactory ( }, extensions?: Partial ): DbAdapterFactory { - const conf = getConfig(metrics, dbUrl, workspace, branding, metrics, opt, extensions) + const conf = getConfig(metrics, dbUrl, metrics, opt, extensions) const adapterName = conf.domains[DOMAIN_TX] ?? conf.defaultAdapter const adapter = conf.adapters[adapterName] return adapter.factory @@ -102,22 +94,19 @@ export function createServerPipeline ( dbUrl: string, model: Tx[], opt: { - fullTextUrl: string - rekoniUrl: string - indexProcessing: number // 1000 - indexParallel: number // 2 + fulltextUrl?: string disableTriggers?: boolean usePassedCtx?: boolean adapterSecurity?: boolean externalStorage: StorageAdapter }, - extensions?: Partial & Partial + extensions?: Partial ): PipelineFactory { return (ctx, workspace, upgrade, broadcast, branding) => { const metricsCtx = opt.usePassedCtx === true ? ctx : metrics const wsMetrics = metricsCtx.newChild('🧲 session', {}) - const conf = getConfig(metrics, dbUrl, workspace, branding, wsMetrics, opt, extensions) + const conf = getConfig(metrics, dbUrl, wsMetrics, opt, extensions) const middlewares: MiddlewareCreator[] = [ LookupMiddleware.create, @@ -128,14 +117,16 @@ export function createServerPipeline ( SpaceSecurityMiddleware.create(opt.adapterSecurity ?? false, ctx, context, next), SpacePermissionsMiddleware.create, ConfigurationMiddleware.create, - LowLevelMiddleware.create, ContextNameMiddleware.create, ConnectionMgrMiddleware.create, MarkDerivedEntryMiddleware.create, ApplyTxMiddleware.create, // Extract apply TxMiddleware.create, // Store tx into transaction domain ...(opt.disableTriggers === true ? [] : [TriggersMiddleware.create]), - FullTextMiddleware.create(conf, upgrade), + ...(opt.fulltextUrl !== undefined + ? [FullTextMiddleware.create(opt.fulltextUrl, generateToken(systemAccountEmail, workspace))] + : []), + LowLevelMiddleware.create, QueryJoinMiddleware.create, LiveQueryMiddleware.create, DomainFindMiddleware.create, @@ -177,34 +168,10 @@ export function createBackupPipeline ( return (ctx, workspace, upgrade, broadcast, branding) => { const metricsCtx = opt.usePassedCtx === true ? ctx : metrics const wsMetrics = metricsCtx.newChild('🧲 backup', {}) - const conf = getConfig( - metrics, - dbUrl, - workspace, - branding, - wsMetrics, - { - ...opt, - fullTextUrl: '', - indexParallel: 0, - indexProcessing: 0, - rekoniUrl: '', - disableTriggers: true - }, - { - adapters: { - FullTextBlob: { - factory: async () => new DummyDbAdapter(), - url: '' - } - }, - fulltextAdapter: { - factory: async () => new DummyFullTextAdapter(), - stages: () => [], - url: '' - } - } - ) + const conf = getConfig(metrics, dbUrl, wsMetrics, { + ...opt, + disableTriggers: true + }) const middlewares: MiddlewareCreator[] = [ LowLevelMiddleware.create, @@ -241,38 +208,11 @@ export async function getServerPipeline ( const storageAdapter = buildStorageFromConfig(storageConfig) - const pipelineFactory = createServerPipeline( - ctx, - dbUrl, - model, - { - externalStorage: storageAdapter, - fullTextUrl: 'http://localhost:9200', - indexParallel: 0, - indexProcessing: 0, - rekoniUrl: '', - usePassedCtx: true, - disableTriggers: false - }, - { - fulltextAdapter: { - factory: async () => new DummyFullTextAdapter(), - url: '', - stages: (adapter, storage, storageAdapter, contentAdapter) => - createIndexStages( - ctx.newChild('stages', {}), - wsUrl, - null, - adapter, - storage, - storageAdapter, - contentAdapter, - 0, - 0 - ) - } - } - ) + const pipelineFactory = createServerPipeline(ctx, dbUrl, model, { + externalStorage: storageAdapter, + usePassedCtx: true, + disableTriggers: false + }) try { return { @@ -288,29 +228,22 @@ export async function getServerPipeline ( export function getConfig ( metrics: MeasureContext, dbUrl: string, - workspace: WorkspaceIdWithUrl, - branding: Branding | null, ctx: MeasureContext, opt: { - fullTextUrl: string - rekoniUrl: string - indexProcessing: number // 1000 - indexParallel: number // 2 disableTriggers?: boolean usePassedCtx?: boolean externalStorage: StorageAdapter }, - extensions?: Partial + extensions?: Partial ): DbConfiguration { const metricsCtx = opt.usePassedCtx === true ? ctx : metrics const wsMetrics = metricsCtx.newChild('🧲 session', {}) - const conf: DbConfiguration & FulltextDBConfiguration = { + const conf: DbConfiguration = { domains: { [DOMAIN_TX]: 'Tx', [DOMAIN_TRANSIENT]: 'InMemory', [DOMAIN_BLOB]: 'StorageData', - [DOMAIN_FULLTEXT_BLOB]: 'FullTextBlob', [DOMAIN_MODEL]: 'Null', [DOMAIN_BENCHMARK]: 'Benchmark', ...extensions?.domains @@ -338,47 +271,13 @@ export function getConfig ( factory: createStorageDataAdapter, url: '' }, - FullTextBlob: { - factory: createElasticBackupDataAdapter, - url: opt.fullTextUrl - }, Benchmark: { factory: createBenchmarkAdapter, url: '' }, ...extensions?.adapters }, - fulltextAdapter: extensions?.fulltextAdapter ?? { - factory: createElasticAdapter, - url: opt.fullTextUrl, - stages: (adapter, storage, storageAdapter, contentAdapter) => - createIndexStages( - wsMetrics.newChild('stages', {}), - workspace, - branding, - adapter, - storage, - storageAdapter, - contentAdapter, - opt.indexParallel, - opt.indexProcessing - ) - }, - serviceAdapters: extensions?.serviceAdapters ?? {}, - contentAdapters: { - Rekoni: { - factory: createRekoniAdapter, - contentType: '*', - url: opt.rekoniUrl - }, - YDoc: { - factory: createYDocAdapter, - contentType: 'application/ydoc', - url: '' - }, - ...extensions?.contentAdapters - }, - defaultContentAdapter: extensions?.defaultContentAdapter ?? 'Rekoni' + serviceAdapters: extensions?.serviceAdapters ?? {} } return conf } diff --git a/server/server-pipeline/src/serverPlugins.ts b/server/server-pipeline/src/serverPlugins.ts index 27a762e5d4..12d7ec124a 100644 --- a/server/server-pipeline/src/serverPlugins.ts +++ b/server/server-pipeline/src/serverPlugins.ts @@ -27,6 +27,7 @@ import { serverTrackerId } from '@hcengineering/server-tracker' import { serverTrainingId } from '@hcengineering/server-training' import { serverViewId } from '@hcengineering/server-view' import { serverAiBotId } from '@hcengineering/server-ai-bot' +import { serverFulltextId } from '@hcengineering/server-fulltext' export function registerServerPlugins (): void { addLocation(serverActivityId, () => import('@hcengineering/server-activity-resources')) @@ -57,4 +58,5 @@ export function registerServerPlugins (): void { addLocation(serverTrainingId, () => import('@hcengineering/server-training-resources')) addLocation(serverGithubId, () => import('@hcengineering/server-github-resources')) addLocation(serverAiBotId, () => import('@hcengineering/server-ai-bot-resources')) + addLocation(serverFulltextId, () => import('@hcengineering/server-fulltext-resources')) } diff --git a/server/server-storage/src/blobStorage.ts b/server/server-storage/src/blobStorage.ts index 209e005a7e..91f3286bff 100644 --- a/server/server-storage/src/blobStorage.ts +++ b/server/server-storage/src/blobStorage.ts @@ -81,8 +81,8 @@ class StorageBlobAdapter implements DbAdapter { return toFindResult([]) } - async groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { - return new Set() + async groupBy(ctx: MeasureContext, domain: Domain, field: string): Promise> { + return new Map() } async tx (ctx: MeasureContext, ...tx: Tx[]): Promise { @@ -135,6 +135,8 @@ export async function createStorageDataAdapter ( throw new Error('minio storage adapter require minio') } // We need to create bucket if it doesn't exist - await storage.make(ctx, workspaceId) + if (!(await storage.exists(ctx, workspaceId))) { + await storage.make(ctx, workspaceId) + } return new StorageBlobAdapter(workspaceId, storage as StorageAdapterEx, ctx) } diff --git a/server/server/src/sessionManager.ts b/server/server/src/sessionManager.ts index d088d99ce3..d4da74a28b 100644 --- a/server/server/src/sessionManager.ts +++ b/server/server/src/sessionManager.ts @@ -46,12 +46,12 @@ import { type Session, type Workspace } from '@hcengineering/server-core' -import { type Token } from '@hcengineering/server-token' +import { generateToken, type Token } from '@hcengineering/server-token' import { sendResponse } from './utils' const ticksPerSecond = 20 -const workspaceSoftShutdownTicks = 3 * ticksPerSecond +const workspaceSoftShutdownTicks = 15 * ticksPerSecond interface WorkspaceLoginInfo extends Omit { upgrade?: { @@ -112,10 +112,13 @@ class TSessionManager implements SessionManager { ) => Session, readonly timeouts: Timeouts, readonly brandingMap: BrandingMap, - readonly profiling?: { + readonly profiling: + | { start: () => void stop: () => Promise } + | undefined, + readonly accountsUrl: string ) { this.checkInterval = setInterval(() => { this.handleTick() @@ -174,15 +177,20 @@ class TSessionManager implements SessionManager { ticks = 0 handleTick (): void { + const now = Date.now() for (const [wsId, workspace] of this.workspaces.entries()) { + if (this.ticks % (60 * ticksPerSecond) === workspace.tickHash) { + // update account lastVisit every minute per every workspace.∏ + void this.getWorkspaceInfo(this.ctx, workspace.token) + } + for (const s of workspace.sessions) { - if (this.ticks % (5 * 60 * ticksPerSecond) === 0) { + if (this.ticks % (5 * 60 * ticksPerSecond) === workspace.tickHash) { s[1].session.mins5.find = s[1].session.current.find s[1].session.mins5.tx = s[1].session.current.tx s[1].session.current = { find: 0, tx: 0 } } - const now = Date.now() const lastRequestDiff = now - s[1].session.lastRequest let timeout = 60000 @@ -259,14 +267,10 @@ class TSessionManager implements SessionManager { return this.sessionFactory(token, pipeline, workspaceId, branding) } - async getWorkspaceInfo ( - ctx: MeasureContext, - accounts: string, - token: string - ): Promise { + async getWorkspaceInfo (ctx: MeasureContext, token: string): Promise { try { const userInfo = await ( - await fetch(accounts, { + await fetch(this.accountsUrl, { method: 'POST', headers: { Authorization: 'Bearer ' + token, @@ -291,7 +295,7 @@ class TSessionManager implements SessionManager { } } - sessionCounter = 0 + tickCounter = 0 @withContext('📲 add-session') async addSession ( @@ -300,8 +304,7 @@ class TSessionManager implements SessionManager { token: Token, rawToken: string, pipelineFactory: PipelineFactory, - sessionId: string | undefined, - accountsUrl: string + sessionId: string | undefined ): Promise< | { session: Session, context: MeasureContext, workspaceId: string } | { upgrade: true, upgradeInfo?: WorkspaceLoginInfo['upgrade'] } @@ -311,8 +314,7 @@ class TSessionManager implements SessionManager { let workspaceInfo: WorkspaceLoginInfo | undefined try { - workspaceInfo = - accountsUrl !== '' ? await this.getWorkspaceInfo(ctx, accountsUrl, rawToken) : this.wsFromToken(token) + workspaceInfo = await this.getWorkspaceInfo(ctx, rawToken) } catch (err: any) { this.updateConnectErrorInfo(token) return { error: err } @@ -448,8 +450,8 @@ class TSessionManager implements SessionManager { session.sessionInstanceId = generateId() this.sessions.set(ws.id, { session, socket: ws }) // We need to delete previous session with Id if found. - this.sessionCounter++ - workspace.sessions.set(session.sessionId, { session, socket: ws, tickHash: this.sessionCounter % ticksPerSecond }) + this.tickCounter++ + workspace.sessions.set(session.sessionId, { session, socket: ws, tickHash: this.tickCounter % ticksPerSecond }) // Mark workspace as init completed and we had at least one client. if (!workspace.workspaceInitCompleted) { @@ -629,7 +631,9 @@ class TSessionManager implements SessionManager { workspaceId: token.workspace, workspaceName, branding, - workspaceInitCompleted: false + workspaceInitCompleted: false, + tickHash: this.tickCounter % ticksPerSecond, + token: generateToken(systemAccountEmail, token.workspace) } this.workspaces.set(toWorkspaceString(token.workspace), workspace) return workspace @@ -1028,12 +1032,15 @@ export function createSessionManager ( ) => Session, brandingMap: BrandingMap, timeouts: Timeouts, - profiling?: { + profiling: + | { start: () => void stop: () => Promise } + | undefined, + accountsUrl: string ): SessionManager { - return new TSessionManager(ctx, sessionFactory, timeouts, brandingMap ?? null, profiling) + return new TSessionManager(ctx, sessionFactory, timeouts, brandingMap ?? null, profiling, accountsUrl) } /** @@ -1069,7 +1076,8 @@ export function startSessionManager ( pingTimeout: opt.pingTimeout ?? 10000, reconnectTimeout: 500 }, - opt.profiling + opt.profiling, + opt.accountsUrl ) return { shutdown: opt.serverFactory( diff --git a/server/server/src/starter.ts b/server/server/src/starter.ts index 9d733ca224..242fbec366 100644 --- a/server/server/src/starter.ts +++ b/server/server/src/starter.ts @@ -1,16 +1,14 @@ export interface ServerEnv { dbUrl: string mongoUrl?: string - elasticUrl: string + fulltextUrl: string serverSecret: string - rekoniUrl: string frontUrl: string filesUrl: string | undefined sesUrl: string | undefined accountsUrl: string serverPort: number enableCompression: boolean - elasticIndexName: string pushPublicKey: string | undefined pushPrivateKey: string | undefined pushSubject: string | undefined @@ -29,14 +27,9 @@ export function serverConfigFromEnv (): ServerEnv { const mongoUrl = process.env.MONGO_URL - const elasticUrl = process.env.ELASTIC_URL - if (elasticUrl === undefined) { - console.error('please provide elastic url') - process.exit(1) - } - const elasticIndexName = process.env.ELASTIC_INDEX_NAME - if (elasticIndexName === undefined) { - console.log('Please provide ELASTIC_INDEX_NAME') + const fulltextUrl = process.env.FULLTEXT_URL + if (fulltextUrl === undefined) { + console.error('please provide Fulltext URL') process.exit(1) } @@ -46,12 +39,6 @@ export function serverConfigFromEnv (): ServerEnv { process.exit(1) } - const rekoniUrl = process.env.REKONI_URL - if (rekoniUrl === undefined) { - console.log('Please provide REKONI_URL url') - process.exit(1) - } - const frontUrl = process.env.FRONT_URL if (frontUrl === undefined) { console.log('Please provide FRONT_URL url') @@ -75,10 +62,8 @@ export function serverConfigFromEnv (): ServerEnv { return { dbUrl, mongoUrl, - elasticUrl, - elasticIndexName, + fulltextUrl, serverSecret, - rekoniUrl, frontUrl, filesUrl, sesUrl, diff --git a/server/ws/src/__tests__/server.test.ts b/server/ws/src/__tests__/server.test.ts index 087f4a1b86..ce07c04c7a 100644 --- a/server/ws/src/__tests__/server.test.ts +++ b/server/ws/src/__tests__/server.test.ts @@ -80,7 +80,7 @@ describe('server', () => { ], close: async () => {}, domains: async () => [], - groupBy: async () => new Set(), + groupBy: async () => new Map(), find: (ctx: MeasureContext, domain: Domain) => ({ next: async (ctx: MeasureContext) => undefined, close: async (ctx: MeasureContext) => {} @@ -190,7 +190,7 @@ describe('server', () => { [], undefined ], - groupBy: async () => new Set(), + groupBy: async () => new Map(), close: async () => {}, domains: async () => [], find: (ctx: MeasureContext, domain: Domain) => ({ diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index 3fa0b4dcbd..1073222f01 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -14,7 +14,7 @@ // import { Analytics } from '@hcengineering/analytics' -import { generateId, toWorkspaceString, type MeasureContext } from '@hcengineering/core' +import { generateId, toWorkspaceString, type MeasureContext, type Tx } from '@hcengineering/core' import { UNAUTHORIZED, unknownStatus } from '@hcengineering/platform' import { RPCHandler, type Response } from '@hcengineering/rpc' import { @@ -31,8 +31,8 @@ import { LOGGING_ENABLED, type ConnectionSocket, type HandleRequestFunction, - type SessionManager, type PipelineFactory, + type SessionManager, type StorageAdapter } from '@hcengineering/server-core' import { decodeToken, type Token } from '@hcengineering/server-token' @@ -288,6 +288,37 @@ export function startHttpServer ( } }) + app.put('/api/v1/broadcast', (req, res) => { + try { + const token = req.query.token as string + decodeToken(token) + const ws = sessions.workspaces.get(req.query.workspace as string) + if (ws !== undefined) { + // push the data to body + const body: Buffer[] = [] + req + .on('data', (chunk) => { + body.push(chunk) + }) + .on('end', () => { + // on end of data, perform necessary action + const data = JSON.parse(Buffer.concat(body as any).toString()) + if (Array.isArray(data)) { + sessions.broadcastAll(ws, data as Tx[]) + } else { + sessions.broadcastAll(ws, [data as unknown as Tx]) + } + res.end() + }) + } + } catch (err: any) { + Analytics.handleError(err) + ctx.error('error', { err }) + res.writeHead(404, {}) + res.end() + } + }) + const httpServer = http.createServer(app) const wss = new WebSocketServer({ noServer: true, @@ -338,7 +369,7 @@ export function startHttpServer ( connectionSocket: cs, payload: token, token: rawToken, - session: sessions.addSession(ctx, cs, token, rawToken, pipelineFactory, sessionId, accountsUrl), + session: sessions.addSession(ctx, cs, token, rawToken, pipelineFactory, sessionId), url: '' } @@ -375,7 +406,7 @@ export function startHttpServer ( if (msg instanceof Buffer) { buff = msg } else if (Array.isArray(msg)) { - buff = Buffer.concat(msg) + buff = Buffer.concat(msg as any) } if (buff !== undefined) { doSessionOp( diff --git a/services/ai-bot/love-agent/package.json b/services/ai-bot/love-agent/package.json index 61a50862a0..8e681df53e 100644 --- a/services/ai-bot/love-agent/package.json +++ b/services/ai-bot/love-agent/package.json @@ -19,11 +19,6 @@ "lint:fix": "eslint --fix src/**/*.ts", "format": "prettier --write src/**/*.ts && pnpm lint:fix" }, - "pnpm": { - "overrides": { - "livekit-server-sdk": "2.7.3" - } - }, "devDependencies": { "@types/node": "~20.11.16", "@typescript-eslint/eslint-plugin": "^6.11.0", @@ -40,7 +35,7 @@ }, "dependencies": { "@deepgram/sdk": "^3.9.0", - "@livekit/agents": "^0.3.5", + "@livekit/agents": "^0.4.1", "@livekit/rtc-node": "^0.11.1", "dotenv": "^16.4.5" } diff --git a/services/ai-bot/love-agent/pnpm-lock.yaml b/services/ai-bot/love-agent/pnpm-lock.yaml index f6d11bcaa8..26f981af21 100644 --- a/services/ai-bot/love-agent/pnpm-lock.yaml +++ b/services/ai-bot/love-agent/pnpm-lock.yaml @@ -4,9 +4,6 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false -overrides: - livekit-server-sdk: 2.7.3 - importers: .: @@ -15,8 +12,8 @@ importers: specifier: ^3.9.0 version: 3.9.0 '@livekit/agents': - specifier: ^0.3.5 - version: 0.3.5 + specifier: ^0.4.1 + version: 0.4.1 '@livekit/rtc-node': specifier: ^0.11.1 version: 0.11.1 @@ -246,8 +243,8 @@ packages: resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} deprecated: Use @eslint/object-schema instead - '@livekit/agents@0.3.5': - resolution: {integrity: sha512-qjEIRkr/HdOvEOnvLKZMfnQ472bShQ1Ai2KKng8a1caSDHiLhQBeSb1tzA1Evsmm1k8hilXqh+81/SDj4cfiDA==} + '@livekit/agents@0.4.1': + resolution: {integrity: sha512-zZnd19CWvm1i6PKzAgUw6gLdZOJ/QKzbIVSLNAZPQphCEEg3cPoe3fEf9XE7o9+n6e5sZ6FfmSUh/c7jxWmujw==} '@livekit/mutex@1.1.0': resolution: {integrity: sha512-XRLG+z/0uoyDioupjUiskjI06Y51U/IXVPJn7qJ+R3J75XX01irYVBM9MpxeJahpVoe9QhU4moIEolX+HO9U9g==} @@ -1021,8 +1018,8 @@ packages: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} - livekit-server-sdk@2.7.3: - resolution: {integrity: sha512-dBiyMJ2o3Adw7aBVuFxVOlYHmiZtGGS9zVksMuv/wiEVHY+6XSDzo0X67pZVkyGlq1moF4YZAReVY2Dbxve8NQ==} + livekit-server-sdk@2.8.1: + resolution: {integrity: sha512-l8egXU10jPuRJM2Df9Gk/KPEk6tBV0JEGG19cD5QeQtyIMgqULCCd/5yyG2FRvcWRf7pEyZZMXi63zDn7uaKHQ==} engines: {node: '>=19'} locate-path@6.0.0: @@ -1069,8 +1066,8 @@ packages: encoding: optional: true - object-inspect@1.13.2: - resolution: {integrity: sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==} + object-inspect@1.13.3: + resolution: {integrity: sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA==} engines: {node: '>= 0.4'} object-keys@1.1.1: @@ -1559,13 +1556,14 @@ snapshots: '@humanwhocodes/object-schema@2.0.3': {} - '@livekit/agents@0.3.5': + '@livekit/agents@0.4.1': dependencies: '@livekit/mutex': 1.1.0 '@livekit/protocol': 1.27.1 '@livekit/rtc-node': 0.11.1 + '@livekit/typed-emitter': 3.0.0 commander: 12.1.0 - livekit-server-sdk: 2.7.3 + livekit-server-sdk: 2.8.1 pino: 8.21.0 pino-pretty: 11.3.0 ws: 8.18.0 @@ -1977,7 +1975,7 @@ snapshots: is-string: 1.0.7 is-typed-array: 1.1.13 is-weakref: 1.0.2 - object-inspect: 1.13.2 + object-inspect: 1.13.3 object-keys: 1.1.1 object.assign: 4.1.5 regexp.prototype.flags: 1.5.3 @@ -2488,7 +2486,7 @@ snapshots: prelude-ls: 1.2.1 type-check: 0.4.0 - livekit-server-sdk@2.7.3: + livekit-server-sdk@2.8.1: dependencies: '@livekit/protocol': 1.27.1 camelcase-keys: 9.1.3 @@ -2527,7 +2525,7 @@ snapshots: dependencies: whatwg-url: 5.0.0 - object-inspect@1.13.2: {} + object-inspect@1.13.3: {} object-keys@1.1.1: {} @@ -2748,7 +2746,7 @@ snapshots: call-bind: 1.0.7 es-errors: 1.3.0 get-intrinsic: 1.2.4 - object-inspect: 1.13.2 + object-inspect: 1.13.3 slash@3.0.0: {} diff --git a/services/ai-bot/love-agent/src/agent.ts b/services/ai-bot/love-agent/src/agent.ts index 8493e7aa4b..e442c2bddb 100644 --- a/services/ai-bot/love-agent/src/agent.ts +++ b/services/ai-bot/love-agent/src/agent.ts @@ -13,12 +13,13 @@ // limitations under the License. // -import { cli, defineAgent, type JobContext, WorkerOptions, WorkerPermissions } from '@livekit/agents' +import { cli, defineAgent, type JobContext, JobRequest, WorkerOptions } from '@livekit/agents' import { fileURLToPath } from 'node:url' import { RemoteParticipant, RemoteTrack, RemoteTrackPublication, RoomEvent, TrackKind } from '@livekit/rtc-node' import { STT } from './stt.js' import { Metadata, TranscriptionStatus } from './type.js' +import config from './config.js' function parseMetadata (metadata: string): Metadata { try { @@ -30,6 +31,45 @@ function parseMetadata (metadata: string): Metadata { return {} } +async function requestIdentity (roomName: string): Promise<{ identity: string, name: string } | undefined> { + try { + const res = await fetch(`${config.PlatformUrl}/love/${roomName}/identity`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + config.PlatformToken + } + }) + + if (!res.ok) { + return undefined + } + return await res.json() + } catch (e) { + console.error('Error during request identity', e) + } +} + +const requestFunc = async (req: JobRequest): Promise => { + const roomName = req.room?.name + + if (roomName == null) { + console.error('Room name is undefined', { room: req.room }) + await req.reject() + return + } + + const identity = await requestIdentity(roomName) + + if (identity?.identity == null) { + console.error('No ai identity', { roomName }) + await req.reject() + return + } + + await req.accept(identity.name, identity.identity) +} + function applyMetadata (data: string | undefined, stt: STT): void { if (data == null || data === '') return const metadata = parseMetadata(data) @@ -39,11 +79,13 @@ function applyMetadata (data: string | undefined, stt: STT): void { } if (metadata.transcription === TranscriptionStatus.InProgress) { + console.log('Starting transcription', stt.name) stt.start() } else if ( metadata.transcription === TranscriptionStatus.Completed || metadata.transcription === TranscriptionStatus.Idle ) { + console.log('Stopping transcription', stt.name) stt.stop() } } @@ -109,7 +151,7 @@ export function runAgent (): void { cli.runApp( new WorkerOptions({ agent: fileURLToPath(import.meta.url), - permissions: new WorkerPermissions(true, true, true, true, [], true) + requestFunc }) ) } diff --git a/services/ai-bot/love-agent/src/stt.ts b/services/ai-bot/love-agent/src/stt.ts index 103f5204a0..02ee5aae86 100644 --- a/services/ai-bot/love-agent/src/stt.ts +++ b/services/ai-bot/love-agent/src/stt.ts @@ -55,25 +55,8 @@ export class STT { private readonly dgConnectionBySid = new Map() private readonly intervalBySid = new Map() - private readonly transcriptsBySid = new Map() - - private readonly interval: NodeJS.Timeout - - constructor (private readonly name: string) { + constructor (readonly name: string) { this.deepgram = createClient(config.DeepgramApiKey) - this.interval = this.interval = setInterval(() => { - this.sendTranscriptToPlatform() - }, config.TranscriptDelay) - } - - sendTranscriptToPlatform (): void { - const now = Date.now() - for (const [sid, transcript] of this.transcriptsBySid.entries()) { - if (now - transcript.startedOn > config.TranscriptDelay) { - void this.sendToPlatform(transcript.value, sid) - this.transcriptsBySid.delete(sid) - } - } } updateLanguage (language: string): void { @@ -175,32 +158,16 @@ export class STT { dgConnection.on(LiveTranscriptionEvents.Open, () => { dgConnection.on(LiveTranscriptionEvents.Transcript, (data: LiveTranscriptionEvent) => { const transcript = data?.channel?.alternatives[0].transcript - if (transcript != null && transcript !== '') { - const prevData = this.transcriptsBySid.get(sid) - const prevValue = prevData?.value ?? '' - if (data.is_final === true) { - // TODO: how to join the final transcript ? - this.transcriptsBySid.set(sid, { - value: prevValue + ' ' + transcript, - startedOn: prevData?.startedOn ?? Date.now() - }) - } - if (data.speech_final === true) { - const result = this.transcriptsBySid.get(sid)?.value - if (result != null) { - void this.sendToPlatform(result, sid) - } - this.transcriptsBySid.delete(sid) - } + const hasTranscript = transcript != null && transcript !== '' + + if (!hasTranscript) { + return } - }) - dgConnection.addListener(LiveTranscriptionEvents.UtteranceEnd, () => { - const result = this.transcriptsBySid.get(sid)?.value ?? '' - if (result.length > 0) { - void this.sendToPlatform(result, sid) - - this.transcriptsBySid.delete(sid) + if (data.speech_final === true) { + void this.sendToPlatform(transcript, sid, true) + } else if (data.is_final === true) { + void this.sendToPlatform(transcript, sid, false) } }) @@ -232,11 +199,12 @@ export class STT { } } - async sendToPlatform (transcript: string, sid: string): Promise { + async sendToPlatform (transcript: string, sid: string, isFinal = false): Promise { const request = { transcript, participant: this.participantBySid.get(sid)?.identity, - roomName: this.name + roomName: this.name, + final: isFinal } try { @@ -254,10 +222,9 @@ export class STT { } close (): void { - clearInterval(this.interval) - for (const sid of this.transcriptsBySid.keys()) { - this.trackBySid.delete(sid) - this.participantBySid.delete(sid) + this.trackBySid.clear() + this.participantBySid.clear() + for (const sid of this.dgConnectionBySid.keys()) { this.stopDeepgram(sid) } } diff --git a/services/ai-bot/pod-ai-bot/src/controller.ts b/services/ai-bot/pod-ai-bot/src/controller.ts index 584ef7cb1b..fdc7b74905 100644 --- a/services/ai-bot/pod-ai-bot/src/controller.ts +++ b/services/ai-bot/pod-ai-bot/src/controller.ts @@ -21,6 +21,7 @@ import { AITransferEventRequest, ConnectMeetingRequest, DisconnectMeetingRequest, + IdentityResponse, OnboardingEvent, OnboardingEventRequest, OpenChatInSidebarData, @@ -282,17 +283,28 @@ export class AIControl { await wsClient.loveDisconnect(request) } - async processLoveTranscript (request: PostTranscriptRequest): Promise { - const parsed = request.roomName.split('_') - - if (parsed.length < 3) return - + async getLoveIdentity (roomName: string): Promise { + const parsed = roomName.split('_') const workspace = parsed[0] - const roomId = parsed[parsed.length - 1] + + if (workspace === null) return const wsClient = await this.getWorkspaceClient(workspace) if (wsClient === undefined) return - await wsClient.processLoveTranscript(request.transcript, request.participant, roomId as Ref) + return await wsClient.getLoveIdentity() + } + + async processLoveTranscript (request: PostTranscriptRequest): Promise { + const parsed = request.roomName.split('_') + const workspace = parsed[0] + const roomId = parsed[parsed.length - 1] + + if (workspace === null || roomId === null) return + + const wsClient = await this.getWorkspaceClient(workspace) + if (wsClient === undefined) return + + await wsClient.processLoveTranscript(request.transcript, request.participant, roomId as Ref, request.final) } } diff --git a/services/ai-bot/pod-ai-bot/src/server/server.ts b/services/ai-bot/pod-ai-bot/src/server/server.ts index 9a385d84a5..9cbda42522 100644 --- a/services/ai-bot/pod-ai-bot/src/server/server.ts +++ b/services/ai-bot/pod-ai-bot/src/server/server.ts @@ -23,7 +23,8 @@ import { AIEventRequest, ConnectMeetingRequest, DisconnectMeetingRequest, - PostTranscriptRequest + PostTranscriptRequest, + aiBotAccountEmail } from '@hcengineering/ai-bot' import { extractToken } from '@hcengineering/server-client' @@ -96,13 +97,18 @@ export function createServer (controller: AIControl): Express { await controller.processEvent(token.workspace.name, events as AIEventRequest[]) }) ) + app.post( '/love/transcript', - wrapRequest(async (req, res) => { + wrapRequest(async (req, res, token) => { if (req.body == null || Array.isArray(req.body) || typeof req.body !== 'object') { throw new ApiError(400) } + if (token.email !== aiBotAccountEmail) { + throw new ApiError(401) + } + await controller.processLoveTranscript(req.body as PostTranscriptRequest) res.status(200) @@ -140,6 +146,25 @@ export function createServer (controller: AIControl): Express { }) ) + app.get( + '/love/:roomName/identity', + wrapRequest(async (req, res, token) => { + if (token.email !== aiBotAccountEmail) { + throw new ApiError(401) + } + + const roomName = req.params.roomName + const resp = await controller.getLoveIdentity(roomName) + + if (resp === undefined) { + throw new ApiError(404) + } + + res.status(200) + res.json(resp) + }) + ) + app.post( '/onboarding', wrapRequest(async (req, res) => { diff --git a/services/ai-bot/pod-ai-bot/src/workspace/love.ts b/services/ai-bot/pod-ai-bot/src/workspace/love.ts index a10b4f0b15..5ff30aaac6 100644 --- a/services/ai-bot/pod-ai-bot/src/workspace/love.ts +++ b/services/ai-bot/pod-ai-bot/src/workspace/love.ts @@ -9,7 +9,8 @@ import core, { TxCreateDoc, TxUpdateDoc, MeasureContext, - Markup + Markup, + generateId } from '@hcengineering/core' import { Person } from '@hcengineering/contact' import love, { @@ -21,11 +22,27 @@ import love, { TranscriptionStatus } from '@hcengineering/love' import { ConnectMeetingRequest } from '@hcengineering/ai-bot' -import chunter from '@hcengineering/chunter' +import chunter, { ChatMessage } from '@hcengineering/chunter' import { jsonToMarkup, MarkupNodeType } from '@hcengineering/text' import config from '../config' +class Transcriptions { + private readonly transcriptionByPerson = new Map, { _id: Ref, text: string }>() + + get (person: Ref): { _id: Ref, text: string } | undefined { + return this.transcriptionByPerson.get(person) + } + + set (person: Ref, value: { _id: Ref, text: string }): void { + this.transcriptionByPerson.set(person, value) + } + + delete (person: Ref): void { + this.transcriptionByPerson.delete(person) + } +} + export class LoveController { private readonly roomSidById = new Map, string>() private readonly connectedRooms = new Set>() @@ -33,6 +50,7 @@ export class LoveController { private participantsInfo: ParticipantInfo[] = [] private rooms: Room[] = [] private readonly meetingMinutes: MeetingMinutes[] = [] + private readonly activeTranscriptions = new Map, Transcriptions>() constructor ( private readonly workspace: string, @@ -47,6 +65,13 @@ export class LoveController { }, 5000) } + getIdentity (): { identity: Ref, name: string } { + return { + identity: this.currentPerson._id, + name: this.currentPerson.name + } + } + txHandler (txes: Tx[]): void { const hierarchy = this.client.getHierarchy() for (const tx of txes) { @@ -141,6 +166,8 @@ export class LoveController { async disconnect (roomId: Ref): Promise { this.ctx.info('Disconnecting', { roomId }) + this.activeTranscriptions.delete(roomId) + const participant = await this.getRoomParticipant(roomId, this.currentPerson._id) if (participant !== undefined) { await this.client.remove(participant) @@ -156,7 +183,7 @@ export class LoveController { this.connectedRooms.delete(roomId) } - async processTranscript (text: string, person: Ref, roomId: Ref): Promise { + async processTranscript (text: string, person: Ref, roomId: Ref, final: boolean): Promise { const room = await this.getRoom(roomId) const participant = await this.getRoomParticipant(roomId, person) @@ -168,19 +195,40 @@ export class LoveController { const personAccount = this.client.getModel().getAccountByPersonId(participant.person)[0] if (doc === undefined) return - await this.client.addCollection( - chunter.class.ChatMessage, - core.space.Workspace, - doc._id, - doc._class, - 'transcription', - { - message: this.transcriptToMarkup(text) - }, - undefined, - undefined, - personAccount._id - ) + const transcriptions = this.activeTranscriptions.get(roomId) ?? new Transcriptions() + const activeTranscription = transcriptions.get(participant.person) + + if (activeTranscription === undefined) { + const _id = generateId() + if (!final) { + transcriptions.set(participant.person, { _id, text }) + this.activeTranscriptions.set(roomId, transcriptions) + } + + await this.client.addCollection( + chunter.class.ChatMessage, + core.space.Workspace, + doc._id, + doc._class, + 'transcription', + { + message: this.transcriptToMarkup(text) + }, + _id, + undefined, + personAccount._id + ) + } else { + const mergedText = activeTranscription.text + ' ' + text + if (!final) { + transcriptions.set(participant.person, { _id: activeTranscription._id, text: mergedText }) + } else { + transcriptions.delete(participant.person) + } + await this.client.updateDoc(chunter.class.ChatMessage, core.space.Workspace, activeTranscription._id, { + message: this.transcriptToMarkup(mergedText) + }) + } } hasActiveConnections (): boolean { diff --git a/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts b/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts index 60e1dc8582..7498f327d7 100644 --- a/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts +++ b/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts @@ -18,7 +18,8 @@ import aiBot, { AIMessageEventRequest, AITransferEventRequest, ConnectMeetingRequest, - DisconnectMeetingRequest + DisconnectMeetingRequest, + IdentityResponse } from '@hcengineering/ai-bot' import chunter, { ChatMessage, @@ -710,7 +711,7 @@ export class WorkspaceClient { await this.love.disconnect(request.roomId) } - async processLoveTranscript (text: string, participant: Ref, room: Ref): Promise { + async processLoveTranscript (text: string, participant: Ref, room: Ref, final: boolean): Promise { // Just wait initialization await this.opClient @@ -719,7 +720,16 @@ export class WorkspaceClient { return } - await this.love.processTranscript(text, participant, room) + await this.love.processTranscript(text, participant, room, final) + } + + async getLoveIdentity (): Promise { + // Just wait initialization + await this.opClient + + if (this.love === undefined) return + + return this.love.getIdentity() } canClose (): boolean { diff --git a/services/rekoni/src/server.ts b/services/rekoni/src/server.ts index f2cc5339ea..15f17c2e13 100644 --- a/services/rekoni/src/server.ts +++ b/services/rekoni/src/server.ts @@ -123,7 +123,7 @@ export const startServer = async (): Promise => { temp.push(d) }) bodyStream.on('end', function () { - resolve(Buffer.concat(temp)) + resolve(Buffer.concat(temp as any)) }) }) diff --git a/tests/docker-compose.override.yaml b/tests/docker-compose.override.yaml index 2fde3c58ee..2a4c26930e 100644 --- a/tests/docker-compose.override.yaml +++ b/tests/docker-compose.override.yaml @@ -7,4 +7,7 @@ services: - DB_URL=postgresql://postgres:example@postgres:5432 workspace: environment: - - DB_URL=postgresql://postgres:example@postgres:5432 \ No newline at end of file + - DB_URL=postgresql://postgres:example@postgres:5432 + fulltext: + environment: + - DB_URL=postgresql://postgres:example@postgres:5432 \ No newline at end of file diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index be43bfc3c8..57800f73ca 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -117,20 +117,17 @@ services: environment: - SERVER_PROVIDER=${SERVER_PROVIDER} - SERVER_PORT=3334 - - SERVER_SECRET=secret - - ELASTIC_URL=http://elastic:9200 + - SERVER_SECRET=secret - DB_URL=mongodb://mongodb:27018 - - MONGO_URL=mongodb://mongodb:27018 - METRICS_CONSOLE=false - METRICS_FILE=metrics.txt - STORAGE_CONFIG=${STORAGE_CONFIG} - - REKONI_URL=http://rekoni:7 - FRONT_URL=http://localhost:8083 - ACCOUNTS_URL=http://account:3003 - COLLABORATOR_URL=http://collaborator:3079 - LAST_NAME_FIRST=true - - ELASTIC_INDEX_NAME=local_storage_index - BRANDING_PATH=/var/cfg/branding.json + - FULLTEXT_URL=http://fulltext:4710 collaborator: image: hardcoreeng/collaborator links: @@ -144,6 +141,7 @@ services: - SECRET=secret - ACCOUNTS_URL=http://account:3003 - STORAGE_CONFIG=${STORAGE_CONFIG} + - FULLTEXT_URL=http://fulltext:4710 restart: unless-stopped rekoni: image: hardcoreeng/rekoni-service @@ -153,4 +151,25 @@ services: deploy: resources: limits: - memory: 1024M \ No newline at end of file + memory: 1024M + fulltext: + image: hardcoreeng/fulltext + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped + links: + - elastic + - mongodb + - postgres + - rekoni + ports: + - 4710:4710 + environment: + - PORT=4710 + - SERVER_SECRET=secret + - DB_URL=mongodb://mongodb:27018 + - FULLTEXT_DB_URL=http://elastic:9200 + - ELASTIC_INDEX_NAME=local_storage_index + - STORAGE_CONFIG=${STORAGE_CONFIG} + - REKONI_URL=http://rekoni:4007 + - ACCOUNTS_URL=http://account:3003 \ No newline at end of file diff --git a/tests/sanity/package.json b/tests/sanity/package.json index fd1e287b57..788775c11b 100644 --- a/tests/sanity/package.json +++ b/tests/sanity/package.json @@ -21,7 +21,7 @@ "test": "", "uitest": "cross-env LOCAL_URL=http://localhost:3003/ DEV_URL= playwright test -c ./tests/playwright.config.ts", "staging-uitest": "cross-env PLATFORM_URI=https://front.hc.engineering/ playwright test -c ./tests/playwright.config.ts --grep @staging", - "dev-uitest": "cross-env PLATFORM_URI=http://localhost:8080 PLATFORM_TRANSACTOR=ws://localhost:3333 SETTING=storage-dev.json SETTING_SECOND=storageSecond-dev.json DEV_URL=http://localhost:8080/account playwright test -c ./tests/playwright.config.ts", + "dev-uitest": "cross-env PLATFORM_URI=http://localhost:8080 PLATFORM_TRANSACTOR=ws://localhost:3333 SETTING=storage-dev.json SETTING_SECOND=storageSecond-dev.json DEV_URL=http://localhost:8080/account playwright test -c ./tests/playwright.config.ts", "debug": "cross-env LOCAL_URL=http://localhost:3003/ DEV_URL= playwright test -c ./tests/playwright.config.ts --debug --headed", "dev-debug": "cross-env PLATFORM_URI=http://localhost:8080 PLATFORM_TRANSACTOR=ws://localhost:3333 SETTING=storage-dev.json SETTING_SECOND=storageSecond-dev.json playwright test -c ./tests/playwright.config.ts --debug --headed", "codegen": "playwright codegen --load-storage storage.json http://localhost:8083/workbench/sanity-ws/", diff --git a/tests/sanity/tests/chat/dynamic-issues-chats.spec.ts b/tests/sanity/tests/chat/dynamic-issues-chats.spec.ts index b6bb17e58d..50f94e0e5f 100644 --- a/tests/sanity/tests/chat/dynamic-issues-chats.spec.ts +++ b/tests/sanity/tests/chat/dynamic-issues-chats.spec.ts @@ -56,7 +56,7 @@ test.describe('Dynamic issues chats', () => { } await test.step('User 1 creates Issue for himself', async () => { - await prepareNewIssueWithOpenStep(page, newIssue) + await prepareNewIssueWithOpenStep(page, newIssue, false) }) await test.step('User 1 has issue and linked chat', async () => { @@ -84,7 +84,7 @@ test.describe('Dynamic issues chats', () => { } await test.step('User 1 creates an issue and assign it to User 2', async () => { - await prepareNewIssueWithOpenStep(page, newIssue) + await prepareNewIssueWithOpenStep(page, newIssue, false) }) await test.step('User 2 has issue and linked chat', async () => { diff --git a/tests/sanity/tests/documents/documents-link.spec.ts b/tests/sanity/tests/documents/documents-link.spec.ts index a9622fedc4..97c8d559d7 100644 --- a/tests/sanity/tests/documents/documents-link.spec.ts +++ b/tests/sanity/tests/documents/documents-link.spec.ts @@ -4,7 +4,7 @@ import { DocumentsPage } from '../model/documents/documents-page' import { NewDocument } from '../model/documents/types' import { LeftSideMenuPage } from '../model/left-side-menu-page' import { PublicLinkPopup } from '../model/tracker/public-link-popup' -import { generateId, PlatformSetting, PlatformURI } from '../utils' +import { generateId, PlatformSetting, PlatformURI, setTestOptions } from '../utils' test.describe('Documents link tests', () => { test('Document public link revoke', async ({ browser }) => { @@ -17,6 +17,7 @@ test.describe('Documents link tests', () => { const page = await newContext.newPage() await page.goto(`${PlatformURI}/workbench/sanity-ws`) + await setTestOptions(page) const leftSideMenuPage = new LeftSideMenuPage(page) await leftSideMenuPage.clickDocuments() diff --git a/tests/sanity/tests/documents/documents.spec.ts b/tests/sanity/tests/documents/documents.spec.ts index 46e45714a6..37ed9b4d03 100644 --- a/tests/sanity/tests/documents/documents.spec.ts +++ b/tests/sanity/tests/documents/documents.spec.ts @@ -1,10 +1,12 @@ -import { test } from '@playwright/test' +import { test, expect } from '@playwright/test' import { generateId, getSecondPage, PlatformSetting, PlatformURI } from '../utils' import { NewDocument, NewTeamspace } from '../model/documents/types' import { LeftSideMenuPage } from '../model/left-side-menu-page' import { DocumentsPage } from '../model/documents/documents-page' import { DocumentContentPage } from '../model/documents/document-content-page' +const retryOptions = { intervals: [100, 200, 1000], timeout: 60000 } + test.use({ storageState: PlatformSetting }) @@ -114,7 +116,11 @@ test.describe('Documents tests', () => { await test.step('Create a child document', async () => { await documentsPage.clickAddDocumentIntoDocument(parentDocument.title) + await expect(async () => { + await documentContentPage.checkDocumentTitle('Untitled') + }).toPass(retryOptions) await documentContentPage.updateDocumentTitle(childDocument.title) + const content = await documentContentPage.addContentToTheNewLine(contentFirst) await documentContentPage.checkContent(content) }) diff --git a/tests/sanity/tests/documents/teamspace.spec.ts b/tests/sanity/tests/documents/teamspace.spec.ts index e902aa8ce0..57d15f28be 100644 --- a/tests/sanity/tests/documents/teamspace.spec.ts +++ b/tests/sanity/tests/documents/teamspace.spec.ts @@ -1,20 +1,21 @@ import { test } from '@playwright/test' -import { - generateId, - PlatformSetting, - PlatformURI, - generateUser, - createAccount, - getInviteLink, - createAccountAndWorkspace, - generateTestData -} from '../utils' +import { TestData } from '../chat/types' +import { SignUpData } from '../model/common-types' +import { DocumentsPage } from '../model/documents/documents-page' import { NewTeamspace } from '../model/documents/types' import { LeftSideMenuPage } from '../model/left-side-menu-page' -import { DocumentsPage } from '../model/documents/documents-page' -import { SignUpData } from '../model/common-types' import { SignInJoinPage } from '../model/signin-page' -import { TestData } from '../chat/types' +import { + createAccount, + createAccountAndWorkspace, + generateId, + generateTestData, + generateUser, + getInviteLink, + PlatformSetting, + PlatformURI, + setTestOptions +} from '../utils' test.use({ storageState: PlatformSetting @@ -95,13 +96,17 @@ test.describe('Teamspace tests', () => { const linkText = await getInviteLink(page) const page2 = await browser.newPage() - await page2.goto(linkText ?? '') - const joinPage: SignInJoinPage = new SignInJoinPage(page2) - await joinPage.join(newUser2) - const documentsSecondPage: DocumentsPage = new DocumentsPage(page2) - await documentsSecondPage.clickDocumentsApp() - await documentsSecondPage.checkTeamspaceExist(autojoinTeamspace.title) - await page2.close() + try { + await page2.goto(linkText ?? '') + await setTestOptions(page2) + const joinPage: SignInJoinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) + const documentsSecondPage: DocumentsPage = new DocumentsPage(page2) + await documentsSecondPage.clickDocumentsApp() + await documentsSecondPage.checkTeamspaceExist(autojoinTeamspace.title) + } finally { + await page2.close() + } }) test('Join teamspace', async ({ page, request, browser }) => { @@ -120,15 +125,19 @@ test.describe('Teamspace tests', () => { const linkText = await getInviteLink(page) const page2 = await browser.newPage() - await page2.goto(linkText ?? '') - const joinPage: SignInJoinPage = new SignInJoinPage(page2) - await joinPage.join(newUser2) - const documentsSecondPage: DocumentsPage = new DocumentsPage(page2) - await documentsSecondPage.clickDocumentsApp() - await documentsSecondPage.checkTeamspaceNotExist(joinTeamspace.title) - await documentsSecondPage.clickTeamspaces() - await documentsSecondPage.joinTeamspace(joinTeamspace.title) - await documentsSecondPage.checkTeamspaceExist(joinTeamspace.title) - await page2.close() + try { + await page2.goto(linkText ?? '') + await setTestOptions(page2) + const joinPage: SignInJoinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) + const documentsSecondPage: DocumentsPage = new DocumentsPage(page2) + await documentsSecondPage.clickDocumentsApp() + await documentsSecondPage.checkTeamspaceNotExist(joinTeamspace.title) + await documentsSecondPage.clickTeamspaces() + await documentsSecondPage.joinTeamspace(joinTeamspace.title) + await documentsSecondPage.checkTeamspaceExist(joinTeamspace.title) + } finally { + await page2.close() + } }) }) diff --git a/tests/sanity/tests/inbox/inbox.spec.ts b/tests/sanity/tests/inbox/inbox.spec.ts index 9f12cf2877..aba43b1668 100644 --- a/tests/sanity/tests/inbox/inbox.spec.ts +++ b/tests/sanity/tests/inbox/inbox.spec.ts @@ -1,20 +1,20 @@ -import { test } from '@playwright/test' -import { PlatformURI, generateTestData, getTimeForPlanner, attachScreenshot } from '../utils' -import { LeftSideMenuPage } from '../model/left-side-menu-page' -import { ApiEndpoint } from '../API/Api' -import { LoginPage } from '../model/login-page' -import { createNewIssueData, prepareNewIssueWithOpenStep } from '../tracker/common-steps' -import { IssuesDetailsPage } from '../model/tracker/issues-details-page' -import { InboxPage } from '../model/inbox.ts/inbox-page' -import { SignUpData } from '../model/common-types' import { faker } from '@faker-js/faker' -import { SignInJoinPage } from '../model/signin-page' +import { test } from '@playwright/test' +import { ApiEndpoint } from '../API/Api' import { ChannelPage } from '../model/channel-page' -import { UserProfilePage } from '../model/profile/user-profile-page' -import { MenuItems, NotificationsPage } from '../model/profile/notifications-page' -import { SelectWorkspacePage } from '../model/select-workspace-page' +import { SignUpData } from '../model/common-types' +import { InboxPage } from '../model/inbox.ts/inbox-page' +import { LeftSideMenuPage } from '../model/left-side-menu-page' +import { LoginPage } from '../model/login-page' import { PlanningPage } from '../model/planning/planning-page' +import { MenuItems, NotificationsPage } from '../model/profile/notifications-page' +import { UserProfilePage } from '../model/profile/user-profile-page' +import { SelectWorkspacePage } from '../model/select-workspace-page' +import { SignInJoinPage } from '../model/signin-page' import { TeamPage } from '../model/team-page' +import { IssuesDetailsPage } from '../model/tracker/issues-details-page' +import { createNewIssueData, prepareNewIssueWithOpenStep } from '../tracker/common-steps' +import { attachScreenshot, generateTestData, getTimeForPlanner, PlatformURI, setTestOptions } from '../utils' test.describe('Inbox tests', () => { let leftSideMenuPage: LeftSideMenuPage @@ -42,6 +42,7 @@ test.describe('Inbox tests', () => { await api.createAccount(newUser2.email, newUser2.password, newUser2.firstName, newUser2.lastName) await api.createWorkspaceWithLogin(data.workspaceName, data.userName, '1234') await (await page.goto(`${PlatformURI}`))?.finished() + await setTestOptions(page) await loginPage.login(data.userName, '1234') const swp = new SelectWorkspacePage(page) await swp.selectWorkspace(data.workspaceName) @@ -50,7 +51,7 @@ test.describe('Inbox tests', () => { test('User is able to create a task, assign a himself and see it inside the inbox', async ({ page }) => { const newIssue = createNewIssueData(data.firstName, data.lastName) - await prepareNewIssueWithOpenStep(page, newIssue) + await prepareNewIssueWithOpenStep(page, newIssue, false) await issuesDetailsPage.checkIssue(newIssue) await leftSideMenuPage.clickTracker() @@ -60,7 +61,7 @@ test.describe('Inbox tests', () => { test('User is able to create a task, assign a himself and open it from inbox', async ({ page }) => { const newIssue = createNewIssueData(data.firstName, data.lastName) - await prepareNewIssueWithOpenStep(page, newIssue) + await prepareNewIssueWithOpenStep(page, newIssue, false) await issuesDetailsPage.checkIssue(newIssue) await leftSideMenuPage.clickTracker() @@ -74,7 +75,7 @@ test.describe('Inbox tests', () => { test.skip('User is able to create a task, assign a himself and close it from inbox', async ({ page }) => { const newIssue = createNewIssueData(data.firstName, data.lastName) - await prepareNewIssueWithOpenStep(page, newIssue) + await prepareNewIssueWithOpenStep(page, newIssue, false) await issuesDetailsPage.checkIssue(newIssue) await leftSideMenuPage.clickTracker() @@ -93,20 +94,24 @@ test.describe('Inbox tests', () => { await leftSideMenuPage.getInviteLink() const linkText = await page.locator('.antiPopup .link').textContent() const page2 = await browser.newPage() - const leftSideMenuPageSecond = new LeftSideMenuPage(page2) - const inboxPageSecond = new InboxPage(page2) - await leftSideMenuPage.clickOnCloseInvite() - await page2.goto(linkText ?? '') - const joinPage = new SignInJoinPage(page2) - await joinPage.join(newUser2) + try { + const leftSideMenuPageSecond = new LeftSideMenuPage(page2) + const inboxPageSecond = new InboxPage(page2) + await leftSideMenuPage.clickOnCloseInvite() + await page2.goto(linkText ?? '') + await setTestOptions(page2) + const joinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) - const newIssue = createNewIssueData(newUser2.firstName, newUser2.lastName) - await prepareNewIssueWithOpenStep(page, newIssue) - await issuesDetailsPage.checkIssue(newIssue) - await leftSideMenuPageSecond.clickTracker() - await leftSideMenuPageSecond.clickNotification() - await inboxPageSecond.checkIfTaskIsPresentInInbox(newIssue.title) - await page2.close() + const newIssue = createNewIssueData(newUser2.firstName, newUser2.lastName) + await prepareNewIssueWithOpenStep(page, newIssue, false) + await issuesDetailsPage.checkIssue(newIssue) + await leftSideMenuPageSecond.clickTracker() + await leftSideMenuPageSecond.clickNotification() + await inboxPageSecond.checkIfTaskIsPresentInInbox(newIssue.title) + } finally { + await page2.close() + } }) test('User is able to assign someone else and he should be able to open the task', async ({ page, browser }) => { @@ -115,24 +120,28 @@ test.describe('Inbox tests', () => { await leftSideMenuPage.getInviteLink() const linkText = await page.locator('.antiPopup .link').textContent() const page2 = await browser.newPage() - const leftSideMenuPageSecond = new LeftSideMenuPage(page2) - const issuesDetailsPageSecond = new IssuesDetailsPage(page2) - const inboxPageSecond = new InboxPage(page2) - await leftSideMenuPage.clickOnCloseInvite() - await page2.goto(linkText ?? '') - const joinPage = new SignInJoinPage(page2) - await joinPage.join(newUser2) + try { + const leftSideMenuPageSecond = new LeftSideMenuPage(page2) + const issuesDetailsPageSecond = new IssuesDetailsPage(page2) + const inboxPageSecond = new InboxPage(page2) + await leftSideMenuPage.clickOnCloseInvite() + await page2.goto(linkText ?? '') + await setTestOptions(page2) + const joinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) - const newIssue = createNewIssueData(newUser2.firstName, newUser2.lastName) - await prepareNewIssueWithOpenStep(page, newIssue) - await issuesDetailsPage.checkIssue(newIssue) - await leftSideMenuPageSecond.clickTracker() - await leftSideMenuPageSecond.clickNotification() - await inboxPageSecond.checkIfTaskIsPresentInInbox(newIssue.title) - await inboxPageSecond.clickOnToDo(newIssue.title) - await inboxPageSecond.clickLeftSidePanelOpen() - await issuesDetailsPageSecond.checkIssue(newIssue) - await page2.close() + const newIssue = createNewIssueData(newUser2.firstName, newUser2.lastName) + await prepareNewIssueWithOpenStep(page, newIssue, false) + await issuesDetailsPage.checkIssue(newIssue) + await leftSideMenuPageSecond.clickTracker() + await leftSideMenuPageSecond.clickNotification() + await inboxPageSecond.checkIfTaskIsPresentInInbox(newIssue.title) + await inboxPageSecond.clickOnToDo(newIssue.title) + await inboxPageSecond.clickLeftSidePanelOpen() + await issuesDetailsPageSecond.checkIssue(newIssue) + } finally { + await page2.close() + } }) test.skip('User is able to create a task, assign a other user and close it from inbox', async ({ page, browser }) => { await leftSideMenuPage.openProfileMenu() @@ -140,26 +149,30 @@ test.describe('Inbox tests', () => { await leftSideMenuPage.getInviteLink() const linkText = await page.locator('.antiPopup .link').textContent() const page2 = await browser.newPage() - const leftSideMenuPageSecond = new LeftSideMenuPage(page2) - const issuesDetailsPageSecond = new IssuesDetailsPage(page2) - const inboxPageSecond = new InboxPage(page2) - await leftSideMenuPage.clickOnCloseInvite() - await page2.goto(linkText ?? '') - const joinPage = new SignInJoinPage(page2) - await joinPage.join(newUser2) + try { + const leftSideMenuPageSecond = new LeftSideMenuPage(page2) + const issuesDetailsPageSecond = new IssuesDetailsPage(page2) + const inboxPageSecond = new InboxPage(page2) + await leftSideMenuPage.clickOnCloseInvite() + await page2.goto(linkText ?? '') + await setTestOptions(page2) + const joinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) - const newIssue = createNewIssueData(newUser2.firstName, newUser2.lastName) - await prepareNewIssueWithOpenStep(page, newIssue) - await issuesDetailsPage.checkIssue(newIssue) - await leftSideMenuPageSecond.clickTracker() - await leftSideMenuPageSecond.clickNotification() - await inboxPageSecond.checkIfTaskIsPresentInInbox(newIssue.title) - await inboxPageSecond.clickOnToDo(newIssue.title) - await inboxPageSecond.clickLeftSidePanelOpen() - await issuesDetailsPageSecond.checkIssue(newIssue) - await inboxPage.clickCloseLeftSidePanel() - // ADD ASSERT ONCE THE ISSUE IS FIXED - await page2.close() + const newIssue = createNewIssueData(newUser2.firstName, newUser2.lastName) + await prepareNewIssueWithOpenStep(page, newIssue, false) + await issuesDetailsPage.checkIssue(newIssue) + await leftSideMenuPageSecond.clickTracker() + await leftSideMenuPageSecond.clickNotification() + await inboxPageSecond.checkIfTaskIsPresentInInbox(newIssue.title) + await inboxPageSecond.clickOnToDo(newIssue.title) + await inboxPageSecond.clickLeftSidePanelOpen() + await issuesDetailsPageSecond.checkIssue(newIssue) + await inboxPage.clickCloseLeftSidePanel() + } finally { + // ADD ASSERT ONCE THE ISSUE IS FIXED + await page2.close() + } }) test('User is able to send message to other user and he should see it in inbox', async ({ page, browser }) => { @@ -169,27 +182,30 @@ test.describe('Inbox tests', () => { await leftSideMenuPage.getInviteLink() const linkText = await page.locator('.antiPopup .link').textContent() const page2 = await browser.newPage() + try { + const leftSideMenuPageSecond = new LeftSideMenuPage(page2) + const inboxPageSecond = new InboxPage(page2) + await leftSideMenuPage.clickOnCloseInvite() + await page2.goto(linkText ?? '') + await setTestOptions(page2) + const joinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) + await page.waitForTimeout(1000) - const leftSideMenuPageSecond = new LeftSideMenuPage(page2) - const inboxPageSecond = new InboxPage(page2) - await leftSideMenuPage.clickOnCloseInvite() - await page2.goto(linkText ?? '') - const joinPage = new SignInJoinPage(page2) - await joinPage.join(newUser2) - await page.waitForTimeout(1000) + await leftSideMenuPage.clickChunter() + await channelPage.clickChannel('general') + await channelPage.sendMessage('Test message') - await leftSideMenuPage.clickChunter() - await channelPage.clickChannel('general') - await channelPage.sendMessage('Test message') - - await channelPage.checkMessageExist('Test message', true, 'Test message') - await leftSideMenuPage.clickNotification() - await inboxPage.checkIfInboxChatExists('Channel general', false) - await leftSideMenuPageSecond.clickNotification() - await inboxPageSecond.checkIfInboxChatExists('Channel general', true) - await inboxPageSecond.clickOnInboxChat('Channel general') - await inboxPageSecond.checkIfTextInChatIsPresent('Test message') - await page2.close() + await channelPage.checkMessageExist('Test message', true, 'Test message') + await leftSideMenuPage.clickNotification() + await inboxPage.checkIfInboxChatExists('Channel general', false) + await leftSideMenuPageSecond.clickNotification() + await inboxPageSecond.checkIfInboxChatExists('Channel general', true) + await inboxPageSecond.clickOnInboxChat('Channel general') + await inboxPageSecond.checkIfTextInChatIsPresent('Test message') + } finally { + await page2.close() + } }) test('User is able to turn off notification and he should not receive messages to inbox', async ({ @@ -202,28 +218,31 @@ test.describe('Inbox tests', () => { await leftSideMenuPage.getInviteLink() const linkText = await page.locator('.antiPopup .link').textContent() const page2 = await browser.newPage() - - const leftSideMenuPageSecond = new LeftSideMenuPage(page2) - const inboxPageSecond = new InboxPage(page2) - const notificationPageSecond = new NotificationsPage(page2) - await leftSideMenuPage.clickOnCloseInvite() - await page2.goto(linkText ?? '') - const joinPage = new SignInJoinPage(page2) - await joinPage.join(newUser2) - const userProfilePageSecond = new UserProfilePage(page2) - await userProfilePageSecond.openProfileMenu() - await userProfilePageSecond.clickSettings() - await userProfilePageSecond.clickOnNotificationsButton() - await notificationPageSecond.clickMenuItem(MenuItems.CHAT) - await notificationPageSecond.toggleChatMessage() - await page.waitForTimeout(1000) - await leftSideMenuPage.clickChunter() - await channelPage.clickChannel('general') - await channelPage.sendMessage('Test message') - await channelPage.checkMessageExist('Test message', true, 'Test message') - await leftSideMenuPageSecond.clickNotification() - await inboxPageSecond.checkIfInboxChatExists('Channel general', false) - await page2.close() + try { + const leftSideMenuPageSecond = new LeftSideMenuPage(page2) + const inboxPageSecond = new InboxPage(page2) + const notificationPageSecond = new NotificationsPage(page2) + await leftSideMenuPage.clickOnCloseInvite() + await page2.goto(linkText ?? '') + await setTestOptions(page2) + const joinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) + const userProfilePageSecond = new UserProfilePage(page2) + await userProfilePageSecond.openProfileMenu() + await userProfilePageSecond.clickSettings() + await userProfilePageSecond.clickOnNotificationsButton() + await notificationPageSecond.clickMenuItem(MenuItems.CHAT) + await notificationPageSecond.toggleChatMessage() + await page.waitForTimeout(1000) + await leftSideMenuPage.clickChunter() + await channelPage.clickChannel('general') + await channelPage.sendMessage('Test message') + await channelPage.checkMessageExist('Test message', true, 'Test message') + await leftSideMenuPageSecond.clickNotification() + await inboxPageSecond.checkIfInboxChatExists('Channel general', false) + } finally { + await page2.close() + } }) test('User is able to change filter in inbox', async ({ page, browser }) => { @@ -233,30 +252,33 @@ test.describe('Inbox tests', () => { await leftSideMenuPage.getInviteLink() const linkText = await page.locator('.antiPopup .link').textContent() const page2 = await browser.newPage() - const leftSideMenuPageSecond = new LeftSideMenuPage(page2) - const inboxPageSecond = new InboxPage(page2) - await leftSideMenuPage.clickOnCloseInvite() - await page2.goto(linkText ?? '') - const joinPage = new SignInJoinPage(page2) - await joinPage.join(newUser2) + try { + const leftSideMenuPageSecond = new LeftSideMenuPage(page2) + const inboxPageSecond = new InboxPage(page2) + await leftSideMenuPage.clickOnCloseInvite() + await page2.goto(linkText ?? '') + const joinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) - await leftSideMenuPage.clickChunter() - await channelPage.clickChannel('general') - await channelPage.sendMessage('Test message') - await leftSideMenuPage.clickTracker() + await leftSideMenuPage.clickChunter() + await channelPage.clickChannel('general') + await channelPage.sendMessage('Test message') + await leftSideMenuPage.clickTracker() - const newIssue = createNewIssueData(newUser2.firstName, newUser2.lastName) - await prepareNewIssueWithOpenStep(page, newIssue) - await issuesDetailsPage.checkIssue(newIssue) - await leftSideMenuPageSecond.clickTracker() - await leftSideMenuPageSecond.clickNotification() - await inboxPageSecond.clickOnInboxFilter('Channels') - await inboxPageSecond.checkIfInboxChatExists(newIssue.title, false) - await inboxPageSecond.checkIfInboxChatExists('Test message', true) - await inboxPageSecond.clickOnInboxFilter('Issues') - await inboxPageSecond.checkIfIssueIsPresentInInbox(newIssue.title) - await inboxPageSecond.checkIfInboxChatExists('Channel general', false) - await page2.close() + const newIssue = createNewIssueData(newUser2.firstName, newUser2.lastName) + await prepareNewIssueWithOpenStep(page, newIssue, false) + await issuesDetailsPage.checkIssue(newIssue) + await leftSideMenuPageSecond.clickTracker() + await leftSideMenuPageSecond.clickNotification() + await inboxPageSecond.clickOnInboxFilter('Channels') + await inboxPageSecond.checkIfInboxChatExists(newIssue.title, false) + await inboxPageSecond.checkIfInboxChatExists('Test message', true) + await inboxPageSecond.clickOnInboxFilter('Issues') + await inboxPageSecond.checkIfIssueIsPresentInInbox(newIssue.title) + await inboxPageSecond.checkIfInboxChatExists('Channel general', false) + } finally { + await page2.close() + } }) test('Checking the ability to receive a task and schedule it', async ({ page, browser }) => { @@ -267,42 +289,46 @@ test.describe('Inbox tests', () => { await leftSideMenuPage.clickOnCloseInvite() const page2 = await browser.newPage() - await page2.goto(linkText ?? '') - const joinPage = new SignInJoinPage(page2) - await joinPage.join(newUser2) + try { + await page2.goto(linkText ?? '') + await setTestOptions(page2) + const joinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) - const newIssue = createNewIssueData(data.firstName, data.lastName, { - status: 'Todo', - assignee: `${newUser2.lastName} ${newUser2.firstName}`, - estimation: '0' - }) - await prepareNewIssueWithOpenStep(page, newIssue) - await issuesDetailsPage.checkIssue(newIssue) + const newIssue = createNewIssueData(data.firstName, data.lastName, { + status: 'Todo', + assignee: `${newUser2.lastName} ${newUser2.firstName}`, + estimation: '0' + }) + await prepareNewIssueWithOpenStep(page, newIssue, false) + await issuesDetailsPage.checkIssue(newIssue) - const leftSideMenuPageSecond = new LeftSideMenuPage(page2) - const inboxPageSecond = new InboxPage(page2) - const issuesDetailsPageSecond = new IssuesDetailsPage(page2) - const planningPageSecond = new PlanningPage(page2) - await leftSideMenuPageSecond.clickNotification() - await inboxPageSecond.checkIfIssueIsPresentInInbox(newIssue.title) - await inboxPageSecond.clickIssuePresentInInbox(newIssue.title) - await inboxPageSecond.clickLeftSidePanelOpen() - await issuesDetailsPageSecond.checkIssue(newIssue) - await leftSideMenuPageSecond.clickPlanner() - await planningPageSecond.closeNotification() - await planningPageSecond.dragToCalendar(newIssue.title, 2, getTimeForPlanner()) - await planningPageSecond.eventInSchedule(newIssue.title).isVisible() - await attachScreenshot('Recive_task_and_scheduled.png', page2) + const leftSideMenuPageSecond = new LeftSideMenuPage(page2) + const inboxPageSecond = new InboxPage(page2) + const issuesDetailsPageSecond = new IssuesDetailsPage(page2) + const planningPageSecond = new PlanningPage(page2) + await leftSideMenuPageSecond.clickNotification() + await inboxPageSecond.checkIfIssueIsPresentInInbox(newIssue.title) + await inboxPageSecond.clickIssuePresentInInbox(newIssue.title) + await inboxPageSecond.clickLeftSidePanelOpen() + await issuesDetailsPageSecond.checkIssue(newIssue) + await leftSideMenuPageSecond.clickPlanner() + await planningPageSecond.closeNotification() + await planningPageSecond.dragToCalendar(newIssue.title, 2, getTimeForPlanner()) + await planningPageSecond.eventInSchedule(newIssue.title).isVisible() + await attachScreenshot('Recive_task_and_scheduled.png', page2) - await attachScreenshot('Recive_task_and_scheduled-Detail.png', page) - await issuesDetailsPage.checkIssue({ ...newIssue, status: 'In Progress' }) - await leftSideMenuPage.clickTeam() - const teamPage = new TeamPage(page) - await teamPage.checkTeamPageIsOpened() - await teamPage.selectTeam('Default') - await teamPage.buttonNextDay().click() - await attachScreenshot('Recive_task_and_scheduled-Tomorrow.png', page) - await teamPage.getItemByText('Tomorrow', newIssue.title).isVisible() - await page2.close() + await attachScreenshot('Recive_task_and_scheduled-Detail.png', page) + await issuesDetailsPage.checkIssue({ ...newIssue, status: 'In Progress' }) + await leftSideMenuPage.clickTeam() + const teamPage = new TeamPage(page) + await teamPage.checkTeamPageIsOpened() + await teamPage.selectTeam('Default') + await teamPage.buttonNextDay().click() + await attachScreenshot('Recive_task_and_scheduled-Tomorrow.png', page) + await teamPage.getItemByText('Tomorrow', newIssue.title).isVisible() + } finally { + await page2.close() + } }) }) diff --git a/tests/sanity/tests/model/recruiting/talents-page.ts b/tests/sanity/tests/model/recruiting/talents-page.ts index 0520bd8c36..9264894f37 100644 --- a/tests/sanity/tests/model/recruiting/talents-page.ts +++ b/tests/sanity/tests/model/recruiting/talents-page.ts @@ -218,11 +218,6 @@ export class TalentsPage extends CommonRecruitingPage { await this.selectFromDropdown(this.page, action) } - async checkMatchVacancy (talentName: string, score: string): Promise { - await expect(this.textVacancyMatchingTalent()).toContainText(talentName, { ignoreCase: true }) - await expect(this.textVacancyMatchingScore()).toContainText(score) - } - async searchTalentByTalentName (talentName: TalentName): Promise { await this.inputSearchIcon().click() await this.inputSearchTalent().fill(`${talentName.lastName} ${talentName.firstName}`) diff --git a/tests/sanity/tests/recruiting/talents.spec.ts b/tests/sanity/tests/recruiting/talents.spec.ts index e13a8453a3..db4aac7be8 100644 --- a/tests/sanity/tests/recruiting/talents.spec.ts +++ b/tests/sanity/tests/recruiting/talents.spec.ts @@ -1,9 +1,8 @@ import { test } from '@playwright/test' -import { generateId, PlatformSetting, PlatformURI } from '../utils' import { NavigationMenuPage } from '../model/recruiting/navigation-menu-page' -import { TalentsPage } from '../model/recruiting/talents-page' import { TalentDetailsPage } from '../model/recruiting/talent-details-page' -import { TalentName } from '../model/recruiting/types' +import { TalentsPage } from '../model/recruiting/talents-page' +import { generateId, PlatformSetting, PlatformURI } from '../utils' test.use({ storageState: PlatformSetting @@ -124,17 +123,6 @@ test.describe('candidate/talents tests', () => { await talentDetailsPage.checkMergeContacts(firstLocation, titleTalent2, sourceTalent1) }) - test('Match to vacancy', async () => { - const talentName: TalentName = { - firstName: 'Software', - lastName: `Engineer-${generateId(4)}` - } - await navigationMenuPage.clickButtonTalents() - await talentsPage.createNewTalentWithName(talentName.firstName, talentName.lastName) - await talentsPage.rightClickAction(talentName, 'Match to vacancy') - await talentsPage.checkMatchVacancy(`${talentName.lastName} ${talentName.firstName}`, '0') - }) - test('Filtering talents by skills', async ({ page }) => { const skillName = `Skill-${generateId(4)}` const talentName = 'P. Andrey' diff --git a/tests/sanity/tests/tracker/common-steps.ts b/tests/sanity/tests/tracker/common-steps.ts index 786397bf20..106d3bd5cb 100644 --- a/tests/sanity/tests/tracker/common-steps.ts +++ b/tests/sanity/tests/tracker/common-steps.ts @@ -14,13 +14,19 @@ export async function prepareNewIssueStep (page: Page, issue: NewIssue): Promise }) } -export async function prepareNewIssueWithOpenStep (page: Page, issue: NewIssue): Promise { +export async function prepareNewIssueWithOpenStep ( + page: Page, + issue: NewIssue, + search: boolean = true +): Promise { return await test.step('Prepare Issue', async () => { const issuesPage = new IssuesPage(page) await issuesPage.linkSidebarAll().click() await issuesPage.clickModelSelectorAll() await issuesPage.createNewIssue(issue) - await issuesPage.searchIssueByName(issue.title) + if (search) { + await issuesPage.searchIssueByName(issue.title) + } await issuesPage.openIssueByName(issue.title) return await issuesPage.getIssueId(issue.title) }) diff --git a/tests/sanity/tests/tracker/public-link.spec.ts b/tests/sanity/tests/tracker/public-link.spec.ts index 6bf76399a4..8c5afc699d 100644 --- a/tests/sanity/tests/tracker/public-link.spec.ts +++ b/tests/sanity/tests/tracker/public-link.spec.ts @@ -1,5 +1,5 @@ import { expect, test } from '@playwright/test' -import { generateId, PlatformSetting, PlatformURI } from '../utils' +import { generateId, PlatformSetting, PlatformURI, setTestOptions } from '../utils' import { LeftSideMenuPage } from '../model/left-side-menu-page' import { IssuesDetailsPage } from '../model/tracker/issues-details-page' import { NewIssue } from '../model/tracker/types' @@ -18,20 +18,23 @@ test.describe('Tracker public link issues tests', () => { const newContext = await browser.newContext({ storageState: PlatformSetting }) const page = await newContext.newPage() - await page.goto(`${PlatformURI}/workbench/sanity-ws`) + try { + await page.goto(`${PlatformURI}/workbench/sanity-ws`) + await setTestOptions(page) - const leftSideMenuPage = new LeftSideMenuPage(page) - await leftSideMenuPage.clickTracker() - await prepareNewIssueWithOpenStep(page, publicLinkIssue) + const leftSideMenuPage = new LeftSideMenuPage(page) + await leftSideMenuPage.clickTracker() + await prepareNewIssueWithOpenStep(page, publicLinkIssue) - const issuesDetailsPage = new IssuesDetailsPage(page) - await issuesDetailsPage.moreActionOnIssue('Public link') + const issuesDetailsPage = new IssuesDetailsPage(page) + await issuesDetailsPage.moreActionOnIssue('Public link') - const publicLinkPopup = new PublicLinkPopup(page) - link = await publicLinkPopup.getPublicLink() - - await page.close() - await newContext.close() + const publicLinkPopup = new PublicLinkPopup(page) + link = await publicLinkPopup.getPublicLink() + } finally { + await page.close() + await newContext.close() + } }) await test.step('Check guest access to the issue', async () => { @@ -40,19 +43,20 @@ test.describe('Tracker public link issues tests', () => { await clearSession.clearPermissions() const clearPage = await clearSession.newPage() + try { + await clearPage.goto(link) - await clearPage.goto(link) - - const clearIssuesDetailsPage = new IssuesDetailsPage(clearPage) - await clearIssuesDetailsPage.waitDetailsOpened(publicLinkIssue.title) - await clearIssuesDetailsPage.checkIssue({ - ...publicLinkIssue, - status: 'Backlog' - }) - expect(clearPage.url()).toContain('guest') - - await clearPage.close() - await clearSession.close() + const clearIssuesDetailsPage = new IssuesDetailsPage(clearPage) + await clearIssuesDetailsPage.waitDetailsOpened(publicLinkIssue.title) + await clearIssuesDetailsPage.checkIssue({ + ...publicLinkIssue, + status: 'Backlog' + }) + expect(clearPage.url()).toContain('guest') + } finally { + await clearPage.close() + await clearSession.close() + } }) }) @@ -64,50 +68,55 @@ test.describe('Tracker public link issues tests', () => { const newContext = await browser.newContext({ storageState: PlatformSetting }) const page = await newContext.newPage() - let link: string - - await test.step('Get public link from popup', async () => { - await page.goto(`${PlatformURI}/workbench/sanity-ws`) - - const leftSideMenuPage = new LeftSideMenuPage(page) - await leftSideMenuPage.clickTracker() - await prepareNewIssueWithOpenStep(page, publicLinkIssue) - - const issuesDetailsPage = new IssuesDetailsPage(page) - await issuesDetailsPage.moreActionOnIssue('Public link') - - const publicLinkPopup = new PublicLinkPopup(page) - link = await publicLinkPopup.getPublicLink() - }) const clearSession = await browser.newContext() const clearPage = await clearSession.newPage() + try { + let link: string - await test.step('Check guest access to the issue', async () => { - await clearPage.goto(link) + await test.step('Get public link from popup', async () => { + await page.goto(`${PlatformURI}/workbench/sanity-ws`) + await setTestOptions(page) - const clearIssuesDetailsPage = new IssuesDetailsPage(clearPage) - await clearIssuesDetailsPage.waitDetailsOpened(publicLinkIssue.title) - await clearIssuesDetailsPage.checkIssue({ - ...publicLinkIssue, - status: 'Backlog' + const leftSideMenuPage = new LeftSideMenuPage(page) + await leftSideMenuPage.clickTracker() + await prepareNewIssueWithOpenStep(page, publicLinkIssue) + + const issuesDetailsPage = new IssuesDetailsPage(page) + await issuesDetailsPage.moreActionOnIssue('Public link') + + const publicLinkPopup = new PublicLinkPopup(page) + link = await publicLinkPopup.getPublicLink() }) - expect(clearPage.url()).toContain('guest') - }) - await test.step('Revoke guest access to the issue', async () => { - const publicLinkPopup = new PublicLinkPopup(page) - await publicLinkPopup.revokePublicLink() - }) + await test.step('Check guest access to the issue', async () => { + await clearPage.goto(link) + await setTestOptions(clearPage) - await test.step('Check guest access to the issue after the revoke', async () => { - await clearPage.goto(link) - await expect(clearPage.locator('div.antiPopup > h1')).toHaveText('Public link was revoked') - }) + const clearIssuesDetailsPage = new IssuesDetailsPage(clearPage) + await clearIssuesDetailsPage.waitDetailsOpened(publicLinkIssue.title) + await clearIssuesDetailsPage.checkIssue({ + ...publicLinkIssue, + status: 'Backlog' + }) + expect(clearPage.url()).toContain('guest') + }) - await clearPage.close() - await clearSession.close() - await page.close() - await newContext.close() + await test.step('Revoke guest access to the issue', async () => { + const publicLinkPopup = new PublicLinkPopup(page) + await publicLinkPopup.revokePublicLink() + }) + + await test.step('Check guest access to the issue after the revoke', async () => { + await clearPage.goto(link) + await setTestOptions(clearPage) + await expect(clearPage.locator('div.antiPopup > h1')).toHaveText('Public link was revoked') + }) + } finally { + await clearPage.close() + await clearSession.close() + await page.close() + await newContext.close() + } }) }) diff --git a/tests/sanity/tests/utils.ts b/tests/sanity/tests/utils.ts index 265baa66a3..2e6816341a 100644 --- a/tests/sanity/tests/utils.ts +++ b/tests/sanity/tests/utils.ts @@ -16,6 +16,7 @@ export const PlatformUser = process.env.PLATFORM_USER as string export const PlatformUserSecond = process.env.PLATFORM_USER_SECOND as string export const PlatformSetting = process.env.SETTING as string export const PlatformSettingSecond = process.env.SETTING_SECOND as string + export const DefaultWorkspace = 'SanityTest' export const LocalUrl = process.env.LOCAL_URL as string export const DevUrl = process.env.DEV_URL as string @@ -83,6 +84,14 @@ function count (): string { return toHex(val, 6) } +export async function setTestOptions (page: Page): Promise { + await page.evaluate(() => { + localStorage.setItem('#platform.notification.timeout', '0') + localStorage.setItem('#platform.testing.enabled', 'true') + localStorage.setItem('#platform.lazy.loading', 'false') + }) +} + /** * @public * @returns diff --git a/tests/sanity/tests/workspace/create.spec.ts b/tests/sanity/tests/workspace/create.spec.ts index d51304d485..409be347a4 100644 --- a/tests/sanity/tests/workspace/create.spec.ts +++ b/tests/sanity/tests/workspace/create.spec.ts @@ -78,7 +78,6 @@ test.describe('Workspace tests', () => { await trackerNavigationMenuPage.openIssuesForProject('Default') await issuesPage.clickModelSelectorAll() await issuesPage.createNewIssue(newIssue) - await issuesPage.searchIssueByName(newIssue.title) await issuesPage.openIssueByName(newIssue.title) const issuesDetailsPage = new IssuesDetailsPage(page) @@ -136,21 +135,24 @@ test.describe('Workspace tests', () => { const linkText = await page.locator('.antiPopup .link').textContent() const page2 = await browser.newPage() - await page2.goto(linkText ?? '') - const newUser2: SignUpData = { - firstName: `FirstName2-${generateId()}`, - lastName: `LastName2-${generateId()}`, - email: `sanity-email+${generateId()}@gmail.com`, - password: '1234' + try { + await page2.goto(linkText ?? '') + const newUser2: SignUpData = { + firstName: `FirstName2-${generateId()}`, + lastName: `LastName2-${generateId()}`, + email: `sanity-email+${generateId()}@gmail.com`, + password: '1234' + } + + await page2.getByRole('link', { name: 'Sign Up' }).click() + const signUpPage2 = new SignUpPage(page2) + await signUpPage2.signUp(newUser2, 'join') + + const leftSideMenuPage2 = new LeftSideMenuPage(page2) + await leftSideMenuPage2.clickTracker() + } finally { + await page2.close() } - - await page2.getByRole('link', { name: 'Sign Up' }).click() - const signUpPage2 = new SignUpPage(page2) - await signUpPage2.signUp(newUser2, 'join') - - const leftSideMenuPage2 = new LeftSideMenuPage(page2) - await leftSideMenuPage2.clickTracker() - await page2.close() }) test('Create a workspace with join link - existing account', async ({ page, browser }) => { @@ -174,28 +176,31 @@ test.describe('Workspace tests', () => { const linkText = await page.locator('.antiPopup .link').textContent() const page2 = await browser.newPage() - const loginPage2 = new LoginPage(page2) - await loginPage2.goto() - await loginPage2.clickSignUp() + try { + const loginPage2 = new LoginPage(page2) + await loginPage2.goto() + await loginPage2.clickSignUp() - const newUser2: SignUpData = { - firstName: `FirstName2-${generateId()}`, - lastName: `LastName2-${generateId()}`, - email: `sanity-email+${generateId()}@gmail.com`, - password: '1234' + const newUser2: SignUpData = { + firstName: `FirstName2-${generateId()}`, + lastName: `LastName2-${generateId()}`, + email: `sanity-email+${generateId()}@gmail.com`, + password: '1234' + } + + const signUpPage2 = new SignUpPage(page2) + await signUpPage2.signUp(newUser2) + + // Ok we signed in, and no workspace present. + await page2.goto(linkText ?? '') + const joinPage = new SignInJoinPage(page2) + await joinPage.join(newUser2) + + const leftSideMenuPage2 = new LeftSideMenuPage(page2) + await leftSideMenuPage2.clickTracker() + } finally { + await page2.close() } - - const signUpPage2 = new SignUpPage(page2) - await signUpPage2.signUp(newUser2) - - // Ok we signed in, and no workspace present. - await page2.goto(linkText ?? '') - const joinPage = new SignInJoinPage(page2) - await joinPage.join(newUser2) - - const leftSideMenuPage2 = new LeftSideMenuPage(page2) - await leftSideMenuPage2.clickTracker() - await page2.close() }) test('Create workspace with LastToken in the localStorage', async ({ page, browser }) => { @@ -210,20 +215,22 @@ test.describe('Workspace tests', () => { await test.step('Check create workspace action', async () => { const newWorkspaceName = `Some HULY #@$ WS - ${generateId(12)}` const pageSecond = await browser.newPage() + try { + // Authenticate in new browser context + await pageSecond.goto(`${PlatformURI}/login/login`) + await leftSideMenuPage.setLastTokenOnPage(pageSecond, await leftSideMenuPage.getLastToken()) + await pageSecond.goto(`${PlatformURI}/login/createWorkspace`) - // Authenticate in new browser context - await pageSecond.goto(`${PlatformURI}/login/login`) - await leftSideMenuPage.setLastTokenOnPage(pageSecond, await leftSideMenuPage.getLastToken()) - await pageSecond.goto(`${PlatformURI}/login/createWorkspace`) + // Create workspace in the second context + const selectWorkspacePageSecond = new SelectWorkspacePage(pageSecond) + await selectWorkspacePageSecond.createWorkspace(newWorkspaceName) - // Create workspace in the second context - const selectWorkspacePageSecond = new SelectWorkspacePage(pageSecond) - await selectWorkspacePageSecond.createWorkspace(newWorkspaceName) - - // Use the tracker in the second context - const leftSideMenuPageSecond = new LeftSideMenuPage(pageSecond) - await leftSideMenuPageSecond.clickTracker() - await pageSecond.close() + // Use the tracker in the second context + const leftSideMenuPageSecond = new LeftSideMenuPage(pageSecond) + await leftSideMenuPageSecond.clickTracker() + } finally { + await pageSecond.close() + } }) })