diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ba3e204f54..d36f7bf0bf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -228,6 +228,9 @@ jobs: run: node common/scripts/install-run-rush.js docker env: DOCKER_CLI_HINTS: false + - name: Configure /etc/hosts + run: | + sudo echo "127.0.0.1 huly.local" | sudo tee -a /etc/hosts - name: Prepare server run: | cd ./tests @@ -343,6 +346,9 @@ jobs: run: node common/scripts/install-run-rush.js docker env: DOCKER_CLI_HINTS: false + - name: Configure /etc/hosts + run: | + sudo echo "127.0.0.1 huly.local" | sudo tee -a /etc/hosts - name: Prepare server run: | cd ./tests diff --git a/.vscode/launch.json b/.vscode/launch.json index 861df5b22c..4be0626890 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,9 +11,7 @@ "port": 9229, "request": "attach", "sourceMaps": true, - "skipFiles": [ - "/**" - ], + "skipFiles": ["/**"], "type": "node" }, { @@ -25,7 +23,7 @@ "console": "integratedTerminal", "justMyCode": true, "env": { - "PYTORCH_ENABLE_MPS_FALLBACK":"1" + "PYTORCH_ENABLE_MPS_FALLBACK": "1" } }, { @@ -54,7 +52,7 @@ "ACCOUNTS_URL": "http://localhost:3000", "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", // "SERVER_PROVIDER":"uweb" - "SERVER_PROVIDER":"ws", + "SERVER_PROVIDER": "ws", "MODEL_VERSION": "0.7.1", // "VERSION": "0.6.289", "ELASTIC_INDEX_NAME": "local_storage_index", @@ -97,13 +95,14 @@ "ACCOUNTS_URL": "http://localhost:3000", "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", // "SERVER_PROVIDER":"uweb" - "SERVER_PROVIDER":"ws", - "MODEL_VERSION": "0.7.1", + "SERVER_PROVIDER": "ws", + "MODEL_VERSION": "0.7.24", // "VERSION": "0.6.289", "ELASTIC_INDEX_NAME": "local_storage_index", "UPLOAD_URL": "/files", "AI_BOT_URL": "http://localhost:4010", - "STATS_URL": "http://huly.local:4900" + "STATS_URL": "http://huly.local:4900", + "QUEUE_CONFIG": "localhost:19092" }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeVersion": "20", @@ -163,8 +162,10 @@ "REKONI_URL": "http://localhost:4004", "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", "ELASTIC_INDEX_NAME": "local_storage_index", - "STATS_URL":"http://huly.local:4900", + "REGION": "cockroach", + "STATS_URL": "http://huly.local:4900", "ACCOUNTS_URL": "http://localhost:3000", + "QUEUE_CONFIG": "localhost:19092" }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeVersion": "20", @@ -184,7 +185,7 @@ "DB_URL": "mongodb://localhost:27017", // "DB_URL": "postgresql://root@huly.local:26257/defaultdb?sslmode=disable", "SERVER_SECRET": "secret", - "REGION_INFO":"|Mongo;cockroach|CockroachDB", + "REGION_INFO": "|Mongo;cockroach|CockroachDB", "TRANSACTOR_URL": "ws://huly.local:3333,ws://huly.local:3332;;cockroach", "ACCOUNTS_URL": "http://localhost:3000", "ACCOUNT_PORT": "3000", @@ -215,8 +216,8 @@ "DB_URL": "mongodb://localhost:27018", // "DB_URL": "postgresql://postgres:example@localhost:5432", "SERVER_SECRET": "secret", - "REGION_INFO":"|Mongo;pg|Postgres;cockroach|CockroachDB", - "TRANSACTOR_URL": "ws://huly.local:3334;;,ws://huly.local:3335;;europe", + "REGION_INFO": "|Mongo;pg|Postgres;cockroach|CockroachDB", + "TRANSACTOR_URL": "ws://huly.local:3334;;,ws://huly.local:3335;;europe", "ACCOUNTS_URL": "http://localhost:3003", "ACCOUNT_PORT": "3003", "FRONT_URL": "http://localhost:8083", @@ -261,7 +262,7 @@ "DB_URL": "mongodb://localhost:27017", // "DB_URL": "postgresql://postgres:example@localhost:5432", // "DB_URL": "postgresql://root@huly.local:26257/defaultdb?sslmode=disable", - + "REGION": "", "SERVER_SECRET": "secret", "TRANSACTOR_URL": "ws://localhost:3333", @@ -274,7 +275,7 @@ "MODEL_VERSION": "v0.7.1", "WS_OPERATION": "all+backup", "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin", - "BACKUP_BUCKET": "dev-backups", + "BACKUP_BUCKET": "dev-backups" // "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init", // "INIT_WORKSPACE": "staging-dev" }, @@ -307,7 +308,7 @@ "MODEL_VERSION": "0.7.1", "WS_OPERATION": "all+backup", "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin", - "BACKUP_BUCKET": "dev-backups", + "BACKUP_BUCKET": "dev-backups" // "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init", // "INIT_WORKSPACE": "staging-dev" }, @@ -447,7 +448,7 @@ "name": "Debug tool upgrade PG(Cockroach)", "type": "node", "request": "launch", - "args": ["src/__start.ts", "upgrade-workspace", "platform" ], + "args": ["src/__start.ts", "upgrade-workspace", "platform"], "env": { "SERVER_SECRET": "secret", "MINIO_ACCESS_KEY": "minioadmin", @@ -505,8 +506,8 @@ "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", "SECRET": "secret", "REGION": "cockroach", - "BUCKET_NAME":"backups", - "INTERVAL":"0" + "BUCKET_NAME": "backups", + "INTERVAL": "0" }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "showAsyncStacks": true, @@ -528,7 +529,7 @@ "DB_URL": "mongodb://localhost:27017", "SERVER_SECRET": "secret", "SECRET": "secret", - "BUCKET_NAME":"backups" + "BUCKET_NAME": "backups" }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "showAsyncStacks": true, @@ -548,7 +549,7 @@ // "DB_URL": "mongodb://localhost:27017", "DB_URL": "postgresql://root@host.docker.internal:26258/defaultdb?sslmode=disable", "SERVER_SECRET": "secret", - "BUCKET_NAME":"dev-backups" + "BUCKET_NAME": "dev-backups" }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "showAsyncStacks": true, @@ -714,7 +715,7 @@ "AVATAR_PATH": "./assets/avatar.png", "AVATAR_CONTENT_TYPE": ".png", "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin", - "LOVE_ENDPOINT": "http://localhost:8096", + "LOVE_ENDPOINT": "http://localhost:8096" }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "sourceMaps": true, @@ -771,14 +772,26 @@ "urlFilter": "http://localhost:8080/*", "port": 9222, "sourceMapPathOverrides": { - "webpack://*": "${workspaceFolder}/*", + "webpack://*": "${workspaceFolder}/*" } }, { "name": "Debug Notion import", "type": "node", "request": "launch", - "args": ["src/__start.ts", "import-notion-to-teamspace", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-u", "user1", "-pw", "1234", "-ws", "ws5", "-ts", "notion", ], + "args": [ + "src/__start.ts", + "import-notion-to-teamspace", + "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", + "-u", + "user1", + "-pw", + "1234", + "-ws", + "ws5", + "-ts", + "notion" + ], // "args": ["src/__start.ts", "import-notion-with-teamspaces", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-u", "user1", "-pw", "1234", "-ws", "ws1"], "env": { "FRONT_URL": "http://localhost:8087" @@ -793,9 +806,19 @@ "name": "Debug ClickUp import", "type": "node", "request": "launch", - "args": ["src/__start.ts", "import-clickup-tasks", "/home/anna/work/clickup/aleksandr/debug/mentions.csv", "-u", "user1", "-pw", "1234", "-ws", "ws10"], + "args": [ + "src/__start.ts", + "import-clickup-tasks", + "/home/anna/work/clickup/aleksandr/debug/mentions.csv", + "-u", + "user1", + "-pw", + "1234", + "-ws", + "ws10" + ], "env": { - "FRONT_URL": "http://localhost:8087", + "FRONT_URL": "http://localhost:8087" }, "runtimeVersion": "20", "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], @@ -807,7 +830,17 @@ "name": "Debug Huly import", "type": "node", "request": "launch", - "args": ["src/__start.ts", "import", "/home/anna/huly/platform/dev/import-tool/docs/huly/example-workspace", "-u", "user1", "-pw", "1234", "-ws", "ws1"], + "args": [ + "src/__start.ts", + "import", + "/home/anna/huly/platform/dev/import-tool/docs/huly/example-workspace", + "-u", + "user1", + "-pw", + "1234", + "-ws", + "ws1" + ], "env": { "FRONT_URL": "http://localhost:8087" }, diff --git a/README.md b/README.md index aef61be37f..0bc63d6ed2 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ Add the following line to your /etc/hosts file ```plain 127.0.0.1 huly.local +::1 huly.local ``` Accessing the URL will lead you to the app in development mode. diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index ddfe4cede8..c62204e3ae 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -316,6 +316,9 @@ importers: '@rush-temp/inventory-resources': specifier: file:./projects/inventory-resources.tgz version: file:projects/inventory-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + '@rush-temp/kafka': + specifier: file:./projects/kafka.tgz + version: file:projects/kafka.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/kanban': specifier: file:./projects/kanban.tgz version: file:projects/kanban.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) @@ -508,9 +511,6 @@ importers: '@rush-temp/model-server-drive': specifier: file:./projects/model-server-drive.tgz version: file:projects/model-server-drive.tgz - '@rush-temp/model-server-fulltext': - specifier: file:./projects/model-server-fulltext.tgz - version: file:projects/model-server-fulltext.tgz '@rush-temp/model-server-gmail': specifier: file:./projects/model-server-gmail.tgz version: file:projects/model-server-gmail.tgz @@ -922,12 +922,6 @@ importers: '@rush-temp/server-drive-resources': specifier: file:./projects/server-drive-resources.tgz version: file:projects/server-drive-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) - '@rush-temp/server-fulltext': - specifier: file:./projects/server-fulltext.tgz - version: file:projects/server-fulltext.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) - '@rush-temp/server-fulltext-resources': - specifier: file:./projects/server-fulltext-resources.tgz - version: file:projects/server-fulltext-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/server-github': specifier: file:./projects/server-github.tgz version: file:projects/server-github.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) @@ -1786,6 +1780,9 @@ importers: jwt-simple: specifier: ^0.5.6 version: 0.5.6 + kafkajs: + specifier: ^2.2.4 + version: 2.2.4 koa: specifier: ^2.15.4 version: 2.16.0 @@ -3830,7 +3827,7 @@ packages: version: 0.0.0 '@rush-temp/ai-bot-resources@file:projects/ai-bot-resources.tgz': - resolution: {integrity: sha512-6E8sBtpn6OP/Vpgmf02OPnglOUBfDvB0iZFmvu6Dp1NhTir70tm8kV8IBCqU4MadnD4f636sZwKEItk4NXv+Rw==, tarball: file:projects/ai-bot-resources.tgz} + resolution: {integrity: sha512-F9ehCKoqDjVOCtNkYkhL5J8f2a6LVuZZlFjnlNllq3cdEv9D5s8q17KREQnVjEwdXK6mPrUbsWaDjNDpyd8Bug==, tarball: file:projects/ai-bot-resources.tgz} version: 0.0.0 '@rush-temp/ai-bot@file:projects/ai-bot.tgz': @@ -4153,6 +4150,10 @@ packages: resolution: {integrity: sha512-FVCeKLDYhO53Xnhz0Q/64nU6cMcGaQWrF8jvheuF14ZhPevDUaQfRMwnLR4RGBzIhiTc9jvUIUM54VI1FSkoAA==, tarball: file:projects/inventory.tgz} version: 0.0.0 + '@rush-temp/kafka@file:projects/kafka.tgz': + resolution: {integrity: sha512-Q5+NPNp08oBRTGukVK1HY4Bx/XI3TSnbIY2TIhi4HhU9etcWhHL2EGcsmWhFRfG0RC2gcXC9LI3prx9HDc2kEw==, tarball: file:projects/kafka.tgz} + version: 0.0.0 + '@rush-temp/kanban@file:projects/kanban.tgz': resolution: {integrity: sha512-a1aAx7zBxeZjaejI86Ho69hHcEm7F4dkdD9rVCv2BVHzjG/1C8mq7pv+QWHFGoxkiQwwm9MatVV4IMCSjDjAZw==, tarball: file:projects/kanban.tgz} version: 0.0.0 @@ -4222,7 +4223,7 @@ packages: version: 0.0.0 '@rush-temp/model-all@file:projects/model-all.tgz': - resolution: {integrity: sha512-G7I5APyAySX8V9/mzUjuXX3LGtRNZ+uYQhHTl4/7ulGaAsuSagzDJIvClO2akY+GhplEfRrKBiI2fb0R0/i30g==, tarball: file:projects/model-all.tgz} + resolution: {integrity: sha512-UxCRw3u19dTizt2hYLWPNCJ468ntnLnwEtP/8mAEb1Ta93SbPgbV9RfCK+hBBM/vp843HVwdNrab1fwqFXb+3Q==, tarball: file:projects/model-all.tgz} version: 0.0.0 '@rush-temp/model-analytics-collector@file:projects/model-analytics-collector.tgz': @@ -4405,10 +4406,6 @@ packages: resolution: {integrity: sha512-MQUIWrqoJZkFbbKhT569m2pfvCJsK7l3rVWhO9DlNfxFmItei5ZSsMzC7+4qpI7iyQsb620y23RkhEl17/9YKQ==, tarball: file:projects/model-server-drive.tgz} version: 0.0.0 - '@rush-temp/model-server-fulltext@file:projects/model-server-fulltext.tgz': - resolution: {integrity: sha512-qOs1G3E2P64j6xsyD1w2+ZPLclBeAyx1xmGEI19kneyA8IB8NNRcpI5GIB9vIrYJ2lJUi0jyI+sHkLoxmgtBaA==, tarball: file:projects/model-server-fulltext.tgz} - version: 0.0.0 - '@rush-temp/model-server-gmail@file:projects/model-server-gmail.tgz': resolution: {integrity: sha512-vdwW7KKXUWLji9gEb2uEGwbjyJfeQr2g4G/vhZH15sy87rI5f7+M7x5p7zZq5IOF9dk6YArAbILD/yl7u+E3Xg==, tarball: file:projects/model-server-gmail.tgz} version: 0.0.0 @@ -4614,7 +4611,7 @@ packages: version: 0.0.0 '@rush-temp/pod-ai-bot@file:projects/pod-ai-bot.tgz': - resolution: {integrity: sha512-sv9r+DDafpUyjHUabAnULCjQC2xnZ8t4AfhIhvXBM6vl500IrhBVkeAGcNeIk0WYepfAnRn4mQVM3yX+0e0Ygw==, tarball: file:projects/pod-ai-bot.tgz} + resolution: {integrity: sha512-iYWt2dpdc5lKElhJEc89Zar3zq7+qXEKsTAlVqFxMtAxwCgDIQqTvds5b/W5FEvgfJsmXUN/Oib9zBr2urn6Fg==, tarball: file:projects/pod-ai-bot.tgz} version: 0.0.0 '@rush-temp/pod-analytics-collector@file:projects/pod-analytics-collector.tgz': @@ -4646,7 +4643,7 @@ packages: version: 0.0.0 '@rush-temp/pod-fulltext@file:projects/pod-fulltext.tgz': - resolution: {integrity: sha512-rDotFbiaTJUlFISEWa7+hd74cQWXGijcNrq3qeAItyw4A2PVKLSQusomUp5Mo1oP0wIurREXjiKCN1I7I27eAA==, tarball: file:projects/pod-fulltext.tgz} + resolution: {integrity: sha512-za9dXz8LTdV3Mwyr3Lbno6Ys7c0tKu2F9+QH0D75THvniEMYUK1mG4Pt/7hByNzTB1VqzIaozs6Id+DfXYqS2Q==, tarball: file:projects/pod-fulltext.tgz} version: 0.0.0 '@rush-temp/pod-github@file:projects/pod-github.tgz': @@ -4654,7 +4651,7 @@ packages: version: 0.0.0 '@rush-temp/pod-gmail@file:projects/pod-gmail.tgz': - resolution: {integrity: sha512-aYecRf97Yj23bi2BdWXg4DrH4oy3AB2UfrpbwNo6nvLhHJLsrcRgOe9zCTakZ7Yb+nDzsxYuOwZiDIr9aWtsyw==, tarball: file:projects/pod-gmail.tgz} + resolution: {integrity: sha512-HyyHadfmgT1OP7pdln0WJvK6cGPTYVVvxq21JBHKshu40WojCB6dLdAch/YrTO0aSS7UzluuLmYnnCn5JGvfcg==, tarball: file:projects/pod-gmail.tgz} version: 0.0.0 '@rush-temp/pod-hook@file:projects/pod-hook.tgz': @@ -4662,7 +4659,7 @@ packages: version: 0.0.0 '@rush-temp/pod-love@file:projects/pod-love.tgz': - resolution: {integrity: sha512-m+9dPMR45FJpQB+M5ka4PiJcTpV8COlQA5F36is2LkMLwnsDjcB94Sx+TFdIyjhVFpfB8l+t886mSPveHql4ag==, tarball: file:projects/pod-love.tgz} + resolution: {integrity: sha512-K9Hvwm8QesWBTSx/A+swmRDUOy0RY8y4tNJkrW85Zh5wMUxHzeYrX/tQL+usk+cRrvPCw9C8yag1gxsKqPkWIQ==, tarball: file:projects/pod-love.tgz} version: 0.0.0 '@rush-temp/pod-mail@file:projects/pod-mail.tgz': @@ -4670,11 +4667,11 @@ packages: version: 0.0.0 '@rush-temp/pod-print@file:projects/pod-print.tgz': - resolution: {integrity: sha512-i0Y9ShIeieNqCNMCXax1CZTStiDTw9M156m7fCDSrNS6VV4v2e5jy/HyNLtpyXz/ZRlOKndBdyiKsls+5pDL8g==, tarball: file:projects/pod-print.tgz} + resolution: {integrity: sha512-KNZrBeofhGFWQHMd1aPU7smmL50h/fGw4tXy7bEV2PcO14vphe2MyMfuWcTw1ksmeXFP6V3C2XxmgKFk+DT0dg==, tarball: file:projects/pod-print.tgz} version: 0.0.0 '@rush-temp/pod-server@file:projects/pod-server.tgz': - resolution: {integrity: sha512-MJ5TQobQy4kv67qy7qpQgz056fhaZw8Zs/NaXniFafG/xjstgWeIJCXNHDkKvdBm+wT6EdHgNLIN3BaRpMSDsA==, tarball: file:projects/pod-server.tgz} + resolution: {integrity: sha512-toWgesqQWTYgkwA6VPDiaJJc7ZmDkHFzNg/MbDjSACtmy0/endv3ioglyXYdoDo/m4IbFZOghMUJOZbQgQvRUA==, tarball: file:projects/pod-server.tgz} version: 0.0.0 '@rush-temp/pod-ses@file:projects/pod-ses.tgz': @@ -4682,7 +4679,7 @@ packages: version: 0.0.0 '@rush-temp/pod-sign@file:projects/pod-sign.tgz': - resolution: {integrity: sha512-CmOWryDSvF5qg0NTlkxT29gZP0qj0DimD25X2U6sQGtowtEytNvVmmg3cuSNTpwN+WR6+eli4JWyK6gathi+fQ==, tarball: file:projects/pod-sign.tgz} + resolution: {integrity: sha512-7ERrvIDW3r+NlglIYXW8kK50EiYLL9za+/EON9pW/8u0gkIaPIjEfL6yvdAh3euZ+khgPkzH+jrV57tu+jxT4w==, tarball: file:projects/pod-sign.tgz} version: 0.0.0 '@rush-temp/pod-stats@file:projects/pod-stats.tgz': @@ -4694,11 +4691,11 @@ packages: version: 0.0.0 '@rush-temp/pod-telegram@file:projects/pod-telegram.tgz': - resolution: {integrity: sha512-FAYbl5GmRCS3fOR5eUXKNS8KmwnwB+7ZZ68vDi5lfeH4hkJky/VdR5yq4dX5SzOB35e7hz9JWriDGxfjUcx6Wg==, tarball: file:projects/pod-telegram.tgz} + resolution: {integrity: sha512-CtJJaVgL0u4w9JUOAXdnAJqUnpSIlIpa2aY7OU3vZPR6hLqXt5pmDFvq3siQzkZ+27mJ6vc+IOBAaqbmumOkXg==, tarball: file:projects/pod-telegram.tgz} version: 0.0.0 '@rush-temp/pod-workspace@file:projects/pod-workspace.tgz': - resolution: {integrity: sha512-6wRqrWl0oZUzQ2CZBpPK09doQpYepraNAwCO3OcPDaRnDfQo0WBwLvfWOdDVW3REunCCh/DHMxGpuiLnaQX4KQ==, tarball: file:projects/pod-workspace.tgz} + resolution: {integrity: sha512-QrThk+6HwkGIdmj2tiPv9LTIOe5+QJChsG9DtIzmxaeyOjnhtpX2ROEuaVXL3oXl97EBoXX8fBsVpDMQ4i+pRw==, tarball: file:projects/pod-workspace.tgz} version: 0.0.0 '@rush-temp/postgres@file:projects/postgres.tgz': @@ -4818,7 +4815,7 @@ packages: version: 0.0.0 '@rush-temp/rekoni-service@file:projects/rekoni-service.tgz': - resolution: {integrity: sha512-drU6icafREaUYcSZfNn/CEYQLJC1ha6FkF7QrRcQ/TsuFaruapqmET0Ttqcr/vQaYemKF6SnshiKRGmeg+tcDA==, tarball: file:projects/rekoni-service.tgz} + resolution: {integrity: sha512-O8bBBE0Bw2m69G/KVLXgHnfa2WzWn9MG1CwklzSf6rkyVT/ltUA7WXwdPEA/ZFjLbkRpgd2RmcVRSERlAXDmRw==, tarball: file:projects/rekoni-service.tgz} version: 0.0.0 '@rush-temp/rekoni@file:projects/rekoni.tgz': @@ -4957,14 +4954,6 @@ packages: resolution: {integrity: sha512-5zQ2vbqHHFW1gdEWDqV8aDGq5u2I4z9qoD4TH0QbBaE5XITQrvJoZfK9WW+CceePEM2O+1clTu9w1KTBIEd5Hg==, tarball: file:projects/server-drive.tgz} version: 0.0.0 - '@rush-temp/server-fulltext-resources@file:projects/server-fulltext-resources.tgz': - resolution: {integrity: sha512-NebNMPNgIeG8N0qA5Q4ROcs5Jn/dmE0WX92uxw/qVUO/k8DLMxRsrFvyKrrMGtYgeivbxJPe6PEd3FijUAMfiw==, tarball: file:projects/server-fulltext-resources.tgz} - version: 0.0.0 - - '@rush-temp/server-fulltext@file:projects/server-fulltext.tgz': - resolution: {integrity: sha512-QYDVer6XKHqKih+CXofNkysDl9TafFIlN5sw4GdJjLT3/AJt7L2FeCKMuPgMtYQhihj2kodtBfNUbnSeLA8qJw==, tarball: file:projects/server-fulltext.tgz} - version: 0.0.0 - '@rush-temp/server-github-model@file:projects/server-github-model.tgz': resolution: {integrity: sha512-U7kynRwConx+S1nLfFluVkWJtFKCEMJRIHx/oN1OFy2zmIGSw1o7z1NIgEYAiYogj8zP1SPYoVKPQTSkSosKqw==, tarball: file:projects/server-github-model.tgz} version: 0.0.0 @@ -5038,7 +5027,7 @@ packages: version: 0.0.0 '@rush-temp/server-pipeline@file:projects/server-pipeline.tgz': - resolution: {integrity: sha512-egC06Gj9kZodIHe8m7g6K99KtOTXZyZJgPymrkDj+GGZX7+7Pd/5Nck0jT4i/Ufk1c0zOxEBXJE4+KTuvo/aew==, tarball: file:projects/server-pipeline.tgz} + resolution: {integrity: sha512-LWUTB3SnKZRoKrdx1ywH5lu/7jjYyD71rffj73ITM9s0Y6G9rSVNF7R4otEwOEmD+ZniJlJTz8h2vqfmEuALEg==, tarball: file:projects/server-pipeline.tgz} version: 0.0.0 '@rush-temp/server-preference@file:projects/server-preference.tgz': @@ -5318,7 +5307,7 @@ packages: version: 0.0.0 '@rush-temp/tool@file:projects/tool.tgz': - resolution: {integrity: sha512-1/Aodec9LySBThOZAeA3XJfP83lSVTVRFdlb8uNTORq1sHJxCADx9BliCh2dTzdhydW46wcAxMPeArhopkcN1Q==, tarball: file:projects/tool.tgz} + resolution: {integrity: sha512-zqh0++u5ETaUNlwkipP+8CeICqrFYEDBO4sAjFRyLPd1SHZbanUpQRAg58t4C+Z7q4Owqr149ee/hCGWPult3w==, tarball: file:projects/tool.tgz} version: 0.0.0 '@rush-temp/tracker-assets@file:projects/tracker-assets.tgz': @@ -10075,6 +10064,10 @@ packages: resolution: {integrity: sha512-40aUybvhH9t2h71ncA1/1SbtTNCVZHgsTsTgqPUxGWDmUDrXyDf2wMNQKEbdBjbf4AI+fQhbECNTV6lWxQKUzg==} engines: {node: '>= 0.4.0'} + kafkajs@2.2.4: + resolution: {integrity: sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==} + engines: {node: '>=14.0.0'} + katex@0.16.15: resolution: {integrity: sha512-yE9YJIEAk2aZ+FL/G8r+UGw0CTUzEA8ZFy6E+8tc3spHUKq3qBnzCkI1CQwGoI9atJhVyFPEypQsTY7mJ1Pi9w==} hasBin: true @@ -11170,8 +11163,8 @@ packages: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} - parse5@7.1.2: - resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} + parse5@7.2.1: + resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==} parseley@0.12.1: resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==} @@ -11481,8 +11474,8 @@ packages: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} - postgres-array@3.0.4: - resolution: {integrity: sha512-nAUSGfSDGOaOAEGwqsRY27GPOea7CNipJPOA7lPbdEpx5Kg3qzdP0AaWC5MlhTWV9s4hFX39nomVZ+C4tnGOJQ==} + postgres-array@3.0.2: + resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} engines: {node: '>=12'} postgres-bytea@1.0.0: @@ -16059,7 +16052,7 @@ snapshots: '@puppeteer/browsers@2.3.0': dependencies: - debug: 4.4.0 + debug: 4.3.5 extract-zip: 2.0.1 progress: 2.0.3 proxy-agent: 6.4.0 @@ -18853,6 +18846,32 @@ snapshots: - supports-color - ts-node + '@rush-temp/kafka@file:projects/kafka.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': + dependencies: + '@types/jest': 29.5.12 + '@types/node': 20.11.19 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + kafkajs: 2.2.4 + prettier: 3.2.5 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) + typescript: 5.7.3 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - babel-jest + - babel-plugin-macros + - esbuild + - node-notifier + - supports-color + - ts-node + '@rush-temp/kanban@file:projects/kanban.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@types/jest': 29.5.12 @@ -19179,17 +19198,17 @@ snapshots: dependencies: '@types/jest': 29.5.12 '@types/node': 20.11.19 - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) prettier: 3.2.5 - ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) - typescript: 5.7.3 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2) + typescript: 5.6.2 transitivePeerDependencies: - '@babel/core' - '@jest/types' @@ -19203,10 +19222,10 @@ snapshots: '@rush-temp/mail-resources@file:projects/mail-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@types/jest': 29.5.12 - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) @@ -19219,9 +19238,9 @@ snapshots: svelte-check: 3.6.9(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19) svelte-eslint-parser: 0.33.1(svelte@4.2.19) svelte-loader: 3.2.0(svelte@4.2.19) - svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.7.3) - ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) - typescript: 5.7.3 + svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.6.2) + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2) + typescript: 5.6.2 transitivePeerDependencies: - '@babel/core' - '@jest/types' @@ -19243,17 +19262,17 @@ snapshots: '@rush-temp/mail@file:projects/mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@types/jest': 29.5.12 - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) prettier: 3.2.5 - ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) - typescript: 5.7.3 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2) + typescript: 5.6.2 transitivePeerDependencies: - '@babel/core' - '@jest/types' @@ -19650,29 +19669,29 @@ snapshots: '@rush-temp/model-mail@file:projects/model-mail.tgz': dependencies: - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) prettier: 3.2.5 - typescript: 5.7.3 + typescript: 5.6.2 transitivePeerDependencies: - supports-color '@rush-temp/model-my-space@file:projects/model-my-space.tgz': dependencies: - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) prettier: 3.2.5 - typescript: 5.7.3 + typescript: 5.6.2 transitivePeerDependencies: - supports-color @@ -20017,20 +20036,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@rush-temp/model-server-fulltext@file:projects/model-server-fulltext.tgz': - dependencies: - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) - eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) - eslint-plugin-import: 2.29.1(eslint@8.56.0) - eslint-plugin-n: 15.7.0(eslint@8.56.0) - eslint-plugin-promise: 6.1.1(eslint@8.56.0) - prettier: 3.2.5 - typescript: 5.6.2 - transitivePeerDependencies: - - supports-color - '@rush-temp/model-server-gmail@file:projects/model-server-gmail.tgz': dependencies: '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) @@ -20601,17 +20606,17 @@ snapshots: dependencies: '@types/jest': 29.5.12 '@types/node': 20.11.19 - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) prettier: 3.2.5 - ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) - typescript: 5.7.3 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2) + typescript: 5.6.2 transitivePeerDependencies: - '@babel/core' - '@jest/types' @@ -20625,10 +20630,10 @@ snapshots: '@rush-temp/my-space-resources@file:projects/my-space-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@types/jest': 29.5.12 - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) @@ -20642,9 +20647,9 @@ snapshots: svelte-check: 3.6.9(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19) svelte-eslint-parser: 0.33.1(svelte@4.2.19) svelte-loader: 3.2.0(svelte@4.2.19) - svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.7.3) - ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) - typescript: 5.7.3 + svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.6.2) + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2) + typescript: 5.6.2 transitivePeerDependencies: - '@babel/core' - '@jest/types' @@ -20666,17 +20671,17 @@ snapshots: '@rush-temp/my-space@file:projects/my-space.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@types/jest': 29.5.12 - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) prettier: 3.2.5 - ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) - typescript: 5.7.3 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2) + typescript: 5.6.2 transitivePeerDependencies: - '@babel/core' - '@jest/types' @@ -23846,56 +23851,6 @@ snapshots: - supports-color - ts-node - '@rush-temp/server-fulltext-resources@file:projects/server-fulltext-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': - dependencies: - '@types/jest': 29.5.12 - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) - eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) - eslint-plugin-import: 2.29.1(eslint@8.56.0) - eslint-plugin-n: 15.7.0(eslint@8.56.0) - eslint-plugin-promise: 6.1.1(eslint@8.56.0) - jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) - prettier: 3.2.5 - ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2) - typescript: 5.6.2 - transitivePeerDependencies: - - '@babel/core' - - '@jest/types' - - '@types/node' - - babel-jest - - babel-plugin-macros - - esbuild - - node-notifier - - supports-color - - ts-node - - '@rush-temp/server-fulltext@file:projects/server-fulltext.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': - dependencies: - '@types/jest': 29.5.12 - '@types/node': 20.11.19 - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) - eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2) - eslint-plugin-import: 2.29.1(eslint@8.56.0) - eslint-plugin-n: 15.7.0(eslint@8.56.0) - eslint-plugin-promise: 6.1.1(eslint@8.56.0) - jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) - prettier: 3.2.5 - ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2) - typescript: 5.6.2 - transitivePeerDependencies: - - '@babel/core' - - '@jest/types' - - babel-jest - - babel-plugin-macros - - esbuild - - node-notifier - - supports-color - - ts-node - '@rush-temp/server-github-model@file:projects/server-github-model.tgz': dependencies: '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) @@ -28120,7 +28075,7 @@ snapshots: dependencies: '@types/node': 20.11.19 '@types/tough-cookie': 4.0.5 - parse5: 7.1.2 + parse5: 7.2.1 '@types/json-schema@7.0.15': {} @@ -32966,7 +32921,7 @@ snapshots: https-proxy-agent: 5.0.1 is-potential-custom-element-name: 1.0.1 nwsapi: 2.2.13 - parse5: 7.1.2 + parse5: 7.2.1 saxes: 6.0.0 symbol-tree: 3.2.4 tough-cookie: 4.1.4 @@ -33076,6 +33031,8 @@ snapshots: jwt-simple@0.5.6: {} + kafkajs@2.2.4: {} + katex@0.16.15: dependencies: commander: 8.3.0 @@ -34249,7 +34206,7 @@ snapshots: json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 - parse5@7.1.2: + parse5@7.2.1: dependencies: entities: 4.5.0 @@ -34369,7 +34326,7 @@ snapshots: dependencies: pg-int8: 1.0.1 pg-numeric: 1.0.2 - postgres-array: 3.0.4 + postgres-array: 3.0.2 postgres-bytea: 3.0.0 postgres-date: 2.1.0 postgres-interval: 3.0.0 @@ -34536,7 +34493,7 @@ snapshots: postgres-array@2.0.0: {} - postgres-array@3.0.4: {} + postgres-array@3.0.2: {} postgres-bytea@1.0.0: {} @@ -34784,7 +34741,7 @@ snapshots: dependencies: '@puppeteer/browsers': 2.3.0 chromium-bidi: 0.6.2(devtools-protocol@0.0.1312386) - debug: 4.4.0 + debug: 4.3.5 devtools-protocol: 0.0.1312386 ws: 8.18.0(bufferutil@4.0.8)(utf-8-validate@6.0.4) transitivePeerDependencies: @@ -35671,7 +35628,7 @@ snapshots: dependencies: component-emitter: 1.3.1 cookiejar: 2.1.4 - debug: 4.4.0 + debug: 4.3.5 fast-safe-stringify: 2.1.1 form-data: 4.0.0 formidable: 2.1.2 diff --git a/dev/.env b/dev/.env index 7828283edc..e0f5c481e2 100644 --- a/dev/.env +++ b/dev/.env @@ -3,3 +3,5 @@ MONGO_URL=mongodb://mongodb:27017?compressors=snappy DB_URL_PG=postgresql://postgres:example@postgres:5432 BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" BACKUP_BUCKET_NAME=dev-backups +DB_CR_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable +QUEUE_CONFIG=redpanda:9092 diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 91c0ed1641..05d44afadc 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -1,5 +1,5 @@ services: - stream: + stream: image: 'hardcoreeng/stream' extra_hosts: - 'huly.local:host-gateway' @@ -44,9 +44,66 @@ services: volumes: - cockroach_db:/cockroach/cockroach-data restart: unless-stopped + redpanda: + image: docker.redpanda.com/redpandadata/redpanda:v24.3.6 + extra_hosts: + - 'huly.local:host-gateway' + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092 + - --advertise-kafka-addr internal://redpanda:9092,external://localhost:19092 + - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082 + - --advertise-pandaproxy-addr internal://redpanda:8082,external://localhost:18082 + - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081 + - --rpc-addr redpanda:33145 + - --advertise-rpc-addr redpanda:33145 + - --mode dev-container + - --smp 1 + - --default-log-level=info + container_name: redpanda + volumes: + - redpanda:/var/lib/redpanda/data + ports: + - 18081:18081 + - 18082:18082 + - 19092:19092 + - 19644:9644 + healthcheck: + test: ['CMD', 'rpk', 'cluster', 'info', '-X', 'user=superuser', '-X', 'pass=secretpassword'] + interval: 10s + timeout: 5s + retries: 10 + redpanda_console: + image: docker.redpanda.com/redpandadata/console:v2.8.3 + extra_hosts: + - 'huly.local:host-gateway' + entrypoint: /bin/sh + command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console' + environment: + CONFIG_FILEPATH: /tmp/config.yml + CONSOLE_CONFIG_FILE: | + kafka: + brokers: ["redpanda:9092"] + schemaRegistry: + enabled: true + urls: ["http://redpanda:8081"] + redpanda: + adminApi: + enabled: true + urls: ["http://redpanda:9644"] + ports: + - 8000:8080 + depends_on: + - redpanda minio: image: 'minio/minio' command: server /data --address ":9000" --console-address ":9001" + extra_hosts: + - 'huly.local:host-gateway' + expose: + - 9000 + - 9001 ports: - 9000:9000 - 9001:9001 @@ -55,9 +112,10 @@ services: restart: unless-stopped elastic: image: 'elasticsearch:7.14.2' - command: | - /bin/sh -c "./bin/elasticsearch-plugin list | grep -q ingest-attachment || yes | ./bin/elasticsearch-plugin install --silent ingest-attachment; - /usr/local/bin/docker-entrypoint.sh eswrapper" + expose: + - 9200 + extra_hosts: + - 'huly.local:host-gateway' volumes: - elastic:/usr/share/elasticsearch/data ports: @@ -67,8 +125,6 @@ services: - BITNAMI_DEBUG=true - discovery.type=single-node - ES_JAVA_OPTS=-Xms1024m -Xmx1024m - - http.cors.enabled=true - - http.cors.allow-origin=http://localhost:8082 healthcheck: interval: 20s retries: 10 @@ -88,21 +144,20 @@ services: - ./branding.json:/var/cfg/branding.json environment: - ACCOUNT_PORT=3000 + - QUEUE_CONFIG=${QUEUE_CONFIG} - SERVER_SECRET=secret - - ADMIN_EMAILS=admin + - ADMIN_EMAILS=admin,${PLATFORM_ADMIN_EMAILS} - STATS_URL=http://huly.local:4900 - WORKSPACE_LIMIT_PER_USER=10000 - - DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable + - DB_URL=${DB_CR_URL} # - DB_URL=${MONGO_URL} # - DB_NS=account-2 - # Pass only one region to disallow selection for new workspaces.Ø + # Pass only one region to disallow selection for new workspaces. - REGION_INFO=|Mongo;cockroach|CockroachDB - # - REGION_INFO=cockroach|CockroachDB - TRANSACTOR_URL=ws://huly.local:3333,ws://huly.local:3332;;cockroach, - MAIL_URL= - STORAGE_CONFIG=${STORAGE_CONFIG} - FRONT_URL=http://huly.local:8087 - - RESERVED_DB_NAMES=telegram,gmail,github - MODEL_ENABLED=* - LAST_NAME_FIRST=true # - WS_LIVENESS_DAYS=1 @@ -133,15 +188,15 @@ services: environment: - WS_OPERATION=all+backup - SERVER_SECRET=secret + - QUEUE_CONFIG=${QUEUE_CONFIG} - DB_URL=${MONGO_URL} - STATS_URL=http://huly.local:4900 - MAIL_URL= - STORAGE_CONFIG=${STORAGE_CONFIG} - - RESERVED_DB_NAMES=telegram,gmail,github - MODEL_ENABLED=* + - REGION= - ACCOUNTS_URL=http://huly.local:3000 - BRANDING_PATH=/var/cfg/branding.json - # - PARALLEL=2 - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} # - INIT_WORKSPACE=staging-dev @@ -158,19 +213,18 @@ services: - ./branding.json:/var/cfg/branding.json environment: - WS_OPERATION=all+backup - - SERVER_SECRET=secret - - DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable - - STATS_URL=http://huly.local:4900 - - MAIL_URL= - REGION=cockroach + - SERVER_SECRET=secret + - QUEUE_CONFIG=${QUEUE_CONFIG} + - DB_URL=${DB_CR_URL} + - STATS_URL=http://huly.local:4900 - STORAGE_CONFIG=${STORAGE_CONFIG} - - RESERVED_DB_NAMES=telegram,gmail,github - MODEL_ENABLED=* - ACCOUNTS_URL=http://huly.local:3000 - BRANDING_PATH=/var/cfg/branding.json - # - PARALLEL=2 - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} + - MAIL_URL= # - INIT_WORKSPACE=staging-dev restart: unless-stopped collaborator: @@ -236,21 +290,19 @@ services: - minio - account - stats - # - apm-server ports: - 3333:3333 volumes: - ./branding.json:/var/cfg/branding.json environment: - # - SERVER_PROVIDER=uweb - # - UWS_HTTP_MAX_HEADERS_SIZE="32768" - - UV_THREADPOOL_SIZE=10 + - QUEUE_CONFIG=${QUEUE_CONFIG} + - REGION= - SERVER_PORT=3333 - SERVER_SECRET=secret - - ENABLE_COMPRESSION=false + - ENABLE_COMPRESSION=true - STATS_URL=http://huly.local:4900 - FULLTEXT_URL=http://huly.local:4700 - # - DB_URL=postgresql://postgres:example@postgres:5432 + # - DB_URL=${DB_CR_URL} - DB_URL=${MONGO_URL} - MONGO_URL=${MONGO_URL} - 'MONGO_OPTIONS={"appName": "transactor", "maxPoolSize": 10}' @@ -258,7 +310,6 @@ services: - METRICS_FILE=metrics.txt - STORAGE_CONFIG=${STORAGE_CONFIG} - FRONT_URL=http://huly.local:8087 - # - APM_SERVER_URL=http://apm-server:8200 - MAIL_URL='' - ACCOUNTS_URL=http://huly.local:3000 - LAST_NAME_FIRST=true @@ -274,52 +325,38 @@ services: - minio - account - stats - # - apm-server ports: - 3332:3332 volumes: - ./branding.json:/var/cfg/branding.json environment: - # - SERVER_PROVIDER=uweb - # - UWS_HTTP_MAX_HEADERS_SIZE="32768" - # - UV_THREADPOOL_SIZE=10 + - QUEUE_CONFIG=${QUEUE_CONFIG} - SERVER_PORT=3332 + - REGION=cockroach - SERVER_SECRET=secret - - ENABLE_COMPRESSION=false + - ENABLE_COMPRESSION=true - FULLTEXT_URL=http://huly.local:4702 - STATS_URL=http://huly.local:4900 - - DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable + - DB_URL=${DB_CR_URL} - METRICS_CONSOLE=false - METRICS_FILE=metrics.txt - STORAGE_CONFIG=${STORAGE_CONFIG} - FRONT_URL=http://huly.local:8087 - # - APM_SERVER_URL=http://apm-server:8200 - MAIL_URL='' - ACCOUNTS_URL=http://huly.local:3000 - LAST_NAME_FIRST=true - BRANDING_PATH=/var/cfg/branding.json - AI_BOT_URL=http://huly.local:4010 restart: unless-stopped - green: - image: hardcoreeng/green - extra_hosts: - - 'huly.local:host-gateway' - links: - - cockroach - - stats - ports: - - 6767:6767 - environment: - - PORT=6767 - - AUTH_TOKEN=secret - - STATS_URL=http://huly.local:4900 - - DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable - restart: unless-stopped rekoni: image: hardcoreeng/rekoni-service + extra_hosts: + - 'huly.local:host-gateway' restart: unless-stopped ports: - 4004:4004 + environment: + - STATS_URL=http://huly.local:4901 fulltext: image: hardcoreeng/fulltext extra_hosts: @@ -328,10 +365,13 @@ services: links: - elastic - mongodb + - rekoni ports: - 4700:4700 environment: + - PORT=4700 - SERVER_SECRET=secret + - QUEUE_CONFIG=${QUEUE_CONFIG} - DB_URL=${MONGO_URL} - FULLTEXT_DB_URL=http://huly.local:9200 - ELASTIC_INDEX_NAME=local_storage_index @@ -347,18 +387,22 @@ services: links: - elastic - cockroach + - rekoni ports: - 4702:4702 environment: - PORT=4702 + - REGION=cockroach - SERVER_SECRET=secret - - DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable + - QUEUE_CONFIG=${QUEUE_CONFIG} + - DB_URL=${DB_CR_URL} - FULLTEXT_DB_URL=http://huly.local:9200 - - ELASTIC_INDEX_NAME=local_storage_index + - ELASTIC_INDEX_NAME=local_storage_index # Same index for simplicity - STORAGE_CONFIG=${STORAGE_CONFIG} - STATS_URL=http://huly.local:4900 - REKONI_URL=http://huly.local:4004 - ACCOUNTS_URL=http://huly.local:3000 + - SERVER_SECRET=secret print: image: hardcoreeng/print extra_hosts: @@ -368,6 +412,7 @@ services: - 4005:4005 environment: - SECRET=secret + - QUEUE_CONFIG=${QUEUE_CONFIG} - STORAGE_CONFIG=${STORAGE_CONFIG} - STATS_URL=http://huly.local:4900 - ACCOUNTS_URL=http://huly.local:3000 @@ -384,6 +429,7 @@ services: environment: - SECRET=secret - MINIO_ENDPOINT=minio + - QUEUE_CONFIG=${QUEUE_CONFIG} - MINIO_ACCESS_KEY=minioadmin - ACCOUNTS_URL=http://huly.local:3000 - MINIO_SECRET_KEY=minioadmin @@ -391,21 +437,21 @@ services: - SERVICE_ID=sign-service - BRANDING_PATH=/var/cfg/branding.json - STATS_URL=http://huly.local:4900 -# analytics: -# image: hardcoreeng/analytics-collector -# extra_hosts: -# - 'huly.local:host-gateway' -# restart: unless-stopped -# ports: -# - 4017:4017 -# environment: -# - SECRET=secret -# - PORT=4017 -# - MONGO_URL=${MONGO_URL} -# - 'MONGO_OPTIONS={"appName":"analytics","maxPoolSize":1}' -# - SERVICE_ID=analytics-collector-service -# - ACCOUNTS_URL=http://huly.local:3000 -# - STATS_URL=http://huly.local:4900 + # analytics: + # image: hardcoreeng/analytics-collector + # extra_hosts: + # - 'huly.local:host-gateway' + # restart: unless-stopped + # ports: + # - 4017:4017 + # environment: + # - SECRET=secret + # - PORT=4017 + # - MONGO_URL=${MONGO_URL} + # - 'MONGO_OPTIONS={"appName":"analytics","maxPoolSize":1}' + # - SERVICE_ID=analytics-collector-service + # - ACCOUNTS_URL=http://huly.local:3000 + # - STATS_URL=http://huly.local:4900 aiBot: image: hardcoreeng/ai-bot ports: @@ -416,6 +462,7 @@ services: environment: - SERVER_SECRET=secret - MONGO_URL=${MONGO_URL} + - QUEUE_CONFIG=${QUEUE_CONFIG} - ACCOUNTS_URL=http://huly.local:3000 - STORAGE_CONFIG=${STORAGE_CONFIG} - FIRST_NAME=Jolie @@ -424,23 +471,24 @@ services: - AVATAR_PATH=./avatar.png - AVATAR_CONTENT_TYPE=.png - STATS_URL=http://huly.local:4900 -# - LOVE_ENDPOINT=http://huly.local:8096 -# - OPENAI_API_KEY=token -# telegram-bot: -# image: hardcoreeng/telegram-bot -# extra_hosts: -# - "huly.local:host-gateway" -# restart: unless-stopped -# environment: -# - PORT=4020 -# - BOT_TOKEN=token -# - MONGO_URL=${MONGO_URL} -# - MONGO_DB=telegram-bot -# - SECRET=secret -# - DOMAIN=domain -# - ACCOUNTS_URL=http://huly.local:3000 -# - SERVICE_ID=telegram-bot-service -# - STATS_URL=http://huly.local:4900 + # - LOVE_ENDPOINT=http://huly.local:8096 + # - OPENAI_API_KEY=token + # telegram-bot: + # image: hardcoreeng/telegram-bot + # extra_hosts: + # - "huly.local:host-gateway" + # restart: unless-stopped + # environment: + # - PORT=4020 + # - QUEUE_CONFIG=${QUEUE_CONFIG} + # - BOT_TOKEN=token + # - MONGO_URL=${MONGO_URL} + # - MONGO_DB=telegram-bot + # - SECRET=secret + # - DOMAIN=domain + # - ACCOUNTS_URL=http://huly.local:3000 + # - SERVICE_ID=telegram-bot-service + # - STATS_URL=http://huly.local:4900 export: image: hardcoreeng/export extra_hosts: @@ -465,3 +513,4 @@ volumes: files: elastic: cockroach_db: + redpanda: diff --git a/dev/tool/package.json b/dev/tool/package.json index afab15a9ff..8a4124f300 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -20,7 +20,7 @@ "docker:abuild": "docker build -t hardcoreeng/tool . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool", - "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4700 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js", + "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4700 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:9092' node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js", "run-local-cr": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 $TOOL_OPT ./bundle/bundle.js", "run-local-brk": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js", "run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js", @@ -171,6 +171,7 @@ "bufferutil": "^4.0.8", "utf-8-validate": "^6.0.4", "msgpackr": "^1.11.2", - "msgpackr-extract": "^3.0.3" + "msgpackr-extract": "^3.0.3", + "@hcengineering/kafka": "^0.6.0" } } diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts index 64c97a5c21..90e58aea9b 100644 --- a/dev/tool/src/clean.ts +++ b/dev/tool/src/clean.ts @@ -26,7 +26,6 @@ import core, { type CollaborativeDoc, type Client as CoreClient, DOMAIN_BENCHMARK, - DOMAIN_DOC_INDEX_STATE, DOMAIN_MIGRATION, DOMAIN_MODEL, DOMAIN_STATUS, @@ -1218,17 +1217,6 @@ export async function removeDuplicateIds ( // await updateArrRefs(txop, newId, doc) -// // update docIndexState -// const docIndexState = await client.findOne(core.class.DocIndexState, { doc: doc._id }) -// if (docIndexState !== undefined) { -// const { _id, space, modifiedBy, modifiedOn, createdBy, createdOn, _class, ...data } = docIndexState -// await txop.createDoc(docIndexState._class, docIndexState.space, { -// ...data, -// removed: false -// }) -// await txop.update(docIndexState, { removed: true, needIndex: true }) -// } - // if (domain !== DOMAIN_MODEL) { // const raw = await db.collection(domain).findOne({ _id: doc._id }) // await db.collection(domain).insertOne({ diff --git a/dev/tool/src/elastic.ts b/dev/tool/src/elastic.ts index 539f316037..b124eaceff 100644 --- a/dev/tool/src/elastic.ts +++ b/dev/tool/src/elastic.ts @@ -14,8 +14,6 @@ // import { Client as ElasticClient } from '@elastic/elasticsearch' -import core, { DOMAIN_DOC_INDEX_STATE } from '@hcengineering/core' -import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' import { type StorageAdapter } from '@hcengineering/server-core' export async function rebuildElastic ( @@ -24,17 +22,6 @@ export async function rebuildElastic ( storageAdapter: StorageAdapter, elasticUrl: string ): Promise { - const client = getMongoClient(mongoUrl) - try { - const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, dataId) - await db - .collection(DOMAIN_DOC_INDEX_STATE) - .updateMany({ _class: core.class.DocIndexState }, { $set: { elastic: false } }) - } finally { - client.close() - } - await dropElastic(elasticUrl, dataId) } diff --git a/dev/tool/src/fulltext.ts b/dev/tool/src/fulltext.ts deleted file mode 100644 index fc31673916..0000000000 --- a/dev/tool/src/fulltext.ts +++ /dev/null @@ -1,33 +0,0 @@ -// -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { type MeasureContext } from '@hcengineering/core' - -export async function reindexWorkspace (ctx: MeasureContext, fulltextUrl: string, token: string): Promise { - try { - const res = await fetch(fulltextUrl + '/api/v1/reindex', { - method: 'PUT', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ token }) - }) - if (!res.ok) { - throw new Error(`HTTP Error ${res.status} ${res.statusText}`) - } - } catch (err: any) { - ctx.error('failed to reset index', { err }) - } -} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 898533dfd1..bbd1941513 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -47,6 +47,7 @@ import { import serverToken, { generateToken } from '@hcengineering/server-token' import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' +import { getPlatformQueue } from '@hcengineering/kafka' import { buildStorageFromConfig, createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { program, type Command } from 'commander' import { updateField } from './workspace' @@ -81,11 +82,15 @@ import { createPostgresTxAdapter, shutdownPostgres } from '@hcengineering/postgres' -import type { StorageAdapter } from '@hcengineering/server-core' +import { + QueueTopic, + workspaceEvents, + type QueueWorkspaceMessage, + type StorageAdapter +} from '@hcengineering/server-core' import { getAccountDBUrl, getMongoDBUrl } from './__start' // import { fillGithubUsers, fixAccountEmails, renameAccount } from './account' import { changeConfiguration } from './configuration' -import { reindexWorkspace } from './fulltext' import { moveAccountDbFromMongoToPG } from './db' import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils' @@ -363,6 +368,13 @@ export function devTool ( progress: 100 }) + const wsProducer = getPlatformQueue('tool', cmd.region).createProducer( + toolCtx, + QueueTopic.Workspace + ) + await wsProducer.send(res.workspaceUuid, [workspaceEvents.created()]) + await wsProducer.close() + console.log('create-workspace done') }) }) @@ -442,6 +454,12 @@ export function devTool ( }) console.log(metricsToString(measureCtx.metrics, 'upgrade', 60)) + const wsProducer = getPlatformQueue('tool', info.region).createProducer( + toolCtx, + QueueTopic.Workspace + ) + await wsProducer.send(info.uuid, [workspaceEvents.upgraded()]) + await wsProducer.close() console.log('upgrade-workspace done') }) }) @@ -1121,6 +1139,12 @@ export function devTool ( storageAdapter: workspaceStorage, historyFile: cmd.historyFile }) + const wsProducer = getPlatformQueue('tool', ws.region).createProducer( + toolCtx, + QueueTopic.Workspace + ) + await wsProducer.send(ws.uuid, [workspaceEvents.fullReindex()]) + await wsProducer.close() await workspaceStorage?.close() }) } @@ -2063,7 +2087,12 @@ export function devTool ( } console.log('reindex workspace', workspace) - await reindexWorkspace(toolCtx, fulltextUrl, getToolToken(ws?.uuid)) + const wsProducer = getPlatformQueue('tool', ws.region).createProducer( + toolCtx, + QueueTopic.Workspace + ) + await wsProducer.send(ws.uuid, [workspaceEvents.fullReindex()]) + await wsProducer.close() console.log('done', workspace) }) }) @@ -2367,6 +2396,15 @@ export function devTool ( // }) // }) + program + .command('queue-init-topics') + .description('create required kafka topics') + .option('-tx ', 'Number of TX partitions', '5') + .action(async (cmd: { tx: string }) => { + const queue = getPlatformQueue('tool') + await queue.createTopics(parseInt(cmd.tx ?? '1')) + }) + extendProgram?.(program) program.parse(process.argv) diff --git a/models/all/package.json b/models/all/package.json index 2fa3d75e4d..13e4547502 100644 --- a/models/all/package.json +++ b/models/all/package.json @@ -112,7 +112,6 @@ "@hcengineering/model-server-ai-bot": "^0.6.0", "@hcengineering/model-ai-bot": "^0.6.0", "@hcengineering/model-recorder": "^0.6.0", - "@hcengineering/model-server-fulltext": "^0.6.0", "@hcengineering/model-test-management": "^0.6.0", "@hcengineering/model-survey": "^0.6.0", "@hcengineering/model-presence": "^0.6.0", diff --git a/models/all/src/index.ts b/models/all/src/index.ts index 4299b58cee..8849751882 100644 --- a/models/all/src/index.ts +++ b/models/all/src/index.ts @@ -110,7 +110,6 @@ import { serverDocumentsId, createModel as serverDocumentsModel } from '@hcengineering/model-server-controlled-documents' -import { serverFulltextId, createModel as serverFulltextModel } from '@hcengineering/model-server-fulltext' import { surveyId, createModel as surveyModel } from '@hcengineering/model-survey' import { presenceId, createModel as presenceModel } from '@hcengineering/model-presence' import processes, { processId, createModel as processModel } from '@hcengineering/model-process' @@ -493,7 +492,6 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[ [serverTrainingModel, serverTrainingId], [serverDocumentsModel, serverDocumentsId], [serverAiBotModel, serverAiBotId], - [serverFulltextModel, serverFulltextId], [serverProcessModel, serverProcessId] ] diff --git a/models/controlled-documents/src/migration.ts b/models/controlled-documents/src/migration.ts index 5ee2dd15b0..b0f80703c6 100644 --- a/models/controlled-documents/src/migration.ts +++ b/models/controlled-documents/src/migration.ts @@ -26,8 +26,6 @@ import { type Class, type Data, type Doc, - type DocIndexState, - DOMAIN_DOC_INDEX_STATE, DOMAIN_SEQUENCE, DOMAIN_TX, generateId, @@ -446,14 +444,6 @@ async function migrateDocumentMetaInternalCode (client: MigrationClient): Promis } await client.bulk(DOMAIN_DOCUMENTS, operations) - await client.update( - DOMAIN_DOC_INDEX_STATE, - { - _id: { $in: Array.from(updatedIds) as any }, - objectClass: documents.class.DocumentMeta - }, - { $set: { needIndex: true } } - ) } export const documentsOperation: MigrateOperation = { diff --git a/models/core/src/core.ts b/models/core/src/core.ts index 4b63730959..5e83792388 100644 --- a/models/core/src/core.ts +++ b/models/core/src/core.ts @@ -26,11 +26,9 @@ import { type Configuration, type ConfigurationElement, type Doc, - type DocIndexState, type Domain, DOMAIN_BLOB, DOMAIN_CONFIGURATION, - DOMAIN_DOC_INDEX_STATE, DOMAIN_MIGRATION, DOMAIN_MODEL, DOMAIN_RELATION, @@ -339,22 +337,6 @@ export class TPluginConfiguration extends TDoc implements PluginConfiguration { beta!: boolean } -@Model(core.class.DocIndexState, core.class.Doc, DOMAIN_DOC_INDEX_STATE) -export class TDocIndexState extends TDoc implements DocIndexState { - @Prop(TypeRef(core.class.Class), core.string.Class) - @Index(IndexKind.Indexed) - @Hidden() - objectClass!: Ref> - - @Prop(TypeBoolean(), getEmbeddedLabel('Removed')) - @Hidden() - removed!: boolean - - @Prop(TypeBoolean(), getEmbeddedLabel('NeedIndexing')) - @Hidden() - needIndex!: boolean -} - @Model(core.class.FullTextSearchContext, core.class.Doc, DOMAIN_MODEL) export class TFullTextSearchContext extends TDoc implements FullTextSearchContext { toClass!: Ref>> diff --git a/models/core/src/index.ts b/models/core/src/index.ts index 8e08f4602c..9b99484b1e 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -17,7 +17,6 @@ import { DOMAIN_BENCHMARK, DOMAIN_BLOB, DOMAIN_CONFIGURATION, - DOMAIN_DOC_INDEX_STATE, DOMAIN_MIGRATION, DOMAIN_SPACE, DOMAIN_STATUS, @@ -38,7 +37,6 @@ import { TConfiguration, TConfigurationElement, TDoc, - TDocIndexState, TDomainIndexConfiguration, TEnum, TEnumOf, @@ -149,7 +147,6 @@ export function createModel (builder: Builder): void { TEnum, TTypeAny, TTypeRelatedDocument, - TDocIndexState, TFullTextSearchContext, TConfiguration, TConfigurationElement, @@ -262,30 +259,6 @@ export function createModel (builder: Builder): void { ] }) - builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, { - domain: DOMAIN_DOC_INDEX_STATE, - indexes: [ - { - keys: { needIndex: 1, objectClass: 1 } - } - ], - disabled: [ - { attachedToClass: 1 }, - { stages: 1 }, - { space: 1 }, - { _class: 1 }, - { needIndex: 1 }, - { objectClass: 1 }, - { _class: 1 }, - { attachedTo: 1 }, - { modifiedBy: 1 }, - { modifiedOn: 1 }, - { createdBy: 1 }, - { createdBy: -1 }, - { createdOn: -1 } - ] - }) - builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { toClass: core.class.Space, childProcessingAllowed: false diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 263cf899af..3b839e72af 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -306,7 +306,7 @@ export function getAccountsFromTxes (accTxes: TxCUD[]): any { return Object.values(byAccounts) .map((txes) => TxProcessor.buildDoc2Doc(txes)) - .filter((it) => it !== undefined) + .filter((it) => it != null) } export async function getSocialKeyByOldAccount (client: MigrationClient): Promise> { diff --git a/models/server-fulltext/package.json b/models/server-fulltext/package.json deleted file mode 100644 index e4dcf66b17..0000000000 --- a/models/server-fulltext/package.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "name": "@hcengineering/model-server-fulltext", - "version": "0.6.0", - "main": "lib/index.js", - "svelte": "src/index.ts", - "types": "types/index.d.ts", - "author": "Anticrm Platform Contributors", - "template": "@hcengineering/model-package", - "license": "EPL-2.0", - "scripts": { - "build": "compile", - "build:watch": "compile", - "format": "format src", - "_phase:build": "compile transpile src", - "_phase:format": "format src", - "_phase:validate": "compile validate" - }, - "devDependencies": { - "@hcengineering/platform-rig": "^0.6.0", - "@typescript-eslint/eslint-plugin": "^6.11.0", - "eslint-plugin-import": "^2.26.0", - "eslint-plugin-promise": "^6.1.1", - "eslint-plugin-n": "^15.4.0", - "eslint": "^8.54.0", - "@typescript-eslint/parser": "^6.11.0", - "eslint-config-standard-with-typescript": "^40.0.0", - "prettier": "^3.1.0", - "typescript": "^5.3.3" - }, - "dependencies": { - "@hcengineering/core": "^0.6.32", - "@hcengineering/model": "^0.6.11", - "@hcengineering/platform": "^0.6.11", - "@hcengineering/server-fulltext": "^0.6.0", - "@hcengineering/server-core": "^0.6.1" - } -} diff --git a/models/server-fulltext/src/index.ts b/models/server-fulltext/src/index.ts deleted file mode 100644 index 0a679bd55b..0000000000 --- a/models/server-fulltext/src/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { type Builder } from '@hcengineering/model' - -import core from '@hcengineering/core' -import serverCore from '@hcengineering/server-core' -import serverFulltext from '@hcengineering/server-fulltext' -export { serverFulltextId } from '@hcengineering/server-fulltext' -export function createModel (builder: Builder): void { - builder.createDoc(serverCore.class.Trigger, core.space.Model, { - trigger: serverFulltext.trigger.OnChange - }) -} diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index 4ab84c423c..d8c1bb7ac2 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -421,8 +421,6 @@ export interface TransientConfiguration extends Class { */ export const DOMAIN_BLOB = 'blob' as Domain -export const DOMAIN_DOC_INDEX_STATE = 'doc-index-state' as Domain - /** * @public */ @@ -581,17 +579,6 @@ export function versionToString (version: Version | Data): string { return `${version?.major}.${version?.minor}.${version?.patch}` } -/** - * @public - * - * Define status for full text indexing - */ -export interface DocIndexState extends Doc { - objectClass: Ref> - needIndex: boolean - removed: boolean -} - /** * @public */ diff --git a/packages/core/src/component.ts b/packages/core/src/component.ts index 3b2e32e59d..4e27228c1c 100644 --- a/packages/core/src/component.ts +++ b/packages/core/src/component.ts @@ -28,7 +28,6 @@ import type { Configuration, ConfigurationElement, Doc, - DocIndexState, DomainIndexConfiguration, Enum, EnumOf, @@ -149,7 +148,6 @@ export default plugin(coreId, { PluginConfiguration: '' as Ref>, UserStatus: '' as Ref>, TypeRelatedDocument: '' as Ref>>, - DocIndexState: '' as Ref>, DomainIndexConfiguration: '' as Ref>, Configuration: '' as Ref>, diff --git a/packages/core/src/measurements/metrics.ts b/packages/core/src/measurements/metrics.ts index 0fe3267f3d..fc6bfe7852 100644 --- a/packages/core/src/measurements/metrics.ts +++ b/packages/core/src/measurements/metrics.ts @@ -198,6 +198,7 @@ function printMetricsChildren (params: Record, offset: number, if (Object.keys(params).length > 0) { r += '\n' + toLen('', ' ', offset) r += Object.entries(params) + .filter((it) => it[1].value > 0.1) .map(([k, vv]) => toString(k, vv, offset, length)) .join('\n' + toLen('', ' ', offset)) } @@ -211,11 +212,13 @@ function printMetricsParams ( ): string { let r = '' const joinP = (key: string, data: Record): string[] => { - return Object.entries(data).map(([k, vv]) => - `${toLen('', ' ', offset)}${toLen(key + '=' + k, '-', length - offset)}: avg ${ - vv.value / (vv.operations > 0 ? vv.operations : 1) - } total: ${vv.value} ops: ${vv.operations}`.trim() - ) + return Object.entries(data) + .filter((it) => it[1].value >= 0.1) + .map(([k, vv]) => + `${toLen('', ' ', offset)}${toLen(key + '=' + k, '-', length - offset)}: avg ${ + Math.round((vv.value / (vv.operations > 0 ? vv.operations : 1)) * 100) / 100 + } total: ${Math.round(vv.value * 100) / 100} ops: ${vv.operations}`.trim() + ) } const joinParams = Object.entries(params).reduce((p, c) => [...p, ...joinP(c[0], c[1])], []) if (Object.keys(joinParams).length > 0) { @@ -227,8 +230,8 @@ function printMetricsParams ( function toString (name: string, m: Metrics, offset: number, length: number): string { let r = `${toLen('', ' ', offset)}${toLen(name, '-', length - offset)}: avg ${ - m.value / (m.operations > 0 ? m.operations : 1) - } total: ${m.value} ops: ${m.operations}`.trim() + Math.round((m.value / (m.operations > 0 ? m.operations : 1)) * 100) / 100 + } total: ${Math.round(m.value * 100) / 100} ops: ${m.operations}`.trim() r += printMetricsParams(m.params, offset + 4, length) r += printMetricsChildren(m.measurements, offset + 4, length) return r diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 0286defb6d..be3bcb5f49 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import type { Account, AccountUuid, Doc, DocIndexState, Domain, PersonId, Ref } from './classes' +import type { Account, AccountUuid, Doc, Domain, PersonId, Ref } from './classes' import { MeasureContext } from './measurements' import { DocumentQuery, FindOptions } from './storage' import type { DocumentUpdate, Tx } from './tx' @@ -52,7 +52,6 @@ export interface SessionData { workspace: WorkspaceIds branding: Branding | null socialStringsToUsers: Map - fulltextUpdates?: Map, DocIndexState> asyncRequests?: (() => Promise)[] } diff --git a/packages/core/src/tx.ts b/packages/core/src/tx.ts index 63ed17bfc4..3be9c812ba 100644 --- a/packages/core/src/tx.ts +++ b/packages/core/src/tx.ts @@ -15,7 +15,6 @@ import type { KeysByType } from 'simplytyped' import type { - PersonId, Arr, AttachedDoc, Class, @@ -23,6 +22,7 @@ import type { Doc, Domain, Mixin, + PersonId, PropertyType, Ref, Space, @@ -399,10 +399,16 @@ export abstract class TxProcessor implements WithTx { return rawDoc } - static buildDoc2Doc(txes: Tx[]): D | undefined { + static buildDoc2Doc(txes: Tx[]): D | undefined | null { let doc: Doc + const deleteTx = txes.find((tx) => tx._class === core.class.TxRemoveDoc) + if (deleteTx !== undefined) { + return null + } const createTx = txes.find((tx) => tx._class === core.class.TxCreateDoc) - if (createTx === undefined) return + if (createTx === undefined) { + return + } doc = TxProcessor.createDoc2Doc(createTx as TxCreateDoc) for (const tx of txes) { if (tx._class === core.class.TxUpdateDoc) { diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index 70d3ab09bf..2758411a21 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -27,7 +27,6 @@ import { Doc, DocData, DOMAIN_BLOB, - DOMAIN_DOC_INDEX_STATE, DOMAIN_MODEL, DOMAIN_TRANSIENT, FullTextSearchContext, @@ -673,7 +672,6 @@ export function isClassIndexable ( } if ( - domain === DOMAIN_DOC_INDEX_STATE || domain === DOMAIN_TX || domain === DOMAIN_MODEL || domain === DOMAIN_BLOB || diff --git a/plugins/board-resources/src/components/editor/CardActions.svelte b/plugins/board-resources/src/components/editor/CardActions.svelte index 03d8e7c15d..04e645160b 100644 --- a/plugins/board-resources/src/components/editor/CardActions.svelte +++ b/plugins/board-resources/src/components/editor/CardActions.svelte @@ -42,7 +42,7 @@ const txes = await client.findAll(core.class.Tx, { objectId: value._id }) while (txes.length) { const doc = TxProcessor.buildDoc2Doc(txes) - if (doc === undefined) { + if (doc == null) { break } targetState = store.byId.get(doc.status) diff --git a/plugins/view-resources/src/inference.ts b/plugins/view-resources/src/inference.ts deleted file mode 100644 index fbe9d0991f..0000000000 --- a/plugins/view-resources/src/inference.ts +++ /dev/null @@ -1,33 +0,0 @@ -import core, { type Class, type Client, type Doc, type DocIndexState, type Ref } from '@hcengineering/core' -import { writable } from 'svelte/store' - -export interface InferenceFocus { - objectId: Ref - objectClass: Ref> - - state: DocIndexState -} -/** - * @public - */ -export const inferenceFocusStore = writable(undefined) - -/** - * @public - */ -export async function updateFocus (client: Client, doc?: { _id: Ref, _class: Ref> }): Promise { - if (doc === undefined) { - inferenceFocusStore.set(undefined) - return - } - - const state = await client.findOne(core.class.DocIndexState, { - _id: doc._id as Ref, - objectClass: doc._class - }) - if (state !== undefined) { - inferenceFocusStore.update(() => { - return { objectId: doc._id, objectClass: doc._class, state } - }) - } -} diff --git a/plugins/workbench-resources/src/components/ServerManagerGeneral.svelte b/plugins/workbench-resources/src/components/ServerManagerGeneral.svelte index 4b752d0b17..53f709db1f 100644 --- a/plugins/workbench-resources/src/components/ServerManagerGeneral.svelte +++ b/plugins/workbench-resources/src/components/ServerManagerGeneral.svelte @@ -35,7 +35,14 @@ let profiling = false async function fetchStats (time: number): Promise { - await fetch(endpoint + `/api/v1/profiling?token=${token}`, {}) + await fetch(endpoint + '/api/v1/profiling', { + method: 'GET', + keepalive: true, + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + token + } + }) .then(async (json) => { data = await json.json() profiling = data?.profiling ?? false diff --git a/pods/fulltext/package.json b/pods/fulltext/package.json index 12d851b97a..6fa4bf7e1b 100644 --- a/pods/fulltext/package.json +++ b/pods/fulltext/package.json @@ -75,6 +75,7 @@ "@hcengineering/server-client": "^0.6.0", "@hcengineering/server-storage": "^0.6.0", "@hcengineering/postgres": "^0.6.0", - "@hcengineering/mongo": "^0.6.1" + "@hcengineering/mongo": "^0.6.1", + "@hcengineering/kafka": "^0.6.0" } } diff --git a/pods/fulltext/src/index.ts b/pods/fulltext/src/index.ts index 4c6ee643d7..84956e5a24 100644 --- a/pods/fulltext/src/index.ts +++ b/pods/fulltext/src/index.ts @@ -24,6 +24,7 @@ import { createRekoniAdapter, type FulltextDBConfiguration } from '@hcengineerin import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { readFileSync } from 'fs' import { startIndexer } from './server' +import { getPlatformQueue } from '@hcengineering/kafka' const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[] @@ -103,7 +104,10 @@ if (accountsUrl === undefined) { const storageConfig: StorageConfiguration = storageConfigFromEnv() const externalStorage = buildStorageFromConfig(storageConfig) +const queue = getPlatformQueue('fulltext') + const onClose = startIndexer(metricsContext, { + queue, model, config, externalStorage, diff --git a/pods/fulltext/src/server.ts b/pods/fulltext/src/server.ts index ad79b53b2b..194043a236 100644 --- a/pods/fulltext/src/server.ts +++ b/pods/fulltext/src/server.ts @@ -4,22 +4,24 @@ import type { Class, Doc, DocumentQuery, + Domain, IndexingUpdateEvent, MeasureContext, Ref, SearchOptions, SearchQuery, Tx, + TxCUD, TxWorkspaceEvent, - WorkspaceUuid, - WorkspaceIds + WorkspaceIds, + WorkspaceUuid } from '@hcengineering/core' import core, { - DOMAIN_DOC_INDEX_STATE, generateId, Hierarchy, ModelDb, systemAccountUuid, + TxProcessor, WorkspaceEvent } from '@hcengineering/core' import { @@ -44,15 +46,22 @@ import { setDBExtraOptions, shutdownPostgres } from '@hcengineering/postgres' -import serverClientPlugin, { getTransactorEndpoint, getAccountClient } from '@hcengineering/server-client' +import serverClientPlugin, { getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client' import serverCore, { createContentAdapter, createPipeline, + QueueTopic, + QueueWorkspaceEvent, + workspaceEvents, + type ConsumerControl, type ContentTextAdapter, type FullTextAdapter, type MiddlewareCreator, type Pipeline, type PipelineContext, + type PlatformQueue, + type QueueWorkspaceMessage, + type QueueWorkspaceReindexMessage, type StorageAdapter } from '@hcengineering/server-core' import { FullTextIndexPipeline, searchFulltext, type FulltextDBConfiguration } from '@hcengineering/server-indexer' @@ -66,12 +75,31 @@ import { setAdapterSecurity, sharedPipelineContextVars } from '@hcengineering/server-pipeline' -import serverToken, { decodeToken, generateToken, type Token } from '@hcengineering/server-token' +import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' import cors from '@koa/cors' import Koa from 'koa' import bodyParser from 'koa-bodyparser' import Router from 'koa-router' +function fulltextModelFilter (h: Hierarchy, model: Tx[]): Tx[] { + const allowedClasess: Ref>[] = [ + core.class.Class, + core.class.Attribute, + core.class.Mixin, + core.class.Type, + core.class.Status, + core.class.Permission, + core.class.Space, + core.class.Tx, + core.class.FullTextSearchContext + ] + return model.filter( + (it) => + TxProcessor.isExtendsCUD(it._class) && + allowedClasess.some((cl) => h.isDerived((it as TxCUD).objectClass, cl)) + ) +} + class WorkspaceIndexer { fulltext!: FullTextIndexPipeline pipeline!: Pipeline @@ -100,7 +128,7 @@ class WorkspaceIndexer { ContextNameMiddleware.create, DomainFindMiddleware.create, DBAdapterInitMiddleware.create, - ModelMiddleware.create(model), + ModelMiddleware.create(model, fulltextModelFilter), // TODO: Add filtration of only class structure and FullTextSearchContext DBAdapterMiddleware.create(dbConf) ] @@ -153,40 +181,37 @@ class WorkspaceIndexer { } // Send tx to pipeline // TODO: Fix me - void fetch(transactorEndpoint + `/api/v1/broadcast?token=${token}&workspace=${workspace.uuid}`, { + void fetch(transactorEndpoint + `/api/v1/broadcast?workspace=${workspace.uuid}`, { method: 'PUT', + keepalive: true, + headers: { + Authorization: `Bearer ${token}` + }, body: JSON.stringify(tx) }) - }, - async () => { - const helper = result.pipeline.context.adapterManager?.domainHelper - if (helper !== undefined) { - const dhelper = result.pipeline.context.adapterManager?.getAdapter(DOMAIN_DOC_INDEX_STATE, true).helper?.() - if (dhelper !== undefined) { - // Force creation of indexes for search domain - await helper.checkDomain(ctx, DOMAIN_DOC_INDEX_STATE, 10000, dhelper) - } - } } ) - await result.fulltext.startIndexing(() => { - result.lastUpdate = Date.now() - }) return result } - async reindex (onlyDrop: boolean): Promise { - await this.fulltext.cancel() - await this.fulltext.clearIndex(onlyDrop) - if (!onlyDrop) { - await this.fulltext.startIndexing(() => { - this.lastUpdate = Date.now() - }) - } + async reindex ( + ctx: MeasureContext, + domain: Domain, + classes: Ref>[], + control?: ConsumerControl + ): Promise { + await this.fulltext.reindex(ctx, domain, classes, control) + } + + async dropWorkspace (): Promise { + await this.fulltext.dropWorkspace() + } + + async getIndexClassess (): Promise<{ domain: Domain, classes: Ref>[] }[]> { + return await this.fulltext.getIndexClassess() } async close (): Promise { - await this.fulltext.cancel() await this.pipeline.close() } } @@ -229,6 +254,7 @@ process.on('exit', () => { export async function startIndexer ( ctx: MeasureContext, opt: { + queue: PlatformQueue model: Tx[] dbURL: string config: FulltextDBConfiguration @@ -292,16 +318,89 @@ export async function startIndexer ( } }, closeTimeout) // Every 5 minutes we should close unused indexes. + const workspaceProducer = opt.queue.createProducer(ctx, QueueTopic.Workspace) + + const workspaceConsumer = opt.queue.createConsumer( + ctx, + QueueTopic.Workspace, + opt.queue.getClientId(), + async (msg, control) => { + for (const m of msg) { + ctx.info('workspace message', { message: m }) + const ws = m.id as WorkspaceUuid + + for (const mm of m.value) { + if ( + mm.type === QueueWorkspaceEvent.Created || + mm.type === QueueWorkspaceEvent.Restored || + mm.type === QueueWorkspaceEvent.FullReindex + ) { + const indexer = await getIndexer(ctx, ws, generateToken(systemAccountUuid, ws), true) + if (indexer !== undefined) { + await indexer.dropWorkspace() // TODO: Add heartbeat + const classes = await indexer.getIndexClassess() + await workspaceProducer.send( + ws, + classes.map((it) => workspaceEvents.reindex(it.domain, it.classes)) + ) + } + } else if ( + mm.type === QueueWorkspaceEvent.Deleted || + mm.type === QueueWorkspaceEvent.Archived || + mm.type === QueueWorkspaceEvent.ClearIndex + ) { + const token = generateToken(systemAccountUuid, ws) + const accountClient = getAccountClient(token) + const workspaceInfo = await accountClient.getWorkspaceInfo(false) + if (workspaceInfo !== undefined) { + if (workspaceInfo.dataId != null) { + await fulltextAdapter.clean(ctx, workspaceInfo.dataId as unknown as WorkspaceUuid) + } + await fulltextAdapter.clean(ctx, workspaceInfo.uuid) + } + } else if (mm.type === QueueWorkspaceEvent.Reindex) { + const indexer = await getIndexer(ctx, ws, generateToken(systemAccountUuid, ws), true) + const mmd = mm as QueueWorkspaceReindexMessage + await indexer?.reindex(ctx, mmd.domain, mmd.classes, control) + } + } + } + } + ) + + let txInformer: any + let txMessages: number = 0 + const txConsumer = opt.queue.createConsumer>( + ctx, + QueueTopic.Tx, + opt.queue.getClientId(), + async (msg, control) => { + clearTimeout(txInformer) + txInformer = setTimeout(() => { + ctx.info('tx message', { count: txMessages }) + txMessages = 0 + }, 5000) + + txMessages += msg.length + for (const m of msg) { + const ws = m.id as WorkspaceUuid + + const indexer = await getIndexer(ctx, ws, generateToken(systemAccountUuid, ws), true) + await indexer?.fulltext.processDocuments(ctx, m.value, control) + } + } + ) + async function getIndexer ( ctx: MeasureContext, workspace: WorkspaceUuid, - token: string, + token: string | undefined, create: boolean = false ): Promise { - const accountClient = getAccountClient(token) - const workspaceInfo = await accountClient.getWorkspaceInfo(false) let idx = indexers.get(workspace) if (idx === undefined && create) { + const accountClient = getAccountClient(token) + const workspaceInfo = await accountClient.getWorkspaceInfo(false) if (workspaceInfo === undefined) { ctx.error('Workspace not available for token') return @@ -339,7 +438,8 @@ export async function startIndexer ( router.put('/api/v1/search', async (req, res) => { try { const request = req.request.body as Search - const decoded = decodeToken(request.token) // Just to be safe + const token = request.token ?? req.headers.authorization?.split(' ')[1] + const decoded = decodeToken(token) // Just to be safe ctx.info('search', { classes: request._classes, query: request.query, workspace: decoded.workspace }) await ctx.with('search', {}, async (ctx) => { @@ -348,8 +448,6 @@ export async function startIndexer ( ) req.body = docs }) - - void triggerIndexer(decoded, request.token) } catch (err: any) { Analytics.handleError(err) console.error(err) @@ -361,7 +459,8 @@ export async function startIndexer ( router.put('/api/v1/full-text-search', async (req, res) => { try { const request = req.request.body as FulltextSearch - const decoded = decodeToken(request.token) // Just to be safe + const token = request.token ?? req.headers.authorization?.split(' ')[1] + const decoded = decodeToken(token) // Just to be safe ctx.info('fulltext-search', { ...request.query, workspace: decoded.workspace }) await ctx.with('full-text-search', {}, async (ctx) => { const result = await ctx.with('searchFulltext', {}, (ctx) => @@ -369,24 +468,6 @@ export async function startIndexer ( ) req.body = result }) - - void triggerIndexer(decoded, request.token) - } catch (err: any) { - Analytics.handleError(err) - console.error(err) - req.res.writeHead(404, {}) - req.res.end() - } - }) - - router.put('/api/v1/warmup', async (req, res) => { - try { - const request = req.request.body as IndexDocuments - const decoded = decodeToken(request.token) // Just to be safe - req.body = {} - - ctx.info('warm-up', { workspace: decoded.workspace }) - void triggerIndexer(decoded, request.token, true) } catch (err: any) { Analytics.handleError(err) console.error(err) @@ -398,7 +479,8 @@ export async function startIndexer ( router.put('/api/v1/close', async (req, res) => { try { const request = req.request.body as IndexDocuments - const decoded = decodeToken(request.token) // Just to be safe + const token = request.token ?? req.headers.authorization?.split(' ')[1] + const decoded = decodeToken(token) // Just to be safe req.body = {} ctx.info('close', { workspace: decoded.workspace }) @@ -422,12 +504,14 @@ export async function startIndexer ( router.put('/api/v1/index-documents', async (req, res) => { try { const request = req.request.body as IndexDocuments - const decoded = decodeToken(request.token) // Just to be safe + const token = request.token ?? req.headers.authorization?.split(' ')[1] + const decoded = decodeToken(token) // Just to be safe - const indexer = await getIndexer(ctx, decoded.workspace, request.token) + const indexer = await getIndexer(ctx, decoded.workspace, token) if (indexer !== undefined) { indexer.lastUpdate = Date.now() - await ctx.with('index-documents', {}, (ctx) => indexer.fulltext.indexDocuments(ctx, request.requests)) + // TODO: Fixme + // await ctx.with('index-documents', {}, (ctx) => indexer.fulltext.indexDocuments(ctx, request.requests)) } req.body = {} } catch (err: any) { @@ -441,14 +525,19 @@ export async function startIndexer ( router.put('/api/v1/reindex', async (req, res) => { try { const request = req.request.body as Reindex - const decoded = decodeToken(request.token) // Just to be safe + const token = request.token ?? req.headers.authorization?.split(' ')[1] + const decoded = decodeToken(token) // Just to be safe req.body = {} ctx.info('reindex', { workspace: decoded.workspace }) - const indexer = await getIndexer(ctx, decoded.workspace, request.token, true) + const indexer = await getIndexer(ctx, decoded.workspace, token, true) if (indexer !== undefined) { indexer.lastUpdate = Date.now() - await indexer.reindex(request?.onlyDrop ?? false) + if (request?.onlyDrop ?? false) { + await workspaceProducer.send(decoded.workspace, [workspaceEvents.clearIndex()]) + } else { + await workspaceProducer.send(decoded.workspace, [workspaceEvents.fullReindex()]) + } } } catch (err: any) { Analytics.handleError(err) @@ -465,19 +554,11 @@ export async function startIndexer ( }) const close = (): void => { + void txConsumer.shutdown() + void workspaceConsumer.shutdown() clearInterval(shutdownInterval) server.close() } return close - - async function triggerIndexer (decoded: Token, token: string, trigger: boolean = false): Promise { - const indexer = await getIndexer(ctx, decoded.workspace, token, trigger) - if (indexer !== undefined) { - indexer.lastUpdate = Date.now() - if (trigger) { - indexer.fulltext.triggerIndexing() - } - } - } } diff --git a/pods/server/package.json b/pods/server/package.json index 3a4ae71b9b..80e004bd76 100644 --- a/pods/server/package.json +++ b/pods/server/package.json @@ -81,6 +81,7 @@ "msgpackr-extract": "^3.0.3", "@hcengineering/postgres": "^0.6.0", "snappy": "^7.2.2", - "@hcengineering/rpc": "^0.6.5" + "@hcengineering/rpc": "^0.6.5", + "@hcengineering/kafka": "^0.6.0" } } diff --git a/pods/server/src/__start.ts b/pods/server/src/__start.ts index ce54b17827..4e0e259358 100644 --- a/pods/server/src/__start.ts +++ b/pods/server/src/__start.ts @@ -4,22 +4,24 @@ // Add this to the VERY top of the first file loaded in your app import { Analytics } from '@hcengineering/analytics' -import { SplitLogger, configureAnalytics } from '@hcengineering/analytics-service' +import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service' import contactPlugin from '@hcengineering/contact' import { MeasureMetricsContext, newMetrics, setOperationLogProfiling } from '@hcengineering/core' +import { getPlatformQueue } from '@hcengineering/kafka' import { setMetadata } from '@hcengineering/platform' +import { setDBExtraOptions } from '@hcengineering/postgres' import { serverConfigFromEnv } from '@hcengineering/server' import serverAiBot from '@hcengineering/server-ai-bot' import serverCalendar from '@hcengineering/server-calendar' import serverCore, { + initStatisticsContext, + loadBrandingMap, type ConnectionSocket, type Session, type StorageConfiguration, type UserStatistics, type Workspace, - type WorkspaceStatistics, - initStatisticsContext, - loadBrandingMap + type WorkspaceStatistics } from '@hcengineering/server-core' import serverNotification from '@hcengineering/server-notification' import { storageConfigFromEnv } from '@hcengineering/server-storage' @@ -29,7 +31,6 @@ import { startHttpServer } from '@hcengineering/server-ws' import { join } from 'path' import { start } from '.' import { profileStart, profileStop } from './inspector' -import { setDBExtraOptions } from '@hcengineering/postgres' configureAnalytics(process.env.SENTRY_DSN, {}) Analytics.setTag('application', 'transactor') @@ -37,6 +38,18 @@ Analytics.setTag('application', 'transactor') let getUsers: () => WorkspaceStatistics[] = () => { return [] } + +const queueConfig = process.env.QUEUE_CONFIG +if (queueConfig === undefined) { + throw new Error('Please provide queue config') +} + +const queue = getPlatformQueue('transactor') + +void queue.createTopics(10).catch((err) => { + console.error('Failed to create required topics', err) +}) + // Force create server metrics context with proper logging const metricsContext = initStatisticsContext('transactor', { getUsers: (): WorkspaceStatistics[] => { @@ -90,7 +103,8 @@ const { shutdown, sessionManager } = start(metricsContext, config.dbUrl, { start: profileStart, stop: profileStop }, - mongoUrl: config.mongoUrl + mongoUrl: config.mongoUrl, + queue }) const entryToUserStats = (session: Session, socket: ConnectionSocket): UserStatistics => { diff --git a/pods/server/src/__tests__/backup.spec.ts b/pods/server/src/__tests__/backup.spec.ts deleted file mode 100644 index 97485f2a43..0000000000 --- a/pods/server/src/__tests__/backup.spec.ts +++ /dev/null @@ -1,138 +0,0 @@ -import core, { - DOMAIN_TX, - generateId, - MeasureMetricsContext, - type WorkspaceDataId, - type WorkspaceUuid, - type Doc, - type LowLevelStorage, - type Ref, - type TxCreateDoc -} from '@hcengineering/core' -import builder from '@hcengineering/model-all' -import { wrapPipeline } from '@hcengineering/server-core' -import { getServerPipeline } from '@hcengineering/server-pipeline' -import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' - -const model = builder().getTxes() -// const dbURL = 'postgresql://root@localhost:26257/defaultdb?sslmode=disable' -const dbURL = 'postgresql://postgres:example@localhost:5432' -const STORAGE_CONFIG = 'minio|localhost:9000?accessKey=minioadmin&secretKey=minioadmin&useSSL=false' - -// jest.setTimeout(4500000) -describe.skip('test-backup-find', () => { - it('check create/load/clean', async () => { - const toolCtx = new MeasureMetricsContext('-', {}) - // We should setup a DB with docuemnts and try to backup them. - const wsIds = { dataId: 'testdb-backup-test' as WorkspaceDataId, uuid: 'test' as WorkspaceUuid, url: 'test' } - const storageConfig = storageConfigFromEnv(STORAGE_CONFIG) - const storageAdapter = buildStorageFromConfig(storageConfig) - const pipeline = await getServerPipeline(toolCtx, model, dbURL, wsIds, storageAdapter, { - disableTriggers: true - }) - try { - const client = wrapPipeline(toolCtx, pipeline, wsIds) - const lowLevel = pipeline.context.lowLevelStorage as LowLevelStorage - - // We need to create a backup docs if they are missing. - await prepareTxes(lowLevel, toolCtx) - - const docs: Doc[] = [] - while (true) { - const chunk = await client.loadChunk(DOMAIN_TX, 0) - const part = await client.loadDocs( - DOMAIN_TX, - chunk.docs.map((doc) => doc.id as Ref) - ) - docs.push(...part) - if (chunk.finished) { - break - } - } - await client.closeChunk(0) - expect(docs.length).toBeGreaterThan(459) - - await client.clean( - DOMAIN_TX, - docs.map((doc) => doc._id) - ) - const findDocs = await client.findAll(core.class.Tx, {}) - expect(findDocs.length).toBe(0) - - // - } finally { - await pipeline.close() - await storageAdapter.close() - } - }) - it('check traverse', async () => { - const toolCtx = new MeasureMetricsContext('-', {}) - // We should setup a DB with docuemnts and try to backup them. - const wsIds = { dataId: 'testdb-backup-test' as WorkspaceDataId, uuid: 'test' as WorkspaceUuid, url: 'test' } - - const storageConfig = storageConfigFromEnv(STORAGE_CONFIG) - const storageAdapter = buildStorageFromConfig(storageConfig) - const pipeline = await getServerPipeline(toolCtx, model, dbURL, wsIds, storageAdapter, { - disableTriggers: true - }) - try { - const client = wrapPipeline(toolCtx, pipeline, wsIds) - const lowLevel = pipeline.context.lowLevelStorage as LowLevelStorage - - // We need to create a backup docs if they are missing. - await prepareTxes(lowLevel, toolCtx, 1500) - - const iter = await lowLevel.traverse(DOMAIN_TX, {}) - - const allDocs: Doc[] = [] - - while (true) { - const docs = await iter.next(50) - if (docs == null || docs?.length === 0) { - break - } - await client.clean( - DOMAIN_TX, - docs.map((doc) => doc._id) - ) - allDocs.push(...docs) - } - expect(allDocs.length).toBeGreaterThan(1449) - - const findDocs = await client.findAll(core.class.Tx, {}) - expect(findDocs.length).toBe(0) - - // - } finally { - await pipeline.close() - await storageAdapter.close() - } - }) -}) -async function prepareTxes ( - lowLevel: LowLevelStorage, - toolCtx: MeasureMetricsContext, - count: number = 500 -): Promise { - const docs = await lowLevel.rawFindAll(DOMAIN_TX, {}) - if ((docs?.length ?? 0) < count) { - // We need to fill some documents to be pressent - const docs: TxCreateDoc[] = [] - for (let i = 0; i < count; i++) { - docs.push({ - _class: core.class.TxCreateDoc, - _id: generateId(), - space: core.space.Tx, - modifiedBy: core.account.ConfigUser, - modifiedOn: Date.now(), - attributes: { - qwe: generateId() - }, - objectClass: core.class.DocIndexState, - objectId: generateId(), - objectSpace: core.space.Workspace - }) - } - await lowLevel.upload(toolCtx, DOMAIN_TX, docs) - } -} diff --git a/pods/server/src/server.ts b/pods/server/src/server.ts index bf0ebbbd34..af73c95669 100644 --- a/pods/server/src/server.ts +++ b/pods/server/src/server.ts @@ -23,7 +23,8 @@ import { type Session, type SessionManager, type StorageConfiguration, - type Workspace + type Workspace, + type PlatformQueue } from '@hcengineering/server-core' import { type Token } from '@hcengineering/server-token' @@ -76,6 +77,7 @@ export function start ( metrics: MeasureContext, dbUrl: string, opt: { + queue: PlatformQueue fulltextUrl: string storageConfig: StorageConfiguration port: number @@ -120,7 +122,7 @@ export function start ( metrics, dbUrl, model, - { ...opt, externalStorage, adapterSecurity: isAdapterSecurity(dbUrl) }, + { ...opt, externalStorage, adapterSecurity: isAdapterSecurity(dbUrl), queue: opt.queue }, {} ) const sessionFactory = (token: Token, workspace: Workspace, account: Account): Session => { @@ -136,7 +138,8 @@ export function start ( enableCompression: opt.enableCompression, accountsUrl: opt.accountsUrl, externalStorage, - profiling: opt.profiling + profiling: opt.profiling, + queue: opt.queue }) return { shutdown: async () => { diff --git a/pods/workspace/package.json b/pods/workspace/package.json index b18ab6e18c..55f860f736 100644 --- a/pods/workspace/package.json +++ b/pods/workspace/package.json @@ -56,6 +56,7 @@ "@hcengineering/server-core": "^0.6.1", "@hcengineering/model-all": "^0.6.0", "@hcengineering/analytics": "^0.6.0", - "@hcengineering/analytics-service": "^0.6.0" + "@hcengineering/analytics-service": "^0.6.0", + "@hcengineering/kafka": "^0.6.0" } } diff --git a/pods/workspace/src/__start.ts b/pods/workspace/src/__start.ts index cc0d93b3d4..52e87fa9ca 100644 --- a/pods/workspace/src/__start.ts +++ b/pods/workspace/src/__start.ts @@ -4,6 +4,7 @@ import { Analytics } from '@hcengineering/analytics' import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service' import { MeasureMetricsContext, newMetrics, type Tx } from '@hcengineering/core' +import { getPlatformQueue } from '@hcengineering/kafka' import builder, { getModelVersion, migrateOperations } from '@hcengineering/model-all' import { initStatisticsContext, loadBrandingMap } from '@hcengineering/server-core' import { serveWorkspaceAccount } from '@hcengineering/workspace-service' @@ -34,8 +35,11 @@ const metricsContext = initStatisticsContext('workspace', { const brandingPath = process.env.BRANDING_PATH +const queue = getPlatformQueue('workspace') + serveWorkspaceAccount( metricsContext, + queue, getModelVersion(), txes, migrateOperations, diff --git a/qms-tests/.env b/qms-tests/.env index ac5825fc99..20daf61d0f 100644 --- a/qms-tests/.env +++ b/qms-tests/.env @@ -1,2 +1,3 @@ STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" -MONGO_URL=mongodb://mongodb:27018?compressors=snappy \ No newline at end of file +MONGO_URL=mongodb://mongodb:27018?compressors=snappy +QUEUE_CONFIG=redpanda:9093 diff --git a/qms-tests/docker-compose.yaml b/qms-tests/docker-compose.yaml index 2585789f1a..69ab03b914 100644 --- a/qms-tests/docker-compose.yaml +++ b/qms-tests/docker-compose.yaml @@ -11,6 +11,55 @@ services: ports: - 27018:27018 restart: unless-stopped + redpanda: + image: docker.redpanda.com/redpandadata/redpanda:v24.3.6 + extra_hosts: + - 'huly.local:host-gateway' + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9093,external://0.0.0.0:19093 + - --advertise-kafka-addr internal://redpanda:9093,external://localhost:19093 + - --pandaproxy-addr internal://0.0.0.0:8084,external://0.0.0.0:18084 + - --advertise-pandaproxy-addr internal://redpanda:8084,external://localhost:18084 + - --schema-registry-addr internal://0.0.0.0:8083,external://0.0.0.0:18083 + - --rpc-addr redpanda:33145 + - --advertise-rpc-addr redpanda:33145 + - --mode dev-container + - --smp 1 + - --default-log-level=info + ports: + - 18083:18083 + - 18084:18084 + - 19093:19093 + - 19645:9644 + healthcheck: + test: ['CMD', 'rpk', 'cluster', 'info', '-X', 'user=superuser', '-X', 'pass=secretpassword'] + interval: 10s + timeout: 5s + retries: 10 + redpanda_console: + image: docker.redpanda.com/redpandadata/console:v2.8.3 + extra_hosts: + - 'huly.local:host-gateway' + entrypoint: /bin/sh + command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console' + environment: + CONFIG_FILEPATH: /tmp/config.yml + CONSOLE_CONFIG_FILE: | + kafka: + brokers: ["redpanda:9093"] + schemaRegistry: + enabled: true + urls: ["http://redpanda:8083"] + redpanda: + adminApi: + enabled: true + urls: ["http://redpanda:9644"] + ports: + - 8001:8080 + depends_on: + - redpanda minio: image: 'minio/minio' command: server /data --address ":9000" @@ -51,6 +100,7 @@ services: volumes: - ./branding-test.json:/var/cfg/branding-test.json environment: + - QUEUE_CONFIG=${QUEUE_CONFIG} - ACCOUNT_PORT=3003 - SERVER_SECRET=secret - WORKSPACE_LIMIT_PER_USER=100 @@ -69,6 +119,7 @@ services: volumes: - ./branding-test.json:/var/cfg/branding.json environment: + - QUEUE_CONFIG=${QUEUE_CONFIG} - SERVER_SECRET=secret - DB_URL=mongodb://mongodb:27018 - MONGO_URL=mongodb://mongodb:27018 @@ -124,6 +175,7 @@ services: volumes: - ./branding-test.json:/var/cfg/branding-test.json environment: + - QUEUE_CONFIG=${QUEUE_CONFIG} - SERVER_PROVIDER=${SERVER_PROVIDER} - SERVER_PORT=3334 - SERVER_SECRET=secret @@ -212,6 +264,7 @@ services: ports: - 4710:4710 environment: + - QUEUE_CONFIG=${QUEUE_CONFIG} - PORT=4710 - SERVER_SECRET=secret - DB_URL=mongodb://mongodb:27018 diff --git a/qms-tests/prepare.sh b/qms-tests/prepare.sh index 5d4cf51e8a..f7245e4c4e 100755 --- a/qms-tests/prepare.sh +++ b/qms-tests/prepare.sh @@ -16,6 +16,8 @@ if [ "x$DO_CLEAN" == 'xtrue' ]; then docker system prune -a -f fi +../tests/wait-elastic.sh 9201 + # Create user record in accounts ./tool.sh create-account user1 -f John -l Appleseed -p 1234 ./tool.sh create-account user2 -f Kainin -l Dirak -p 1234 diff --git a/qms-tests/tool-local.sh b/qms-tests/tool-local.sh index 6a76ff2081..06462d36cf 100755 --- a/qms-tests/tool-local.sh +++ b/qms-tests/tool-local.sh @@ -8,6 +8,7 @@ export ACCOUNT_DB_URL=mongodb://localhost:27017 export ACCOUNTS_URL=http://localhost:3000 export TRANSACTOR_URL=ws://localhost:3333 export SERVER_SECRET=secret +export QUEUE_CONFIG=huly.local:19092 # Restore workspace contents in mongo/elastic node ../dev/tool/bundle/bundle.js $@ \ No newline at end of file diff --git a/qms-tests/tool.sh b/qms-tests/tool.sh index 6b05f411ef..1a5a531691 100755 --- a/qms-tests/tool.sh +++ b/qms-tests/tool.sh @@ -10,5 +10,6 @@ export ACCOUNT_DB_URL=mongodb://localhost:27018 export DB_URL=mongodb://localhost:27018 export SERVER_SECRET=secret export STORAGE_CONFIG="minio|localhost:9002?accessKey=minioadmin&secretKey=minioadmin" +export QUEUE_CONFIG=huly.local:19093 node ../dev/tool/bundle/bundle.js $@ \ No newline at end of file diff --git a/rush.json b/rush.json index c9d99c5960..5c9f517cf7 100644 --- a/rush.json +++ b/rush.json @@ -1518,21 +1518,6 @@ "projectFolder": "pods/fulltext", "shouldPublish": false }, - { - "packageName": "@hcengineering/server-fulltext", - "projectFolder": "server-plugins/fulltext", - "shouldPublish": false - }, - { - "packageName": "@hcengineering/server-fulltext-resources", - "projectFolder": "server-plugins/fulltext-resources", - "shouldPublish": false - }, - { - "packageName": "@hcengineering/model-server-fulltext", - "projectFolder": "models/server-fulltext", - "shouldPublish": false - }, { "packageName": "@hcengineering/hr", "projectFolder": "plugins/hr", @@ -2397,6 +2382,11 @@ "packageName": "@hcengineering/server-process-resources", "projectFolder": "server-plugins/process-resources", "shouldPublish": false + }, + { + "packageName": "@hcengineering/kafka", + "projectFolder": "server/kafka", + "shouldPublish": false } ] } diff --git a/server-plugins/ai-bot-resources/src/utils.ts b/server-plugins/ai-bot-resources/src/utils.ts index cbf93a3c55..bcba148ee0 100644 --- a/server-plugins/ai-bot-resources/src/utils.ts +++ b/server-plugins/ai-bot-resources/src/utils.ts @@ -38,7 +38,7 @@ export async function createAccountRequest (workspace: WorkspaceUuid, ctx: Measu body: JSON.stringify({}) }) } catch (err) { - ctx.error('Could not send create ai account request', { err }) + ctx.warn('Could not send create ai account request', { err }) } } diff --git a/server-plugins/fulltext-resources/.eslintrc.js b/server-plugins/fulltext-resources/.eslintrc.js deleted file mode 100644 index 72235dc283..0000000000 --- a/server-plugins/fulltext-resources/.eslintrc.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = { - extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], - parserOptions: { - tsconfigRootDir: __dirname, - project: './tsconfig.json' - } -} diff --git a/server-plugins/fulltext-resources/.npmignore b/server-plugins/fulltext-resources/.npmignore deleted file mode 100644 index e3ec093c38..0000000000 --- a/server-plugins/fulltext-resources/.npmignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!/lib/** -!CHANGELOG.md -/lib/**/__tests__/ diff --git a/server-plugins/fulltext-resources/config/rig.json b/server-plugins/fulltext-resources/config/rig.json deleted file mode 100644 index 0110930f55..0000000000 --- a/server-plugins/fulltext-resources/config/rig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", - "rigPackageName": "@hcengineering/platform-rig" -} diff --git a/server-plugins/fulltext-resources/src/index.ts b/server-plugins/fulltext-resources/src/index.ts deleted file mode 100644 index f1e2a9b11f..0000000000 --- a/server-plugins/fulltext-resources/src/index.ts +++ /dev/null @@ -1,103 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import core, { - DocIndexState, - isClassIndexable, - isFullTextAttribute, - Tx, - TxProcessor, - type AnyAttribute, - type Class, - type Doc, - type FullTextSearchContext, - type Ref, - type TxCUD, - type TxUpdateDoc -} from '@hcengineering/core' -import { TriggerControl } from '@hcengineering/server-core' - -const indexingCtx = 'indexing_contexts' -export async function OnChange (txes: Tx[], control: TriggerControl): Promise { - let contexts = control.contextCache.get(indexingCtx) as Map>, FullTextSearchContext> - if (contexts === undefined) { - contexts = new Map() - control.contextCache.set(indexingCtx, contexts) - } - - if (control.ctx.contextData.fulltextUpdates === undefined) { - control.ctx.contextData.fulltextUpdates = new Map() - } - const docsToUpsert = control.ctx.contextData.fulltextUpdates - - for (const tx of txes) { - const attrs = new Map() - if (TxProcessor.isExtendsCUD(tx._class)) { - const cud = tx as TxCUD - - if (!isClassIndexable(control.hierarchy, cud.objectClass, contexts)) { - // No need, since no indixable fields or attachments. - continue - } - - // We need to check if operation has indexable attributes. - if (cud._class === core.class.TxUpdateDoc) { - let hasFulltext = false - // We could skip updates for a lot of transactions if they have non idexable attributes - const upd = cud as TxUpdateDoc - for (const [k] of Object.entries(upd.operations)) { - if (k.startsWith('$') || k === 'modifiedOn') { - // Is operator - // Skip operator changes - } else { - const key = upd.objectClass + '.' + k - const attr = attrs.get(key) ?? control.hierarchy.findAttribute(upd.objectClass, k) - if (attr !== undefined) { - attrs.set(key, attr ?? null) - } - if (attr != null) { - if (isFullTextAttribute(attr)) { - hasFulltext = true - } - } - } - } - if (!hasFulltext) { - // No full text changes, no indexing is required - continue - } - } - - docsToUpsert.set(cud.objectId as Ref, { - _id: cud.objectId as Ref, - _class: core.class.DocIndexState, - objectClass: cud.objectClass, - needIndex: true, - modifiedBy: tx.modifiedBy, - modifiedOn: tx.modifiedOn, - space: cud.objectSpace, - removed: cud._class === core.class.TxRemoveDoc - }) - } - } - return [] -} - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export default async () => ({ - trigger: { - OnChange - } -}) diff --git a/server-plugins/fulltext-resources/tsconfig.json b/server-plugins/fulltext-resources/tsconfig.json deleted file mode 100644 index 59e4fd4297..0000000000 --- a/server-plugins/fulltext-resources/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", - - "compilerOptions": { - "rootDir": "./src", - "outDir": "./lib", - "declarationDir": "./types", - "tsBuildInfoFile": ".build/build.tsbuildinfo" - } -} \ No newline at end of file diff --git a/server-plugins/fulltext/.eslintrc.js b/server-plugins/fulltext/.eslintrc.js deleted file mode 100644 index 72235dc283..0000000000 --- a/server-plugins/fulltext/.eslintrc.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = { - extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], - parserOptions: { - tsconfigRootDir: __dirname, - project: './tsconfig.json' - } -} diff --git a/server-plugins/fulltext/.npmignore b/server-plugins/fulltext/.npmignore deleted file mode 100644 index e3ec093c38..0000000000 --- a/server-plugins/fulltext/.npmignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!/lib/** -!CHANGELOG.md -/lib/**/__tests__/ diff --git a/server-plugins/fulltext/config/rig.json b/server-plugins/fulltext/config/rig.json deleted file mode 100644 index 0110930f55..0000000000 --- a/server-plugins/fulltext/config/rig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", - "rigPackageName": "@hcengineering/platform-rig" -} diff --git a/server-plugins/fulltext/jest.config.js b/server-plugins/fulltext/jest.config.js deleted file mode 100644 index 2cfd408b67..0000000000 --- a/server-plugins/fulltext/jest.config.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], - roots: ["./src"], - coverageReporters: ["text-summary", "html"] -} diff --git a/server-plugins/fulltext/package.json b/server-plugins/fulltext/package.json deleted file mode 100644 index 55059569a3..0000000000 --- a/server-plugins/fulltext/package.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "name": "@hcengineering/server-fulltext", - "version": "0.6.0", - "main": "lib/index.js", - "svelte": "src/index.ts", - "types": "types/index.d.ts", - "files": [ - "lib/**/*", - "types/**/*", - "tsconfig.json" - ], - "author": "Anticrm Platform Contributors", - "license": "EPL-2.0", - "scripts": { - "build": "compile", - "build:watch": "compile", - "format": "format src", - "test": "jest --passWithNoTests --silent", - "_phase:build": "compile transpile src", - "_phase:test": "jest --passWithNoTests --silent", - "_phase:format": "format src", - "_phase:validate": "compile validate" - }, - "devDependencies": { - "@hcengineering/platform-rig": "^0.6.0", - "@types/node": "~20.11.16", - "@typescript-eslint/eslint-plugin": "^6.11.0", - "eslint-plugin-import": "^2.26.0", - "eslint-plugin-promise": "^6.1.1", - "eslint-plugin-n": "^15.4.0", - "eslint": "^8.54.0", - "@typescript-eslint/parser": "^6.11.0", - "eslint-config-standard-with-typescript": "^40.0.0", - "prettier": "^3.1.0", - "typescript": "^5.3.3", - "jest": "^29.7.0", - "ts-jest": "^29.1.1", - "@types/jest": "^29.5.5" - }, - "dependencies": { - "@hcengineering/platform": "^0.6.11", - "@hcengineering/server-core": "^0.6.1" - } -} diff --git a/server-plugins/fulltext/src/index.ts b/server-plugins/fulltext/src/index.ts deleted file mode 100644 index b42b59341b..0000000000 --- a/server-plugins/fulltext/src/index.ts +++ /dev/null @@ -1,33 +0,0 @@ -// -// Copyright © 2020, 2021 Anticrm Platform Contributors. -// Copyright © 2021, 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import type { Plugin, Resource } from '@hcengineering/platform' -import { plugin } from '@hcengineering/platform' -import type { TriggerFunc } from '@hcengineering/server-core' - -/** - * @public - */ -export const serverFulltextId = 'server-fulltext' as Plugin - -/** - * @public - */ -export default plugin(serverFulltextId, { - trigger: { - OnChange: '' as Resource - } -}) diff --git a/server-plugins/fulltext/tsconfig.json b/server-plugins/fulltext/tsconfig.json deleted file mode 100644 index 59e4fd4297..0000000000 --- a/server-plugins/fulltext/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", - - "compilerOptions": { - "rootDir": "./src", - "outDir": "./lib", - "declarationDir": "./types", - "tsBuildInfoFile": ".build/build.tsbuildinfo" - } -} \ No newline at end of file diff --git a/server-plugins/recruit-resources/src/index.ts b/server-plugins/recruit-resources/src/index.ts index 4634b31fa2..6304ab04ec 100644 --- a/server-plugins/recruit-resources/src/index.ts +++ b/server-plugins/recruit-resources/src/index.ts @@ -125,8 +125,8 @@ async function handleVacancyUpdate (control: TriggerControl, cud: TxCUD, re objectId: updateTx.objectId }) ).filter((it) => it._id !== updateTx._id) - const vacancy = TxProcessor.buildDoc2Doc(txes) as Vacancy - if (vacancy.company != null) { + const vacancy = TxProcessor.buildDoc2Doc(txes) + if (vacancy?.company != null) { // We have old value res.push( control.txFactory.createTxMixin( diff --git a/server-plugins/tracker-resources/src/index.ts b/server-plugins/tracker-resources/src/index.ts index 4b5d0948a1..da0d603fae 100644 --- a/server-plugins/tracker-resources/src/index.ts +++ b/server-plugins/tracker-resources/src/index.ts @@ -331,7 +331,7 @@ async function doTimeReportUpdate (cud: TxCUD, control: Trigger objectId: cud.objectId }) ).filter((it) => it._id !== cud._id) - const doc: TimeSpendReport | undefined = TxProcessor.buildDoc2Doc(logTxes) + const doc = TxProcessor.buildDoc2Doc(logTxes) const res: Tx[] = [] const [currentIssue] = await control.findAll( @@ -340,7 +340,7 @@ async function doTimeReportUpdate (cud: TxCUD, control: Trigger { _id: attachedTo }, { limit: 1 } ) - if (doc !== undefined) { + if (doc != null) { res.push( control.txFactory.createTxUpdateDoc( attachedToClass, @@ -370,8 +370,8 @@ async function doTimeReportUpdate (cud: TxCUD, control: Trigger objectId: cud.objectId }) ).filter((it) => it._id !== cud._id) - const doc: TimeSpendReport | undefined = TxProcessor.buildDoc2Doc(logTxes) - if (doc !== undefined) { + const doc = TxProcessor.buildDoc2Doc(logTxes) + if (doc != null) { const [currentIssue] = await control.findAll( control.ctx, tracker.class.Issue, diff --git a/server/account/src/utils.ts b/server/account/src/utils.ts index 2cc16811bd..549757cb01 100644 --- a/server/account/src/utils.ts +++ b/server/account/src/utils.ts @@ -562,7 +562,7 @@ export async function selectWorkspace ( throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspaceUrl })) } - if (accountUuid === systemAccountUuid) { + if (accountUuid === systemAccountUuid || extra?.admin === 'true') { return { account: accountUuid, token: generateToken(accountUuid, workspace.uuid, extra), diff --git a/server/backup/src/backup.ts b/server/backup/src/backup.ts index 5ec23d9a43..1a848d45d4 100644 --- a/server/backup/src/backup.ts +++ b/server/backup/src/backup.ts @@ -21,7 +21,6 @@ import core, { Doc, Domain, DOMAIN_BLOB, - DOMAIN_DOC_INDEX_STATE, DOMAIN_MODEL, DOMAIN_MODEL_TX, DOMAIN_TRANSIENT, @@ -36,7 +35,6 @@ import core, { TxProcessor, type BackupStatus, type Blob, - type DocIndexState, type Tx, type TxCUD, type WorkspaceIds, @@ -623,22 +621,6 @@ export async function cloneWorkspace ( // ) } -// function prepareClonedDocuments (docs: Doc[]): Doc[] { -// docs = docs.map((p) => { -// // if full text is skipped, we need to clean stages for indexes. -// if (p._class === core.class.DocIndexState) { -// ;(p as DocIndexState).needIndex = true -// } - -// return { -// ...p, -// modifiedOn: Date.now(), -// createdOn: Date.now() -// } -// }) -// return docs -// } - // async function cleanDomain (ctx: MeasureContext, connection: CoreClient & BackupClient, domain: Domain): Promise { // // Load all digest from collection. // let idx: number | undefined @@ -1861,6 +1843,7 @@ export async function restore ( // We do not backup elastic anymore domains.delete('fulltext-blob' as Domain) + domains.delete('doc-index-state' as Domain) let uploadedMb = 0 let uploaded = 0 @@ -2000,11 +1983,6 @@ export async function restore ( tx.objectSpace = core.space.Workspace } } - if (opt.cleanIndexState === true) { - if (d._class === core.class.DocIndexState) { - ;(d as DocIndexState).needIndex = true - } - } } if (opt.recheck === true) { @@ -2220,34 +2198,10 @@ export async function restore ( } } } - async function performCleanDocIndexState (docsToRemove: Ref[]): Promise { - ctx.info('cleanup', { toRemove: docsToRemove.length, workspace: workspaceId, domain: c }) - while (docsToRemove.length > 0) { - const part = docsToRemove.splice(0, 1000) - try { - const docs = (await connection.loadDocs(DOMAIN_DOC_INDEX_STATE, part)) as DocIndexState[] - docs.forEach((it) => { - it.needIndex = true - it.removed = true - it.modifiedOn = Date.now() - }) - await connection.upload(DOMAIN_DOC_INDEX_STATE, docs) - } catch (err: any) { - ctx.error('failed to clean, will retry', { error: err, workspaceId }) - docsToRemove.push(...part) - } - } - } if (c !== DOMAIN_BLOB) { // Clean domain documents if not blob if (docsToRemove.length > 0 && opt.merge !== true) { - if (c === DOMAIN_DOC_INDEX_STATE) { - // We need to create a request to clean fulltext for selected docuemnts. - // We need o clean a FULLTEXT domain as well - await performCleanDocIndexState([...docsToRemove]) - } else { - await performCleanOfDomain(docsToRemove, c) - } + await performCleanOfDomain(docsToRemove, c) } } } diff --git a/server/core/src/adapter.ts b/server/core/src/adapter.ts index 9411dd8d41..5be268f620 100644 --- a/server/core/src/adapter.ts +++ b/server/core/src/adapter.ts @@ -14,7 +14,6 @@ // import { - type WorkspaceIds, type Class, type Doc, type DocumentQuery, @@ -28,6 +27,7 @@ import { type Ref, type Tx, type TxResult, + type WorkspaceIds, type WorkspaceUuid } from '@hcengineering/core' import { type StorageAdapter } from './storage' @@ -61,6 +61,11 @@ export type DbAdapterHandler = ( count: number, helper: DomainHelperOperations ) => void + +export interface RawFindIterator { + find: (ctx: MeasureContext) => Promise + close: () => Promise +} /** * @public */ @@ -87,6 +92,8 @@ export interface DbAdapter extends LowLevelStorage { // Allow to register a handler to listen for domain operations on?: (handler: DbAdapterHandler) => void + + rawFind: (ctx: MeasureContext, domain: Domain) => RawFindIterator } /** diff --git a/server/core/src/base.ts b/server/core/src/base.ts index 070d1d5cff..c23e7950e7 100644 --- a/server/core/src/base.ts +++ b/server/core/src/base.ts @@ -104,6 +104,10 @@ export abstract class BaseMiddleware implements Middleware { return this.next?.handleBroadcast(ctx) ?? emptyBroadcastResult } + provideBroadcast (ctx: MeasureContext): Promise { + return this.next?.handleBroadcast(ctx) ?? emptyBroadcastResult + } + protected provideTx (ctx: MeasureContext, tx: Tx[]): Promise { if (this.next !== undefined) { return this.next.tx(ctx, tx) diff --git a/server/core/src/index.ts b/server/core/src/index.ts index 0d1c107963..8ca7cb2739 100644 --- a/server/core/src/index.ts +++ b/server/core/src/index.ts @@ -34,3 +34,4 @@ export * from './nullAdapter' export * from './service' export * from './stats' export * from './triggers' +export * from './queue' diff --git a/server/core/src/mem.ts b/server/core/src/mem.ts index dc8cd0deb2..3a3d683a43 100644 --- a/server/core/src/mem.ts +++ b/server/core/src/mem.ts @@ -35,7 +35,7 @@ import core, { type TxResult, type WorkspaceIds } from '@hcengineering/core' -import { type DbAdapter, type DbAdapterHandler, type DomainHelperOperations } from './adapter' +import { type DbAdapter, type DbAdapterHandler, type DomainHelperOperations, type RawFindIterator } from './adapter' /** * @public */ @@ -62,6 +62,13 @@ export class DummyDbAdapter implements DbAdapter { return toFindResult([]) } + rawFind (ctx: MeasureContext, domain: Domain): RawFindIterator { + return { + find: async () => [], + close: async () => {} + } + } + async init (): Promise {} helper (): DomainHelperOperations { diff --git a/server/core/src/plugin.ts b/server/core/src/plugin.ts index 31836fef0f..e1fea8facf 100644 --- a/server/core/src/plugin.ts +++ b/server/core/src/plugin.ts @@ -16,7 +16,7 @@ import { type Metadata, type Plugin, plugin } from '@hcengineering/platform' -import type { Class, Ref, Space, Mixin } from '@hcengineering/core' +import type { Class, Mixin, Ref } from '@hcengineering/core' import type { ObjectDDParticipant, SearchPresenter, Trigger } from './types' /** @@ -35,9 +35,6 @@ const serverCore = plugin(serverCoreId, { ObjectDDParticipant: '' as Ref, SearchPresenter: '' as Ref> }, - space: { - DocIndexState: '' as Ref - }, metadata: { FrontUrl: '' as Metadata, FilesUrl: '' as Metadata, diff --git a/server/core/src/queue/dummyQueue.ts b/server/core/src/queue/dummyQueue.ts new file mode 100644 index 0000000000..20e049fcba --- /dev/null +++ b/server/core/src/queue/dummyQueue.ts @@ -0,0 +1,65 @@ +import type { MeasureContext, WorkspaceUuid } from '@hcengineering/core' +import { type ConsumerHandle, type PlatformQueue, type PlatformQueueProducer, type QueueTopic } from './types' + +/** + * A dummy implementation of PlatformQueueProducer for testing and development + */ +class DummyQueueProducer implements PlatformQueueProducer { + async send (id: WorkspaceUuid | string, msgs: T[]): Promise { + await Promise.resolve() + } + + async close (): Promise { + await Promise.resolve() + } +} + +/** + * A dummy implementation of PlatformQueue for testing and development + */ +export class DummyQueue implements PlatformQueue { + createProducer(ctx: MeasureContext, topic: QueueTopic): PlatformQueueProducer { + return new DummyQueueProducer() + } + + getClientId (): string { + return 'dummy' + } + + createConsumer( + ctx: MeasureContext, + topic: QueueTopic, + groupId: string, + onMessage: ( + msg: { id: WorkspaceUuid | string, value: T }[], + queue: { + pause: () => void + heartbeat: () => Promise + } + ) => Promise, + options?: { + bulkSize?: number + fromBegining?: boolean + } + ): ConsumerHandle { + return { + shutdown: async (): Promise => { + await Promise.resolve() + }, + isConnected: (): boolean => { + return false + } + } + } + + async createTopics (tx: number): Promise { + await Promise.resolve() + } +} + +/** + * Creates a new instance of the dummy queue implementation + */ +export function createDummyQueue (): PlatformQueue { + return new DummyQueue() +} diff --git a/server/core/src/queue/index.ts b/server/core/src/queue/index.ts new file mode 100644 index 0000000000..044d5f3cd8 --- /dev/null +++ b/server/core/src/queue/index.ts @@ -0,0 +1,4 @@ +export * from './types' +export * from './workspace' +export * from './dummyQueue' +export * from './users' diff --git a/server/core/src/queue/types.ts b/server/core/src/queue/types.ts new file mode 100644 index 0000000000..517add9bed --- /dev/null +++ b/server/core/src/queue/types.ts @@ -0,0 +1,59 @@ +import type { MeasureContext, WorkspaceUuid } from '@hcengineering/core' + +export enum QueueTopic { + // Topic with partitions to split workspace transactions into + Tx = 'tx', + + // Topic to send workspace information into + Workspace = 'workspace', + + // A fulltext topic to hold extra fulltext request, like re-indexing requests etc, private to fulltext service. + Fulltext = 'fulltext', + + // A topic about user activity. + Users = 'users' +} + +export interface ConsumerHandle { + shutdown: () => Promise + isConnected: () => boolean +} + +export interface ConsumerMessage { + id: WorkspaceUuid | string + value: T[] +} + +export interface ConsumerControl { + pause: () => void + heartbeat: () => Promise +} + +export interface PlatformQueue { + createProducer: (ctx: MeasureContext, topic: QueueTopic) => PlatformQueueProducer + + /** + * Create a consumer for a topic. + * return a function to close the reciever. + */ + createConsumer: ( + ctx: MeasureContext, + topic: QueueTopic, + groupId: string, + onMessage: (msg: ConsumerMessage[], queue: ConsumerControl) => Promise, + options?: { + fromBegining?: boolean + } + ) => ConsumerHandle + createTopics: (tx: number) => Promise + + getClientId: () => string +} + +/** + * Create a producer for a topic. + */ +export interface PlatformQueueProducer { + send: (id: WorkspaceUuid | string, msgs: T[]) => Promise + close: () => Promise +} diff --git a/server/core/src/queue/users.ts b/server/core/src/queue/users.ts new file mode 100644 index 0000000000..40c25970f0 --- /dev/null +++ b/server/core/src/queue/users.ts @@ -0,0 +1,32 @@ +import type { AccountUuid, PersonId } from '@hcengineering/core' + +export enum QueueUserEvent { + login = 'login', + logout = 'logout' +} + +export interface QueueUserMessage { + type: QueueUserEvent + user: AccountUuid +} + +export interface QueueUserLogin extends QueueUserMessage { + type: QueueUserEvent.login | QueueUserEvent.logout + sessions: number + socialIds: PersonId[] +} + +export const userEvents = { + login: function userLogin (data: Omit): QueueUserLogin { + return { + type: QueueUserEvent.login, + ...data + } + }, + logout: function userLogout (data: Omit): QueueUserLogin { + return { + type: QueueUserEvent.logout, + ...data + } + } +} diff --git a/server/core/src/queue/workspace.ts b/server/core/src/queue/workspace.ts new file mode 100644 index 0000000000..859e714404 --- /dev/null +++ b/server/core/src/queue/workspace.ts @@ -0,0 +1,46 @@ +import type { Class, Doc, Domain, Ref } from '@hcengineering/core' + +export enum QueueWorkspaceEvent { + Up = 'up', + Down = 'down', + Created = 'created', + CreateFailed = 'create-failed', + Upgraded = 'upgraded', + Upgradefailed = 'upgrade-failed', + Deleted = 'deleted', + Archived = 'archived', + Restored = 'restored', + FullReindex = 'full-fulltext-reindex', + Reindex = 'fulltext-reindex', + ClearIndex = 'clear-fulltext-index' +} + +export interface QueueWorkspaceMessage { + type: QueueWorkspaceEvent +} + +export interface QueueWorkspaceReindexMessage extends QueueWorkspaceMessage { + type: QueueWorkspaceEvent.Reindex + + domain: Domain + classes: Ref>[] +} + +export const workspaceEvents = { + open: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Up }), + down: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Down }), + created: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Created }), + upgraded: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Upgraded }), + upgradeFailed: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Upgradefailed }), + createFailed: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.CreateFailed }), + deleted: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Deleted }), + archived: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Archived }), + restored: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Restored }), + fullReindex: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.FullReindex }), + clearIndex: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.ClearIndex }), + reindex: (domain: Domain, classes: Ref>[]): QueueWorkspaceReindexMessage => ({ + type: QueueWorkspaceEvent.Reindex, + domain, + classes + }) +} diff --git a/server/indexer/src/indexer/indexer.ts b/server/indexer/src/indexer/indexer.ts index c3254c9a36..b2096ff639 100644 --- a/server/indexer/src/indexer/indexer.ts +++ b/server/indexer/src/indexer/indexer.ts @@ -21,14 +21,10 @@ import core, { type AttachedDoc, type Blob, type Class, - DOMAIN_DOC_INDEX_STATE, DOMAIN_MIGRATION, + DOMAIN_MODEL, type Doc, - type DocIndexState, - type DocumentQuery, - type DocumentUpdate, type Domain, - type FindResult, type FullTextSearchContext, type Hierarchy, type IdMap, @@ -37,8 +33,8 @@ import core, { type ModelDb, type Ref, type Space, - TxFactory, - type WithLookup, + type TxCUD, + TxProcessor, type WorkspaceIds, type WorkspaceUuid, coreId, @@ -49,12 +45,14 @@ import core, { isClassIndexable, isFullTextAttribute, isIndexedAttribute, + platformNow, systemAccount, toIdMap, withContext } from '@hcengineering/core' import drivePlugin, { type FileVersion } from '@hcengineering/drive' import type { + ConsumerControl, ContentTextAdapter, DbAdapter, FullTextAdapter, @@ -64,8 +62,8 @@ import type { import { RateLimiter, SessionDataImpl } from '@hcengineering/server-core' import { jsonToText, markupToJSON, markupToText } from '@hcengineering/text' import { findSearchPresenter, updateDocWithPresenter } from '../mapper' -import { docStructure, fullReindex, indexes, type FullTextPipeline } from './types' -import { createIndexedDoc, createStateDoc, getContent } from './utils' +import { type FullTextPipeline } from './types' +import { createIndexedDoc, getContent } from './utils' export * from './types' export * from './utils' @@ -82,8 +80,6 @@ export const globalIndexer = { processingSize: 100 } -const rateLimiter = new RateLimiter(globalIndexer.allowParallel) - let indexCounter = 0 function extractValues (obj: any): string { @@ -104,20 +100,76 @@ function extractValues (obj: any): string { } return res } + +class ElasticPushQueue { + pushQueue = new RateLimiter(5) + indexedDocs: IndexedDoc[] = [] + + constructor ( + readonly fulltextAdapter: FullTextAdapter, + readonly workspace: WorkspaceIds, + readonly ctx: MeasureContext, + readonly control?: ConsumerControl + ) {} + + async push (doc: IndexedDoc): Promise { + this.indexedDocs.push(doc) + if (this.indexedDocs.length > 25) { + await this.pushToIndex() + } + } + + async waitProcessing (): Promise { + if (this.indexedDocs.length > 0) { + await this.pushToIndex() + } + await this.pushQueue.waitProcessing() + } + + async pushToIndex (): Promise { + const docs = [...this.indexedDocs] + this.indexedDocs = [] + if (docs.length === 0) { + return + } + await this.pushQueue.add(async () => { + try { + try { + await this.ctx.with('push-elastic', {}, () => + this.fulltextAdapter.updateMany(this.ctx, this.workspace.uuid, docs) + ) + + await this.control?.heartbeat() + } catch (err: any) { + Analytics.handleError(err) + // Try to push one by one + await this.ctx.with('push-elastic-by-one', {}, async () => { + for (const d of docs) { + try { + await this.fulltextAdapter.update(this.ctx, this.workspace.uuid, d.id, d) + } catch (err2: any) { + Analytics.handleError(err2) + } + } + }) + } + } catch (err: any) { + Analytics.handleError(err) + } + }) + } +} + /** * @public */ export class FullTextIndexPipeline implements FullTextPipeline { cancelling: boolean = false - indexing: Promise | undefined - verify: Promise | undefined indexId = indexCounter++ contexts: Map>, FullTextSearchContext> - byDomain = new Map>[]>() - constructor ( readonly fulltextAdapter: FullTextAdapter, private readonly storage: DbAdapter, @@ -127,66 +179,15 @@ export class FullTextIndexPipeline implements FullTextPipeline { readonly model: ModelDb, readonly storageAdapter: StorageAdapter, readonly contentAdapter: ContentTextAdapter, - readonly broadcastUpdate: (ctx: MeasureContext, classes: Ref>[]) => void, - readonly checkIndexes: () => Promise + readonly broadcastUpdate: (ctx: MeasureContext, classes: Ref>[]) => void ) { this.contexts = new Map(model.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it])) } - async cancel (): Promise { - this.cancelling = true - await this.verify - this.triggerIndexing() - await this.indexing - this.metrics.warn('Cancel indexing', { workspace: this.workspace.uuid, indexId: this.indexId }) - } - - async markRemove (doc: DocIndexState): Promise { - const ops = new TxFactory(core.account.System, true) - await this.storage.tx( - this.metrics, - ops.createTxUpdateDoc(doc._class, doc.space, doc._id, { - removed: true, - needIndex: true - }) - ) - } - - updateDoc (doc: DocIndexState, tx: DocumentUpdate, finish: boolean): DocIndexState { - if (finish) { - doc.needIndex = false - doc.modifiedBy = core.account.System - doc.modifiedOn = Date.now() - } - return doc - } - - triggerCounts = 0 - - triggerIndexing = (): void => {} - - async startIndexing (indexing: () => void): Promise { - this.cancelling = false - this.verify = this.verifyWorkspace(this.metrics, indexing) - void this.verify.then(() => { - this.indexing = this.doIndexing(indexing) - }) - } - - @withContext('verify-workspace') - async verifyWorkspace (ctx: MeasureContext, indexing: () => void): Promise { - // We need to apply migrations if required. - const migrations = await this.storage.findAll(ctx, core.class.MigrationState, { - plugin: coreId - }) - - // Verify class integrity if required + async getIndexClassess (): Promise<{ domain: Domain, classes: Ref>[] }[]> { const allClasses = this.hierarchy.getDescendants(core.class.Doc) - const contexts = new Map(this.model.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it])) - const allIndexed = allClasses.filter((it) => isClassIndexable(this.hierarchy, it, contexts)) - const domainPriorities: Record = { space: 1500, contact: 1000, @@ -196,112 +197,81 @@ export class FullTextIndexPipeline implements FullTextPipeline { attachment: -100 } - this.byDomain = groupByArray(allIndexed, (it) => this.hierarchy.getDomain(it)) - this.byDomain = new Map( - Array.from(this.byDomain.entries()).sort((a, b) => { + const byDomain = groupByArray(allIndexed, (it) => this.hierarchy.getDomain(it)) + return Array.from(byDomain.entries()) + .sort((a, b) => { const ap = domainPriorities[a[0]] ?? 0 const bp = domainPriorities[b[0]] ?? 0 return bp - ap }) - ) - - if (migrations.find((it) => it.state === indexes) === undefined) { - ctx.warn('Rebuild DB index', { workspace: this.workspace.uuid }) - // Clean all existing docs, they will be re-created on verify stage - await this.checkIndexes() - - await this.addMigration(ctx, indexes) - ctx.warn('Rebuild DB index complete', { workspace: this.workspace.uuid }) - } - - if (migrations.find((it) => it.state === fullReindex) === undefined) { - ctx.warn('rebuilding index to v5', { workspace: this.workspace.uuid }) - // Clean all existing docs, they will be re-created on verify stage - await this.storage.rawDeleteMany(DOMAIN_DOC_INDEX_STATE, {}) - if (this.workspace.dataId != null) { - await this.fulltextAdapter.clean(ctx, this.workspace.dataId as unknown as WorkspaceUuid) - } - await this.fulltextAdapter.clean(ctx, this.workspace.uuid) - ctx.warn('rebuilding index to v5 complete', { workspace: this.workspace.uuid }) - - await this.addMigration(ctx, fullReindex) - } - - if (migrations.find((it) => it.state === docStructure) === undefined) { - ctx.warn('verify document structure', { version: docStructure, workspace: this.workspace.uuid }) - - for (const [domain, classes] of this.byDomain.entries()) { - await ctx.with('verify-domain', { domain }, async () => { - // Iterate over all domain documents and add appropriate entries - const allDocs = await this.storage.rawFindAll( - domain, - { _class: { $in: classes } }, - { projection: { _class: 1, _id: 1 } } - ) - try { - let processed = 0 - while (true) { - indexing() - const docs = allDocs.splice(0, 1000) - if (docs.length === 0) { - break - } - const states = toIdMap( - await this.storage.rawFindAll( - DOMAIN_DOC_INDEX_STATE, - { _id: { $in: docs.map((it) => it._id) } }, - { - projection: { _id: 1 } - } - ) - ) - // Find missing documents - const missingDocs = docs - .filter((it) => !states.has(it._id)) - .map((it) => createStateDoc(it._id, it._class, { needIndex: true, removed: false })) - - if (missingDocs.length > 0) { - await this.storage.upload(ctx, DOMAIN_DOC_INDEX_STATE, missingDocs) - } - processed += docs.length - ctx.info('check-processed', { processed, allDocs: allDocs.length, domain }) - } - } catch (err: any) { - ctx.error('failed to restore index state', { err }) - } - }) - } - // Mark missing classes as deleted ones. - await this.storage.rawUpdate( - DOMAIN_DOC_INDEX_STATE, - { objectClass: { $nin: allIndexed } }, - { - removed: true, - needIndex: true - } - ) - await this.addMigration(ctx, docStructure) - } + .map((it) => ({ + domain: it[0], + classes: it[1] + })) } - async clearIndex (onlyDrop = false): Promise { - if (!onlyDrop) { - const ctx = this.metrics - const migrations = await this.storage.findAll(ctx, core.class.MigrationState, { - plugin: coreId, - state: { - $in: [indexes, fullReindex, docStructure] - } - }) + async reindex ( + ctx: MeasureContext, + domain: Domain, + classes: Ref>[], + control?: ConsumerControl + ): Promise { + ctx.warn('verify document structure', { workspace: this.workspace.uuid }) - const refs = migrations.map((it) => it._id) - await this.storage.clean(ctx, DOMAIN_MIGRATION, refs) - } else { - if (this.workspace.dataId != null) { - await this.fulltextAdapter.clean(this.metrics, this.workspace.dataId as unknown as WorkspaceUuid) + let processed = 0 + await ctx.with('reindex-domain', { domain }, async (ctx) => { + // Iterate over all domain documents and add appropriate entries + const allDocs = this.storage.rawFind(ctx, domain) + try { + let lastPrint = 0 + const pushQueue = new ElasticPushQueue(this.fulltextAdapter, this.workspace, ctx, control) + while (true) { + if (control !== undefined) { + await control?.heartbeat() + } + const docs = await allDocs.find(ctx) + if (docs.length === 0) { + break + } + const byClass = groupByArray>>(docs, (it) => it._class) + + for (const [v, values] of byClass.entries()) { + if (!isClassIndexable(this.hierarchy, v, this.contexts)) { + // Skip non indexable classes + continue + } + + await this.indexDocuments(ctx, v, values, pushQueue) + await control?.heartbeat() + } + + processed += docs.length + + // Define the thresholds for logging + + // Find the next threshold to print + + const now = platformNow() + if (now - lastPrint > 2500) { + ctx.info('processed', { processed, elapsed: Math.round(now - lastPrint), domain }) + lastPrint = now + } + } + await pushQueue.waitProcessing() + } catch (err: any) { + ctx.error('failed to restore index state', { err }) + } finally { + await allDocs.close() } - await this.fulltextAdapter.clean(this.metrics, this.workspace.uuid) + }) + ctx.warn('reinex done', { domain, processed }) + } + + async dropWorkspace (control?: ConsumerControl): Promise { + if (this.workspace.dataId != null) { + await this.fulltextAdapter.clean(this.metrics, this.workspace.dataId as unknown as WorkspaceUuid) } + await this.fulltextAdapter.clean(this.metrics, this.workspace.uuid) } broadcastClasses = new Set>>() @@ -320,232 +290,21 @@ export class FullTextIndexPipeline implements FullTextPipeline { await this.storage.upload(ctx, DOMAIN_MIGRATION, [mstate]) } - async doIndexing (indexing: () => void): Promise { - while (!this.cancelling) { - // Clear triggers - this.triggerCounts = 0 - - const { classUpdate: _classes, processed } = await this.metrics.with( - 'processIndex', - { workspace: this.workspace.uuid }, - (ctx) => this.processIndex(ctx, indexing) - ) - - // Also update doc index state queries. - _classes.forEach((it) => this.broadcastClasses.add(it)) - - if (this.triggerCounts > 0) { - this.metrics.info('No wait, trigger counts', { triggerCount: this.triggerCounts }) - } - - if (this.triggerCounts === 0) { - if (processed > 0) { - this.metrics.warn('Indexing complete', { indexId: this.indexId, workspace: this.workspace.uuid, processed }) - } - if (!this.cancelling) { - // We need to send index update event - if (this.broadcasts === 0) { - this.broadcasts++ - setTimeout(() => { - this.broadcasts = 0 - this.broadcastClasses.delete(core.class.DocIndexState) - if (this.broadcastClasses.size > 0) { - const toSend = Array.from(this.broadcastClasses.values()) - this.broadcastClasses.clear() - this.broadcastUpdate(this.metrics, toSend) - } - }, 5000) + scheduleBroadcast (): void { + if (!this.cancelling) { + // We need to send index update event + if (this.broadcasts === 0) { + this.broadcasts++ + setTimeout(() => { + this.broadcasts = 0 + if (this.broadcastClasses.size > 0) { + const toSend = Array.from(this.broadcastClasses.values()) + this.broadcastClasses.clear() + this.broadcastUpdate(this.metrics, toSend) } - - await new Promise((resolve) => { - this.triggerIndexing = () => { - this.triggerCounts++ - resolve(null) - } - }) - } + }, 5000) } } - this.metrics.warn('Exit indexer', { indexId: this.indexId, workspace: this.workspace.uuid }) - } - - async indexDocuments ( - ctx: MeasureContext, - docs: { - _class: Ref> - _id: Ref - }[] - ): Promise { - const parts = [...docs] - while (parts.length > 0) { - const part = parts.splice(0, 50) - - const states = toIdMap( - await this.storage.findAll( - ctx, - core.class.DocIndexState, - { _id: { $in: part.map((it) => it._id as Ref) } }, - { projection: { _id: 1 } } - ) - ) - - const toInsert: DocIndexState[] = [] - const toUpdate: Ref[] = [] - for (const p of part) { - if (states.has(p._id as Ref)) { - toUpdate.push(p._id as Ref) - } else { - // Space will be updated on indexing - toInsert.push( - createStateDoc(p._id, p._class, { - needIndex: true, - removed: false - }) - ) - } - } - if (toInsert.length > 0) { - await this.storage.upload(ctx, DOMAIN_DOC_INDEX_STATE, toInsert) - } - if (toUpdate.length > 0) { - await this.storage.rawUpdate( - DOMAIN_DOC_INDEX_STATE, - { _id: { $in: toUpdate } }, - { needIndex: true } - ) - } - } - this.triggerIndexing() - } - - private async processIndex ( - ctx: MeasureContext, - indexing: () => void - ): Promise<{ classUpdate: Ref>[], processed: number }> { - const _classUpdate = new Set>>() - let processed = 0 - await rateLimiter.exec(async () => { - let st = Date.now() - - let groupBy = await this.storage.groupBy(ctx, DOMAIN_DOC_INDEX_STATE, 'objectClass', { needIndex: true }) - const total = Array.from(groupBy.values()).reduce((a, b) => a + b, 0) - while (true) { - try { - if (this.cancelling) { - return Array.from(_classUpdate.values()) - } - const q: DocumentQuery = { - needIndex: true - } - let domainLookup: Domain | undefined - // set a class for more priority domains if they have documents - for (const [domain, classes] of this.byDomain.entries()) { - // Calc if we have classes pending for this domain. - const pending = classes.filter((it) => (groupBy.get(it) ?? 0) > 0) - if (pending.length > 0) { - // We have some classes pending indexing in this domain. - q.objectClass = { $in: pending } - domainLookup = domain - break - } - } - if (domainLookup === undefined) { - // Nothing to index, we should remove pending needIndex requests, since they are not required anymore - if (Array.from(groupBy.values()).reduce((a, b) => a + b, 0) > 0) { - await this.storage.rawDeleteMany(DOMAIN_DOC_INDEX_STATE, { - needIndex: true, - objectClass: { $in: Array.from(groupBy.keys()) as Ref>[] } - }) - } - break - } - - let result: FindResult | WithLookup[] = await ctx.with( - 'get-indexable', - {}, - (ctx) => { - return this.storage.findAll(ctx, core.class.DocIndexState, q, { - limit: globalIndexer.processingSize, - skipClass: true, - skipSpace: true, - domainLookup: { - field: '_id', - domain: domainLookup - } - }) - } - ) - if (result.length === 0) { - if (q.objectClass !== undefined) { - // If we searched with objectClass, we need to recalculate groupBy - groupBy = await this.storage.groupBy(ctx, DOMAIN_DOC_INDEX_STATE, 'objectClass', { needIndex: true }) - continue - } - // No more results - break - } - - indexing() // Update no sleep - - for (const r of result) { - if (typeof r._id === 'object') { - r._id = (r._id as any).toString() - } - const p = groupBy.get(r.objectClass) - if (p !== undefined) { - if (p - 1 === 0) { - groupBy.delete(r.objectClass) - } else { - groupBy.set(r.objectClass, p - 1) - } - } - } - - const toRemove: DocIndexState[] = await this.processRemove(ctx, result) - result = result.filter((it) => !it.removed) - // Check and remove missing class documents. - result = result.filter((doc) => { - const _class = this.model.findObject(doc.objectClass) - if (_class === undefined) { - // no _class present, remove doc - toRemove.push(doc) - return false - } - return true - }) - - if (toRemove.length > 0) { - try { - await this.storage.clean( - this.metrics, - DOMAIN_DOC_INDEX_STATE, - toRemove.map((it) => it._id) - ) - } catch (err: any) { - Analytics.handleError(err) - // QuotaExceededError, ignore - } - } - - await this.processDocuments(result, ctx, _classUpdate) - processed += result.length - if (Date.now() - st > 5000) { - st = Date.now() - this.metrics.info('Full text: Indexing', { - indexId: this.indexId, - workspace: this.workspace.uuid, - domain: domainLookup, - processed, - total - }) - } - } catch (err: any) { - Analytics.handleError(err) - this.metrics.error('error during index', { error: err }) - } - } - }) - return { classUpdate: Array.from(_classUpdate.values()), processed } } private async findParents (ctx: MeasureContext, docs: AttachedDoc[]): Promise> { @@ -581,51 +340,12 @@ export class FullTextIndexPipeline implements FullTextPipeline { return toIdMap(result) } - private async processDocuments ( - result: WithLookup[], + async indexDocuments ( ctx: MeasureContext, - _classUpdate: Set>> + _class: Ref>, + docs: Doc[], + pushQueue: ElasticPushQueue ): Promise { - const contextData = this.createContextData() - ctx.contextData = contextData - let indexedDocs: IndexedDoc[] = [] - // Find documents matching query - - const byClass = groupByArray, Ref>>(result, (it) => it.objectClass) - - const docUpdates = new Map, Partial>() - - const pushQueue = new RateLimiter(5) - - const pushToIndex = async (): Promise => { - const docs = [...indexedDocs] - indexedDocs = [] - if (docs.length === 0) { - return - } - await pushQueue.add(async () => { - try { - try { - await ctx.with('push-elastic', {}, () => this.fulltextAdapter.updateMany(ctx, this.workspace.uuid, docs)) - } catch (err: any) { - Analytics.handleError(err) - // Try to push one by one - await ctx.with('push-elastic-by-one', {}, async () => { - for (const d of docs) { - try { - await this.fulltextAdapter.update(ctx, this.workspace.uuid, d.id, d) - } catch (err2: any) { - Analytics.handleError(err2) - } - } - }) - } - } catch (err: any) { - Analytics.handleError(err) - } - }) - } - let spaceDocs: IdMap | undefined const updateSpaces = async (): Promise => { spaceDocs = toIdMap( @@ -635,7 +355,7 @@ export class FullTextIndexPipeline implements FullTextPipeline { core.class.Space, { _id: { - $in: Array.from(new Set(result.map((doc) => doc.space))) + $in: Array.from(new Set(docs.map((doc) => doc.space))) } }, { @@ -646,144 +366,195 @@ export class FullTextIndexPipeline implements FullTextPipeline { ) ) } + const searchPresenter = findSearchPresenter(this.hierarchy, _class) + + let parentDocs: IdMap | undefined const rateLimit = new RateLimiter(10) - - for (const [v, values] of byClass.entries()) { - const searchPresenter = findSearchPresenter(this.hierarchy, v) - // Obtain real documents - const valueIds = new Map(values.map((it) => [it._id, it])) - const docs = values.map((it) => it.$lookup?._id as Doc).filter((it) => it !== undefined) - let parentDocs: IdMap | undefined - - // We need to add all to broadcast update - for (const doc of docs) { - if (typeof doc._id === 'object') { - doc._id = (doc._id as any).toString() - } - _classUpdate.add(doc._class) - if (this.cancelling) { - return - } - const docState = valueIds.get(doc._id as Ref) as WithLookup - const indexedDoc = createIndexedDoc(doc, this.hierarchy.findAllMixins(doc), doc.space) - - await rateLimit.add(async () => { - await ctx.with('process-document', { _class: doc._class }, async (ctx) => { - try { - // Copy content attributes as well. - const docUpdate: DocumentUpdate = { - needIndex: false - } - if (docState.space !== doc.space) { - docUpdate.space = doc.space - } - - // TODO: Check if need to include parent values here as well - - // Collect all indexable values - const attributes = getFullTextIndexableAttributes(this.hierarchy, doc._class) - const content = getContent(this.hierarchy, attributes, doc) - - indexedDoc.fulltextSummary = '' - - for (const [, v] of Object.entries(content)) { - if (v.attr.type._class === core.class.TypeBlob) { - await this.processBlob(ctx, v, doc, indexedDoc) - continue - } - - if (v.attr.type._class === core.class.TypeCollaborativeDoc) { - await this.processCollaborativeDoc(ctx, v, indexedDoc) - continue - } - if ((isFullTextAttribute(v.attr) || v.attr.isCustom === true) && v.value !== undefined) { - if (v.attr.type._class === core.class.TypeMarkup) { - ctx.withSync('markup-to-json-text', {}, () => { - indexedDoc.fulltextSummary += '\n' + jsonToText(markupToJSON(v.value)) - }) - } else { - indexedDoc.fulltextSummary += '\n' + v.value - } - - continue - } - - if (isIndexedAttribute(v.attr)) { - // We need to put indexed attr in place - - // Check for content changes and collect update - const dKey = docKey(v.attr.name, v.attr.attributeOf) - if (dKey !== '_class') { - if (typeof v.value !== 'object') { - indexedDoc[dKey] = v.value - } else { - // We need to extract only values - indexedDoc[dKey] = extractValues(v.value) - } - } - continue - } - } - - // trim to large content - if (indexedDoc.fulltextSummary.length > textLimit) { - indexedDoc.fulltextSummary = indexedDoc.fulltextSummary.slice(0, textLimit) - } - - if (searchPresenter !== undefined) { - await ctx.with('update-search-presenter', { _class: doc._class }, async () => { - if (parentDocs === undefined) { - parentDocs = this.hierarchy.isDerived(v, core.class.AttachedDoc) - ? await this.findParents(ctx, docs as unknown as AttachedDoc[]) - : undefined - } - const parentDoc = parentDocs?.get((doc as AttachedDoc).attachedTo as Ref) - if (spaceDocs === undefined) { - await updateSpaces() - } - const spaceDoc = spaceDocs?.get(doc.space) // docState.$lookup?.space - await updateDocWithPresenter(this.hierarchy, doc, indexedDoc, parentDoc, spaceDoc, searchPresenter) - }) - } - - indexedDoc.id = doc._id - indexedDoc.space = doc.space - indexedDocs.push(indexedDoc) - - if (indexedDocs.length > 25) { - void pushToIndex() - } - - docUpdates.set(docState._id, docUpdate) - } catch (err: any) { - ctx.error('failed to process document', { - id: doc._id, - class: doc._class, - workspace: this.workspace.uuid - }) - Analytics.handleError(err) - } - }) - }) + // We need to add all to broadcast update + for (const doc of docs) { + if (typeof doc._id === 'object') { + doc._id = (doc._id as any).toString() } + this.broadcastClasses.add(doc._class) + if (this.cancelling) { + return + } + const indexedDoc = createIndexedDoc(doc, this.hierarchy.findAllMixins(doc), doc.space) + + await rateLimit.exec(async () => { + await ctx.with('process-document', { _class: doc._class }, async (ctx) => { + try { + // Collect all indexable values + const attributes = getFullTextIndexableAttributes(this.hierarchy, doc._class) + const content = getContent(this.hierarchy, attributes, doc) + + indexedDoc.fulltextSummary = '' + + for (const [, v] of Object.entries(content)) { + if (v.attr.type._class === core.class.TypeBlob) { + await this.processBlob(ctx, v, doc, indexedDoc) + continue + } + + if (v.attr.type._class === core.class.TypeCollaborativeDoc) { + await this.processCollaborativeDoc(ctx, v, indexedDoc) + continue + } + if ((isFullTextAttribute(v.attr) || v.attr.isCustom === true) && v.value !== undefined) { + if (v.attr.type._class === core.class.TypeMarkup) { + ctx.withSync('markup-to-json-text', {}, () => { + indexedDoc.fulltextSummary += '\n' + jsonToText(markupToJSON(v.value)) + }) + } else { + indexedDoc.fulltextSummary += '\n' + v.value + } + + continue + } + + if (isIndexedAttribute(v.attr)) { + // We need to put indexed attr in place + + // Check for content changes and collect update + const dKey = docKey(v.attr.name, v.attr.attributeOf) + if (dKey !== '_class') { + if (typeof v.value !== 'object') { + indexedDoc[dKey] = v.value + } else { + // We need to extract only values + indexedDoc[dKey] = extractValues(v.value) + } + } + continue + } + } + + // trim to large content + if (indexedDoc.fulltextSummary.length > textLimit) { + indexedDoc.fulltextSummary = indexedDoc.fulltextSummary.slice(0, textLimit) + } + + if (searchPresenter !== undefined) { + await ctx.with('update-search-presenter', { _class: doc._class }, async () => { + if (parentDocs === undefined) { + parentDocs = this.hierarchy.isDerived(_class, core.class.AttachedDoc) + ? await this.findParents(ctx, docs as unknown as AttachedDoc[]) + : undefined + } + const parentDoc = parentDocs?.get((doc as AttachedDoc).attachedTo) + if (spaceDocs === undefined) { + await updateSpaces() + } + const spaceDoc = spaceDocs?.get(doc.space) // docState.$lookup?.space + await updateDocWithPresenter(this.hierarchy, doc, indexedDoc, parentDoc, spaceDoc, searchPresenter) + }) + } + + indexedDoc.id = doc._id + indexedDoc.space = doc.space + + await pushQueue.push(indexedDoc) + } catch (err: any) { + ctx.error('failed to process document', { + id: doc._id, + class: doc._class, + workspace: this.workspace.uuid + }) + Analytics.handleError(err) + } + }) + }) } await rateLimit.waitProcessing() + } - await pushToIndex() - await pushQueue.waitProcessing() + public async processDocuments (ctx: MeasureContext, result: TxCUD[], control: ConsumerControl): Promise { + const contextData = this.createContextData() + ctx.contextData = contextData + // Find documents matching query - await ctx.with('update-index-state', {}, async (ctx) => { - const ids = [...docUpdates.entries()] - const groups = groupByArray(ids, (it) => JSON.stringify(it[1])) - for (const [, values] of groups.entries()) { - const ids = values.map((it) => it[0]) - while (ids.length > 0) { - const part = ids.splice(0, 200) - await this.storage.rawUpdate(DOMAIN_DOC_INDEX_STATE, { _id: { $in: part } }, values[0][1]) - } + // We need to update hierarchy and local model if required. + + for (const tx of result) { + this.hierarchy.tx(tx) + const domain = this.hierarchy.findDomain(tx.objectClass) + if (domain === DOMAIN_MODEL) { + await this.model.tx(tx) } - }) + } + + const byClass = groupByArray, Ref>>(result, (it) => it.objectClass) + + const pushQueue = new ElasticPushQueue(this.fulltextAdapter, this.workspace, ctx, control) + + const toRemove: { _id: Ref, _class: Ref> }[] = [] + + for (const [v, values] of byClass.entries()) { + if (!isClassIndexable(this.hierarchy, v, this.contexts)) { + // Skip non indexable classes + continue + } + + // We need to load documents from storage + const docs: Doc[] = await this.loadDocsFromTx(values, toRemove, ctx, v) + + await this.indexDocuments(ctx, v, docs, pushQueue) + } + + try { + if (toRemove.length !== 0) { + // We need to add broadcast information + for (const _cl of new Set(toRemove.values().map((it) => it._class))) { + this.broadcastClasses.add(_cl) + } + await this.fulltextAdapter.remove( + ctx, + this.workspace.uuid, + toRemove.map((it) => it._id) + ) + } + } catch (err: any) { + Analytics.handleError(err) + } + + await pushQueue.waitProcessing() + this.scheduleBroadcast() + } + + private async loadDocsFromTx ( + values: TxCUD>[], + toRemove: { _id: Ref>, _class: Ref> }[], + ctx: MeasureContext, + v: Ref>> + ): Promise { + const byDoc = groupByArray(values, (it) => it.objectId) + + const docsToRetrieve = new Set>() + + const docs: Doc[] = [] + + // We need to find documents we need to retrieve + for (const [docId, txes] of byDoc.entries()) { + // Check if we have create tx, we do not need to retrieve a doc, just combine all updates + txes.sort((a, b) => a.modifiedOn - b.modifiedOn) + const doc = TxProcessor.buildDoc2Doc(txes) + + switch (doc) { + case null: + toRemove.push({ _id: docId, _class: txes[0].objectClass }) + break + case undefined: + docsToRetrieve.add(docId) + break + default: + docs.push(doc) + } + } + if (docsToRetrieve.size > 0) { + docs.push(...(await this.storage.findAll(ctx, v, { _id: { $in: Array.from(docsToRetrieve) } }))) + } + return docs } private createContextData (): SessionDataImpl { @@ -875,7 +646,7 @@ export class FullTextIndexPipeline implements FullTextPipeline { id: doc._id, _class: doc._class, field: v.attr.name, - err, + err: err.message, workspace: this.workspace.uuid }) } @@ -926,23 +697,6 @@ export class FullTextIndexPipeline implements FullTextPipeline { indexedDoc.fulltextSummary += '\n' + textContent } } - - private async processRemove (ctx: MeasureContext, docs: DocIndexState[]): Promise { - try { - const toRemove = docs.filter((it) => it.removed) - if (toRemove.length !== 0) { - await this.fulltextAdapter.remove( - ctx, - this.workspace.uuid, - toRemove.map((it) => it._id) - ) - } - return toRemove - } catch (err: any) { - Analytics.handleError(err) - } - return [] - } } function isBlobAllowed (contentType: string): boolean { return ( diff --git a/server/indexer/src/indexer/types.ts b/server/indexer/src/indexer/types.ts index cae2aefe02..6f1336a4b5 100644 --- a/server/indexer/src/indexer/types.ts +++ b/server/indexer/src/indexer/types.ts @@ -16,8 +16,6 @@ import { type Class, type Doc, - type DocIndexState, - type DocumentUpdate, type FullTextSearchContext, type Hierarchy, type ModelDb, @@ -34,12 +32,3 @@ export interface FullTextPipeline { cancelling: boolean } - -/** - * @public - */ -export type DocUpdateHandler = (doc: DocIndexState, update: DocumentUpdate) => Promise - -export const indexes = 'verify-indexes-v2' -export const fullReindex = 'full-text-indexer-v5' -export const docStructure = 'full-text-structure-v6' diff --git a/server/indexer/src/indexer/utils.ts b/server/indexer/src/indexer/utils.ts index a4d634633f..80184db437 100644 --- a/server/indexer/src/indexer/utils.ts +++ b/server/indexer/src/indexer/utils.ts @@ -13,20 +13,17 @@ // limitations under the License. // -import core, { +import { type AnyAttribute, type Class, - type Data, type Doc, - type DocIndexState, type FullTextSearchContext, getFullTextContext, type Hierarchy, - type Obj, type Ref, type Space } from '@hcengineering/core' -import plugin, { type IndexedDoc } from '@hcengineering/server-core' +import { type IndexedDoc } from '@hcengineering/server-core' import { type FullTextPipeline } from './types' export { docKey, isFullTextAttribute } from '@hcengineering/core' @@ -55,25 +52,6 @@ export function getContent ( return attrs } -/** - * @public - */ -export function createStateDoc ( - id: Ref, - objectClass: Ref>, - data: Omit, 'objectClass'> & { space?: Ref } -): DocIndexState { - return { - _class: core.class.DocIndexState, - _id: id as Ref, - space: data.space ?? plugin.space.DocIndexState, - objectClass, - modifiedBy: core.account.System, - modifiedOn: Date.now(), - ...data - } -} - /** * @public */ diff --git a/models/server-fulltext/.eslintrc.js b/server/kafka/.eslintrc.js similarity index 85% rename from models/server-fulltext/.eslintrc.js rename to server/kafka/.eslintrc.js index c1cf82cba0..ce90fb9646 100644 --- a/models/server-fulltext/.eslintrc.js +++ b/server/kafka/.eslintrc.js @@ -1,5 +1,5 @@ module.exports = { - extends: ['./node_modules/@hcengineering/platform-rig/profiles/model/eslint.config.json'], + extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'], parserOptions: { tsconfigRootDir: __dirname, project: './tsconfig.json' diff --git a/models/server-fulltext/.npmignore b/server/kafka/.npmignore similarity index 100% rename from models/server-fulltext/.npmignore rename to server/kafka/.npmignore diff --git a/models/server-fulltext/config/rig.json b/server/kafka/config/rig.json similarity index 85% rename from models/server-fulltext/config/rig.json rename to server/kafka/config/rig.json index 2f6be36605..78cc5a1733 100644 --- a/models/server-fulltext/config/rig.json +++ b/server/kafka/config/rig.json @@ -1,5 +1,5 @@ { "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", "rigPackageName": "@hcengineering/platform-rig", - "rigProfile": "model" + "rigProfile": "node" } diff --git a/server-plugins/fulltext-resources/jest.config.js b/server/kafka/jest.config.js similarity index 100% rename from server-plugins/fulltext-resources/jest.config.js rename to server/kafka/jest.config.js diff --git a/server-plugins/fulltext-resources/package.json b/server/kafka/package.json similarity index 74% rename from server-plugins/fulltext-resources/package.json rename to server/kafka/package.json index a7fdd1933b..9f8d47bfc4 100644 --- a/server-plugins/fulltext-resources/package.json +++ b/server/kafka/package.json @@ -1,23 +1,19 @@ { - "name": "@hcengineering/server-fulltext-resources", + "name": "@hcengineering/kafka", "version": "0.6.0", "main": "lib/index.js", "svelte": "src/index.ts", "types": "types/index.d.ts", - "files": [ - "lib/**/*", - "types/**/*", - "tsconfig.json" - ], "author": "Anticrm Platform Contributors", + "template": "@hcengineering/node-package", "license": "EPL-2.0", "scripts": { "build": "compile", "build:watch": "compile", + "test": "jest --passWithNoTests --silent --forceExit", "format": "format src", - "test": "jest --passWithNoTests --silent", "_phase:build": "compile transpile src", - "_phase:test": "jest --passWithNoTests --silent", + "_phase:test": "jest --passWithNoTests --silent --forceExit", "_phase:format": "format src", "_phase:validate": "compile validate" }, @@ -32,6 +28,7 @@ "eslint-config-standard-with-typescript": "^40.0.0", "prettier": "^3.1.0", "typescript": "^5.3.3", + "@types/node": "~20.11.16", "jest": "^29.7.0", "ts-jest": "^29.1.1", "@types/jest": "^29.5.5" @@ -39,6 +36,8 @@ "dependencies": { "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", - "@hcengineering/server-core": "^0.6.1" + "@hcengineering/server-core": "^0.6.1", + "@hcengineering/storage": "^0.6.0", + "kafkajs": "^2.2.4" } } diff --git a/server/kafka/src/index.ts b/server/kafka/src/index.ts new file mode 100644 index 0000000000..1f6ecf1ed7 --- /dev/null +++ b/server/kafka/src/index.ts @@ -0,0 +1,262 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type MeasureContext, type WorkspaceUuid } from '@hcengineering/core' +import { + QueueTopic, + type ConsumerControl, + type ConsumerHandle, + type ConsumerMessage, + type PlatformQueue, + type PlatformQueueProducer +} from '@hcengineering/server-core' +import { Kafka, Partitioners, type Consumer, type Producer } from 'kafkajs' +import type * as tls from 'tls' + +export interface QueueConfig { + postfix: string // a topic prefix to be used do distinguish between staging and production topics in same broker + brokers: string[] + clientId: string + region: string + + ssl?: tls.ConnectionOptions | boolean +} + +/** + * Query config parser in the form of 'brokers;clientId' + * brokers are comma separated + */ +export function parseQueueConfig (config: string, serviceId: string, region: string): QueueConfig { + const [brokers, postfix] = config.split(';') + return { + postfix: postfix ?? '', // Empty by default + brokers: brokers.split(','), + clientId: serviceId, + region + } +} + +function getKafkaTopicId (topic: QueueTopic, config: QueueConfig): string { + if (config.region !== '') { + return `${config.region}.${topic}${config.postfix ?? ''}` + } + return `${topic}${config.postfix ?? ''}` +} + +class PlatformQueueImpl implements PlatformQueue { + constructor ( + private readonly kafka: Kafka, + readonly config: QueueConfig + ) {} + + getClientId (): string { + return this.config.clientId + } + + createProducer(ctx: MeasureContext, topic: QueueTopic): PlatformQueueProducer { + return new PlatformQueueProducerImpl(ctx, this.kafka, getKafkaTopicId(topic, this.config)) + } + + createConsumer( + ctx: MeasureContext, + topic: QueueTopic, + groupId: string, + onMessage: (msg: ConsumerMessage[], queue: ConsumerControl) => Promise, + options?: { + fromBegining?: boolean + } + ): ConsumerHandle { + return new PlatformQueueConsumerImpl(ctx, this.kafka, this.config, topic, groupId, onMessage, options) + } + + async createTopics (tx: number): Promise { + const topics = new Set(await this.kafka.admin({}).listTopics()) + if (!topics.has(QueueTopic.Tx)) { + await this.kafka.admin().createTopics({ + topics: [ + { + topic: getKafkaTopicId(QueueTopic.Tx, this.config), + numPartitions: tx + } + ] + }) + } + } +} + +class PlatformQueueProducerImpl implements PlatformQueueProducer { + txProducer: Producer + connected: Promise | undefined + constructor ( + readonly ctx: MeasureContext, + kafka: Kafka, + private readonly topic: string + ) { + this.txProducer = kafka.producer({ + allowAutoTopicCreation: true, + createPartitioner: Partitioners.DefaultPartitioner + }) + this.connected = this.ctx.with('connect-broker', {}, () => this.txProducer.connect()) + } + + async send (id: WorkspaceUuid | string, msgs: any[]): Promise { + if (this.connected !== undefined) { + await this.connected + this.connected = undefined + } + await this.ctx.with('send', { topic: this.topic }, () => + this.txProducer.send({ + topic: this.topic, + messages: msgs.map((m) => ({ + key: Buffer.from(`${id}`), + value: Buffer.from(JSON.stringify(m)) + })) + }) + ) + } + + async close (): Promise { + await this.ctx.with('disconnect', {}, () => this.txProducer.disconnect()) + } +} + +class PlatformQueueConsumerImpl implements ConsumerHandle { + connected = false + cc: Consumer + constructor ( + readonly ctx: MeasureContext, + readonly kafka: Kafka, + readonly config: QueueConfig, + private readonly topic: QueueTopic, + groupId: string, + private readonly onMessage: (msg: ConsumerMessage[], queue: ConsumerControl) => Promise, + private readonly options?: { + fromBegining?: boolean + } + ) { + this.cc = this.kafka.consumer({ groupId: `${getKafkaTopicId(this.topic, this.config)}-${groupId}` }) + + void this.start().catch((err) => { + ctx.error('failed to consume', { err }) + }) + } + + async start (): Promise { + await this.doConnect() + await this.doSubscribe() + + await this.cc.run({ + eachMessage: async ({ topic, message, pause, heartbeat }) => { + await this.onMessage( + [ + { + id: message.key?.toString() ?? '', + value: [JSON.parse(message.value?.toString() ?? '{}')] + } + ], + { + heartbeat, + pause + } + ) + } + // , // TODO: Finish testinf + // eachBatch: async ({ batch, pause, heartbeat, resolveOffset }) => { + // const queueInfo = { + // pause, + // heartbeat + // } + // const batchMessages = batch.messages + + // const currentMsg: ConsumerMessage = { + // id: '', + // value: [] + // } + // let lastOffset: string = ' ' + + // const sendLast = async (): Promise => { + // await this.onMessage([currentMsg], queueInfo) + // // Mark last offset as cusomed + // resolveOffset(lastOffset) + // await heartbeat() + // } + + // for (const v of batchMessages) { + // const id = v.key?.toString() ?? '' + // if (currentMsg.id !== id && currentMsg.value.length > 0) { + // await sendLast() // Send last message + // currentMsg.id = id + // currentMsg.value = [] + // } + // lastOffset = v.offset + // currentMsg.value.push(JSON.parse(v.value?.toString() ?? '{}')) + // } + // await sendLast() + // } + }) + } + + async doConnect (): Promise { + this.cc.on('consumer.connect', () => { + this.connected = true + this.ctx.info('consumer connected to queue') + }) + this.cc.on('consumer.disconnect', () => { + this.connected = false + this.ctx.warn('consumer disconnected from queue') + }) + await this.cc.connect() + } + + async doSubscribe (): Promise { + await this.cc.subscribe({ + topic: getKafkaTopicId(this.topic, this.config), + fromBeginning: this.options?.fromBegining + }) + } + + isConnected (): boolean { + return this.connected + } + + shutdown (): Promise { + return this.cc.disconnect() + } +} + +/** + * Constructs a platform queue. + */ +export function getPlatformQueue (serviceId: string, region?: string): PlatformQueue { + const queueConfig = process.env.QUEUE_CONFIG ?? 'huly.local:9092' + if (queueConfig === undefined) { + throw new Error('Please provide queue config') + } + const config = parseQueueConfig(queueConfig, serviceId, region ?? process.env.REGION ?? '') + return createPlatformQueue(config) +} + +export function createPlatformQueue (config: QueueConfig): PlatformQueue { + console.info({ message: 'Using queue', config }) + + return new PlatformQueueImpl( + new Kafka({ + clientId: config.clientId, + brokers: config.brokers, + ssl: config.ssl + }), + config + ) +} diff --git a/models/server-fulltext/tsconfig.json b/server/kafka/tsconfig.json similarity index 89% rename from models/server-fulltext/tsconfig.json rename to server/kafka/tsconfig.json index 9765b6086d..f017cc597c 100644 --- a/models/server-fulltext/tsconfig.json +++ b/server/kafka/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "./node_modules/@hcengineering/platform-rig/profiles/model/tsconfig.json", + "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json", "compilerOptions": { "rootDir": "./src", diff --git a/server/middleware/src/fulltext.ts b/server/middleware/src/fulltext.ts index 7bce329c68..07cc53da2a 100644 --- a/server/middleware/src/fulltext.ts +++ b/server/middleware/src/fulltext.ts @@ -16,7 +16,6 @@ import { Analytics } from '@hcengineering/analytics' import core, { docKey, - DOMAIN_DOC_INDEX_STATE, isFullTextAttribute, isIndexedAttribute, toFindResult, @@ -33,17 +32,9 @@ import core, { type Ref, type SearchOptions, type SearchQuery, - type SearchResult, - type SessionData, - type Tx + type SearchResult } from '@hcengineering/core' -import type { - IndexedDoc, - Middleware, - MiddlewareCreator, - PipelineContext, - TxMiddlewareResult -} from '@hcengineering/server-core' +import type { IndexedDoc, Middleware, MiddlewareCreator, PipelineContext } from '@hcengineering/server-core' import { BaseMiddleware } from '@hcengineering/server-core' /** * @public @@ -52,8 +43,6 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware { fulltextEndpoint: string contexts = new Map>, FullTextSearchContext>() - warmupTriggered: boolean = false - constructor ( context: PipelineContext, next: Middleware | undefined, @@ -79,54 +68,10 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware { } } - async handleBroadcast (ctx: MeasureContext): Promise { - if (ctx.contextData.fulltextUpdates !== undefined && ctx.contextData.fulltextUpdates.size > 0) { - const toUpdate = Array.from(ctx.contextData.fulltextUpdates.values()) - ctx.contextData.fulltextUpdates.clear() - await this.context.lowLevelStorage?.upload(ctx, DOMAIN_DOC_INDEX_STATE, toUpdate) - this.sendWarmup() - } - await super.handleBroadcast(ctx) - } - async init (ctx: MeasureContext): Promise { this.contexts = new Map( this.context.modelDb.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it]) ) - - // Send one warumup so reindex will - this.sendWarmup() - } - - sendWarmup (): void { - if (!this.warmupTriggered) { - this.warmupTriggered = true - setTimeout(() => { - this.warmupTriggered = false - void this._sendWarmup().catch(() => {}) - }, 25) - } - } - - async _sendWarmup (): Promise { - // Just send a warmup to indexer to start do something, or just reindex if some work are pending. - try { - await fetch(this.fulltextEndpoint + '/api/v1/warmup', { - method: 'PUT', - keepalive: true, - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - token: this.token - }) - }) - } catch (err: any) { - if (err?.cause?.code === 'ECONNRESET' || err?.cause?.code === 'ECONNREFUSED') { - return - } - Analytics.handleError(err) - } } async search( @@ -385,10 +330,6 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware { } } - tx (ctx: MeasureContext, txes: Tx[]): Promise { - return this.provideTx(ctx, txes) - } - async close (): Promise { try { await fetch(this.fulltextEndpoint + '/api/v1/close', { diff --git a/server/middleware/src/index.ts b/server/middleware/src/index.ts index f34b8d2f2d..e4a4c71f3b 100644 --- a/server/middleware/src/index.ts +++ b/server/middleware/src/index.ts @@ -35,3 +35,4 @@ export * from './spacePermissions' export * from './spaceSecurity' export * from './triggers' export * from './txPush' +export * from './queue' diff --git a/server/middleware/src/model.ts b/server/middleware/src/model.ts index 19f2c14525..8a6344260a 100644 --- a/server/middleware/src/model.ts +++ b/server/middleware/src/model.ts @@ -19,6 +19,7 @@ import core, { type DocumentQuery, type FindOptions, type FindResult, + type Hierarchy, type LoadModelResponse, type MeasureContext, type Ref, @@ -59,7 +60,8 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware { constructor ( context: PipelineContext, next: Middleware | undefined, - readonly systemTx: Tx[] + readonly systemTx: Tx[], + readonly filter?: (h: Hierarchy, model: Tx[]) => Tx[] ) { super(context, next) } @@ -69,16 +71,17 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware { ctx: MeasureContext, context: PipelineContext, next: Middleware | undefined, - systemTx: Tx[] + systemTx: Tx[], + filter?: (h: Hierarchy, model: Tx[]) => Tx[] ): Promise { - const middleware = new ModelMiddleware(context, next, systemTx) + const middleware = new ModelMiddleware(context, next, systemTx, filter) await middleware.init(ctx) return middleware } - static create (tx: Tx[]): MiddlewareCreator { + static create (tx: Tx[], filter?: (h: Hierarchy, model: Tx[]) => Tx[]): MiddlewareCreator { return (ctx, context, next) => { - return this.doCreate(ctx, context, next, tx) + return this.doCreate(ctx, context, next, tx, filter) } } @@ -116,9 +119,10 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware { ctx.warn('failed to apply model transaction, skipping', { tx: JSON.stringify(tx), err }) } } - this.context.modelDb.addTxes(ctx, model, true) + const fmodel = this.filter !== undefined ? this.filter(this.context.hierarchy, model) : model + this.context.modelDb.addTxes(ctx, fmodel, true) - this.setModel(model) + this.setModel(fmodel) } private addModelTx (tx: Tx): void { diff --git a/server/middleware/src/queue.ts b/server/middleware/src/queue.ts new file mode 100644 index 0000000000..191ec46b47 --- /dev/null +++ b/server/middleware/src/queue.ts @@ -0,0 +1,56 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type MeasureContext, type SessionData, type Tx } from '@hcengineering/core' +import { + BaseMiddleware, + type Middleware, + type MiddlewareCreator, + type PipelineContext, + type PlatformQueue, + type PlatformQueueProducer, + QueueTopic +} from '@hcengineering/server-core' + +export class QueueMiddleware extends BaseMiddleware { + txProducer: PlatformQueueProducer + connected: Promise | undefined + constructor ( + ctx: MeasureContext, + readonly context: PipelineContext, + protected readonly next: Middleware | undefined, + readonly queue: PlatformQueue + ) { + super(context, next) + this.txProducer = queue.createProducer(ctx, QueueTopic.Tx) + } + + static create (queue: PlatformQueue): MiddlewareCreator { + return async (ctx, context, next) => { + return new QueueMiddleware(ctx, context, next, queue) + } + } + + async handleBroadcast (ctx: MeasureContext): Promise { + if (this.connected !== undefined) { + await this.connected + this.connected = undefined + } + await Promise.all([ + this.provideBroadcast(ctx), + this.txProducer.send(this.context.workspace.uuid, ctx.contextData.broadcast.txes) + ]) + } +} diff --git a/server/middleware/src/spacePermissions.ts b/server/middleware/src/spacePermissions.ts index 4d27800d61..9ac194b406 100644 --- a/server/middleware/src/spacePermissions.ts +++ b/server/middleware/src/spacePermissions.ts @@ -174,7 +174,7 @@ export class SpacePermissionsMiddleware extends BaseMiddleware implements Middle if (!this.context.hierarchy.isDerived(createTx.objectClass, core.class.Space)) return if (this.isTypedSpaceClass(createTx.objectClass)) { const res = TxProcessor.buildDoc2Doc([createTx]) - if (res !== undefined) { + if (res != null) { this.addRestrictedSpace(res) } } else { diff --git a/server/mongo/src/__tests__/minmodel.ts b/server/mongo/src/__tests__/minmodel.ts index a59cd3c806..be5c1b7655 100644 --- a/server/mongo/src/__tests__/minmodel.ts +++ b/server/mongo/src/__tests__/minmodel.ts @@ -21,7 +21,6 @@ import core, { ClassifierKind, type Data, type Doc, - DOMAIN_DOC_INDEX_STATE, DOMAIN_MODEL, DOMAIN_RELATION, DOMAIN_TX, @@ -136,14 +135,6 @@ export function genMinModel (): TxCUD[] { domain: DOMAIN_MODEL }) ) - txes.push( - createClass(core.class.DocIndexState, { - label: 'DocIndexState' as IntlString, - extends: core.class.Doc, - kind: ClassifierKind.CLASS, - domain: DOMAIN_DOC_INDEX_STATE - }) - ) txes.push( createClass(core.class.Tx, { diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index 07e0dc1f28..db3c6acfb2 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -26,10 +26,10 @@ import core, { groupByArray, isOperator, matchQuery, + platformNow, toFindResult, withContext, type AssociationQuery, - type WorkspaceIds, type Class, type Doc, type DocInfo, @@ -62,16 +62,17 @@ import core, { type TxResult, type TxUpdateDoc, type WithLookup, - platformNow + type WorkspaceIds } from '@hcengineering/core' import { + calcHashHash, type DbAdapter, type DbAdapterHandler, type DomainHelperOperations, + type RawFindIterator, type ServerFindOptions, type StorageAdapter, - type TxAdapter, - calcHashHash + type TxAdapter } from '@hcengineering/server-core' import { type AbstractCursor, @@ -1145,6 +1146,33 @@ abstract class MongoAdapterBase implements DbAdapter { } } + rawFind (_ctx: MeasureContext, domain: Domain): RawFindIterator { + const ctx = _ctx.newChild('findRaw', { domain }) + const coll = this.db.collection(domain) + let iterator: FindCursor + + return { + find: async () => { + if (iterator === undefined) { + iterator = coll.find({}) + } + const d = await ctx.with('next', {}, () => iterator.next()) + const result: Doc[] = [] + if (d != null) { + result.push(this.stripHash(d) as Doc) + } + if (iterator.bufferedCount() > 0) { + result.push(...(this.stripHash(iterator.readBufferedDocuments()) as Doc[])) + } + return result ?? [] + }, + close: async () => { + await ctx.with('close', {}, () => iterator.close()) + ctx.end() + } + } + } + load (ctx: MeasureContext, domain: Domain, docs: Ref[]): Promise { return ctx.with('load', { domain }, async () => { if (docs.length === 0) { diff --git a/server/postgres/migrations/docIndexStateSchema.sql b/server/postgres/migrations/docIndexStateSchema.sql deleted file mode 100644 index 06ccb44898..0000000000 --- a/server/postgres/migrations/docIndexStateSchema.sql +++ /dev/null @@ -1,11 +0,0 @@ -ALTER TABLE doc_index_state -ADD "needIndex" bool; - -ALTER TABLE doc_index_state -DROP COLUMN "attachedTo"; - -UPDATE doc_index_state -SET "needIndex" = (data->>'needIndex')::boolean; - -ALTER TABLE doc_index_state -ALTER COLUMN "needIndex" SET NOT NULL; \ No newline at end of file diff --git a/server/postgres/src/__tests__/minmodel.ts b/server/postgres/src/__tests__/minmodel.ts index 41a9596717..2089d852ae 100644 --- a/server/postgres/src/__tests__/minmodel.ts +++ b/server/postgres/src/__tests__/minmodel.ts @@ -22,7 +22,6 @@ import core, { type Data, type Doc, type Domain, - DOMAIN_DOC_INDEX_STATE, DOMAIN_MODEL, DOMAIN_RELATION, DOMAIN_TX, @@ -155,14 +154,6 @@ export function genMinModel (): TxCUD[] { domain: DOMAIN_MODEL }) ) - txes.push( - createClass(core.class.DocIndexState, { - label: 'DocIndexState' as IntlString, - extends: core.class.Doc, - kind: ClassifierKind.CLASS, - domain: DOMAIN_DOC_INDEX_STATE - }) - ) txes.push( createClass(core.class.Tx, { diff --git a/server/postgres/src/schemas.ts b/server/postgres/src/schemas.ts index 1405af15c1..e25d708220 100644 --- a/server/postgres/src/schemas.ts +++ b/server/postgres/src/schemas.ts @@ -1,4 +1,4 @@ -import { DOMAIN_DOC_INDEX_STATE, DOMAIN_MODEL_TX, DOMAIN_RELATION, DOMAIN_SPACE, DOMAIN_TX } from '@hcengineering/core' +import { DOMAIN_MODEL_TX, DOMAIN_RELATION, DOMAIN_SPACE, DOMAIN_TX } from '@hcengineering/core' export type DataType = 'bigint' | 'bool' | 'text' | 'text[]' @@ -170,20 +170,6 @@ const userNotificationSchema: Schema = { } } -const docIndexStateSchema: Schema = { - ...baseSchema, - needIndex: { - type: 'bool', - notNull: true, - index: true - }, - objectClass: { - type: 'text', - notNull: true, - index: false - } -} - const timeSchema: Schema = { ...baseSchema, workslots: { @@ -305,7 +291,6 @@ export const domainSchemas: Record = { [translateDomain('time')]: timeSchema, [translateDomain('calendar')]: calendarSchema, [translateDomain('event')]: eventSchema, - [translateDomain(DOMAIN_DOC_INDEX_STATE)]: docIndexStateSchema, notification: notificationSchema, [translateDomain('notification-dnc')]: dncSchema, [translateDomain('notification-user')]: userNotificationSchema, diff --git a/server/postgres/src/storage.ts b/server/postgres/src/storage.ts index 7810a2a658..a06bdbb4b3 100644 --- a/server/postgres/src/storage.ts +++ b/server/postgres/src/storage.ts @@ -66,6 +66,7 @@ import { type DbAdapter, type DbAdapterHandler, type DomainHelperOperations, + type RawFindIterator, type ServerFindOptions, type TxAdapter } from '@hcengineering/server-core' @@ -1615,6 +1616,54 @@ abstract class PostgresAdapterBase implements DbAdapter { } } + rawFind (_ctx: MeasureContext, domain: Domain): RawFindIterator { + const ctx = _ctx.newChild('findRaw', { domain }) + + let initialized: boolean = false + let client: DBClient + + const tdomain = translateDomain(domain) + const schema = getSchema(domain) + + const workspaceId = this.workspaceId + + function createBulk (projection: string, limit = 1000): AsyncGenerator { + const sql = ` + SELECT ${projection} + FROM ${tdomain} + WHERE "workspaceId" = '${workspaceId}' + ` + + return createCursorGenerator(client.raw(), sql, undefined, schema, limit) + } + let bulk: AsyncGenerator + + return { + find: async () => { + if (!initialized) { + if (client === undefined) { + client = await this.client.reserve() + } + initialized = true + bulk = createBulk('*') + } + + const docs = await ctx.with('next', {}, () => bulk.next()) + if (docs.done === true || docs.value.length === 0) { + return [] + } + const result: Doc[] = [] + result.push(...(this.stripHash(docs.value) as Doc[])) + return result + }, + close: async () => { + await bulk.return([]) // We need to close generator, just in case + client?.release() + ctx.end() + } + } + } + load (ctx: MeasureContext, domain: Domain, docs: Ref[]): Promise { return ctx.with('load', { domain }, async () => { if (docs.length === 0) { diff --git a/server/server-pipeline/package.json b/server/server-pipeline/package.json index 15530213f9..82477eebbc 100644 --- a/server/server-pipeline/package.json +++ b/server/server-pipeline/package.json @@ -168,11 +168,10 @@ "@hcengineering/server-ai-bot-resources": "^0.6.0", "@hcengineering/time": "^0.6.0", "@hcengineering/time-assets": "^0.6.0", - "@hcengineering/server-fulltext": "^0.6.0", - "@hcengineering/server-fulltext-resources": "^0.6.0", "@hcengineering/survey": "^0.6.0", "@hcengineering/survey-assets": "^0.6.0", "@hcengineering/server-process": "^0.6.0", - "@hcengineering/server-process-resources": "^0.6.0" + "@hcengineering/server-process-resources": "^0.6.0", + "@hcengineering/kafka": "^0.6.0" } } diff --git a/server/server-pipeline/src/blobStorage.ts b/server/server-pipeline/src/blobStorage.ts index 279d4d7348..0f7904e521 100644 --- a/server/server-pipeline/src/blobStorage.ts +++ b/server/server-pipeline/src/blobStorage.ts @@ -39,6 +39,7 @@ import { PlatformError, unknownError } from '@hcengineering/platform' import { type DbAdapter, type DbAdapterHandler, + type RawFindIterator, type StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core' @@ -78,6 +79,13 @@ class StorageBlobAdapter implements DbAdapter { return [] } + rawFind (ctx: MeasureContext, domain: Domain): RawFindIterator { + return { + find: async () => [], + close: async () => {} + } + } + async rawUpdate( domain: Domain, query: DocumentQuery, diff --git a/server/server-pipeline/src/pipeline.ts b/server/server-pipeline/src/pipeline.ts index 8e7cc4e692..b1f3082f95 100644 --- a/server/server-pipeline/src/pipeline.ts +++ b/server/server-pipeline/src/pipeline.ts @@ -32,6 +32,7 @@ import { NotificationsMiddleware, PrivateMiddleware, QueryJoinMiddleware, + QueueMiddleware, SpacePermissionsMiddleware, SpaceSecurityMiddleware, TriggersMiddleware, @@ -50,10 +51,12 @@ import { type PipelineContext, type PipelineFactory, type StorageAdapter, - type WorkspaceDestroyAdapter + type WorkspaceDestroyAdapter, + type PlatformQueue } from '@hcengineering/server-core' import { generateToken } from '@hcengineering/server-token' import { createStorageDataAdapter } from './blobStorage' + /** * @public */ @@ -99,6 +102,8 @@ export function createServerPipeline ( externalStorage: StorageAdapter + queue?: PlatformQueue + extraLogging?: boolean // If passed, will log every request/etc. pipelineContextVars?: Record }, @@ -131,6 +136,7 @@ export function createServerPipeline ( LiveQueryMiddleware.create, DomainFindMiddleware.create, DomainTxMiddleware.create, + ...(opt.queue !== undefined ? [QueueMiddleware.create(opt.queue)] : []), DBAdapterInitMiddleware.create, ModelMiddleware.create(model), DBAdapterMiddleware.create(conf), // Configure DB adapters @@ -205,6 +211,7 @@ export async function getServerPipeline ( wsUrl: WorkspaceIds, storageAdapter: StorageAdapter, opt?: { + queue?: PlatformQueue disableTriggers?: boolean } ): Promise { @@ -212,7 +219,8 @@ export async function getServerPipeline ( externalStorage: storageAdapter, usePassedCtx: true, disableTriggers: opt?.disableTriggers ?? false, - adapterSecurity: isAdapterSecurity(dbUrl) + adapterSecurity: isAdapterSecurity(dbUrl), + queue: opt?.queue }) return await pipelineFactory(ctx, wsUrl, true, () => {}, null) diff --git a/server/server-pipeline/src/serverPlugins.ts b/server/server-pipeline/src/serverPlugins.ts index c930810011..36298ce9d5 100644 --- a/server/server-pipeline/src/serverPlugins.ts +++ b/server/server-pipeline/src/serverPlugins.ts @@ -28,7 +28,6 @@ import { serverTrackerId } from '@hcengineering/server-tracker' import { serverTrainingId } from '@hcengineering/server-training' import { serverViewId } from '@hcengineering/server-view' import { serverAiBotId } from '@hcengineering/server-ai-bot' -import { serverFulltextId } from '@hcengineering/server-fulltext' import { serverProcessId } from '@hcengineering/server-process' export function registerServerPlugins (): void { @@ -61,6 +60,5 @@ export function registerServerPlugins (): void { addLocation(serverTrainingId, () => import('@hcengineering/server-training-resources')) addLocation(serverGithubId, () => import('@hcengineering/server-github-resources')) addLocation(serverAiBotId, () => import('@hcengineering/server-ai-bot-resources')) - addLocation(serverFulltextId, () => import('@hcengineering/server-fulltext-resources')) addLocation(serverProcessId, () => import('@hcengineering/server-process-resources')) } diff --git a/server/server/src/sessionManager.ts b/server/server/src/sessionManager.ts index 3dfed22971..9796ea9562 100644 --- a/server/server/src/sessionManager.ts +++ b/server/server/src/sessionManager.ts @@ -13,38 +13,38 @@ // limitations under the License. // +import { getClient as getAccountClient, isWorkspaceLoginInfo } from '@hcengineering/account-client' import { Analytics } from '@hcengineering/analytics' import core, { AccountRole, cutObjectArray, + Data, generateId, isArchivingMode, isMigrationMode, isRestoringMode, isWorkspaceCreating, - TxFactory, - WorkspaceEvent, + pickPrimarySocialId, + platformNow, + platformNowDiff, systemAccountUuid, + TxFactory, + Version, versionToString, withContext, + WorkspaceEvent, + type Account, + type AccountUuid, type Branding, type BrandingMap, type MeasureContext, + type PersonId, type Tx, type TxWorkspaceEvent, - type WorkspaceUuid, - type WorkspaceInfoWithStatus, - Account, - pickPrimarySocialId, - type PersonId, type WorkspaceDataId, - Data, - Version, - platformNow, - platformNowDiff, - AccountUuid + type WorkspaceInfoWithStatus, + type WorkspaceUuid } from '@hcengineering/core' -import { getClient as getAccountClient, isWorkspaceLoginInfo } from '@hcengineering/account-client' import { unknownError, type Status } from '@hcengineering/platform' import { type HelloRequest, type HelloResponse, type Request, type Response } from '@hcengineering/rpc' import { @@ -52,14 +52,21 @@ import { pingConst, Pipeline, PipelineFactory, + QueueTopic, + QueueUserMessage, ServerFactory, SessionManager, StorageAdapter, - type SessionFactory, + userEvents, + workspaceEvents, type AddSessionResponse, type ClientSessionCtx, type ConnectionSocket, + type PlatformQueue, + type PlatformQueueProducer, + type QueueWorkspaceMessage, type Session, + type SessionFactory, type Workspace } from '@hcengineering/server-core' import { generateToken, type Token } from '@hcengineering/server-token' @@ -108,7 +115,8 @@ export class TSessionManager implements SessionManager { oldClientErrors: number = 0 clientErrors: number = 0 lastClients: string[] = [] - + workspaceProducer: PlatformQueueProducer + usersProducer: PlatformQueueProducer constructor ( readonly ctx: MeasureContext, readonly sessionFactory: SessionFactory, @@ -122,13 +130,16 @@ export class TSessionManager implements SessionManager { | undefined, readonly accountsUrl: string, readonly enableCompression: boolean, - readonly doHandleTick: boolean = true + readonly doHandleTick: boolean = true, + readonly queue: PlatformQueue ) { if (this.doHandleTick) { this.checkInterval = setInterval(() => { this.handleTick() }, 1000 / ticksPerSecond) } + this.workspaceProducer = this.queue.createProducer(ctx.newChild('queue', {}), QueueTopic.Workspace) + this.usersProducer = this.queue.createProducer(ctx.newChild('queue', {}), QueueTopic.Users) } scheduleMaintenance (timeMinutes: number): void { @@ -357,6 +368,12 @@ export class TSessionManager implements SessionManager { } } + countUserSessions (workspace: Workspace, accountUuid: AccountUuid): number { + return Array.from(workspace.sessions.values()) + .filter((it) => it.session.getUser() === accountUuid) + .reduce((acc) => acc + 1, 0) + } + tickCounter = 0 @withContext('📲 add-session') @@ -479,6 +496,7 @@ export class TSessionManager implements SessionManager { workspaceInfo.dataId, branding ) + await this.workspaceProducer.send(workspaceInfo.uuid, [workspaceEvents.open()]) } let pipeline: Pipeline @@ -546,13 +564,26 @@ export class TSessionManager implements SessionManager { this.tickCounter++ workspace.sessions.set(session.sessionId, { session, socket: ws, tickHash: this.tickCounter % ticksPerSecond }) + const accountUuid = account.uuid + await this.usersProducer.send(workspaceInfo.uuid, [ + userEvents.login({ + user: accountUuid, + sessions: this.countUserSessions(workspace, accountUuid), + socialIds: account.socialIds + }) + ]) + // Mark workspace as init completed and we had at least one client. if (!workspace.workspaceInitCompleted) { workspace.workspaceInitCompleted = true } if (this.timeMinutes > 0) { - void ws.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression) + void ws + .send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression) + .catch((err) => { + ctx.error('failed to send maintenance warning', err) + }) } return { session, context: workspace.context, workspaceId: workspaceInfo.uuid } } @@ -809,8 +840,19 @@ export class TSessionManager implements SessionManager { totalUsers: this.sessions.size }) this.sessions.delete(ws.id) + if (workspace !== undefined) { workspace.sessions.delete(sessionRef.session.sessionId) + + const userUuid = sessionRef.session.getUser() + await this.usersProducer.send(workspace.workspaceUuid, [ + userEvents.logout({ + user: userUuid, + sessions: this.countUserSessions(workspace, userUuid), + socialIds: sessionRef.session.getUserSocialIds() + }) + ]) + const pipeline = workspace.pipeline instanceof Promise ? await workspace.pipeline : workspace.pipeline if (this.doHandleTick) { @@ -967,6 +1009,8 @@ export class TSessionManager implements SessionManager { if (LOGGING_ENABLED) { this.ctx.warn('Closed workspace', logParams) } + + await this.workspaceProducer.send(workspace.workspaceUuid, [workspaceEvents.down()]) } } catch (err: any) { Analytics.handleError(err) @@ -1266,7 +1310,8 @@ export function createSessionManager ( | undefined, accountsUrl: string, enableCompression: boolean, - doHandleTick: boolean = true + doHandleTick: boolean = true, + queue: PlatformQueue ): SessionManager { return new TSessionManager( ctx, @@ -1276,7 +1321,8 @@ export function createSessionManager ( profiling, accountsUrl, enableCompression, - doHandleTick + doHandleTick, + queue ) } @@ -1298,6 +1344,7 @@ export function startSessionManager ( start: () => void stop: () => Promise } + queue: PlatformQueue } & Partial ): { shutdown: () => Promise, sessionManager: SessionManager } { const sessions = createSessionManager( @@ -1310,7 +1357,9 @@ export function startSessionManager ( }, opt.profiling, opt.accountsUrl, - opt.enableCompression ?? false + opt.enableCompression ?? false, + true, + opt.queue ) return { shutdown: opt.serverFactory( diff --git a/server/workspace-service/src/index.ts b/server/workspace-service/src/index.ts index d54309e2da..b679d68eea 100644 --- a/server/workspace-service/src/index.ts +++ b/server/workspace-service/src/index.ts @@ -25,6 +25,7 @@ import { import { type MigrateOperation } from '@hcengineering/model' import { setMetadata } from '@hcengineering/platform' import serverClientPlugin from '@hcengineering/server-client' +import { QueueTopic, type PlatformQueue, type QueueWorkspaceMessage } from '@hcengineering/server-core' import serverNotification from '@hcengineering/server-notification' import { createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import serverToken from '@hcengineering/server-token' @@ -37,6 +38,7 @@ export * from './ws-operations' */ export function serveWorkspaceAccount ( measureCtx: MeasureContext, + queue: PlatformQueue, version: Data, txes: Tx[], migrateOperations: [string, MigrateOperation][], @@ -122,6 +124,7 @@ export function serveWorkspaceAccount ( let canceled = false const worker = new WorkspaceWorker( + queue.createProducer(measureCtx, QueueTopic.Workspace), version, txes, migrateOperations, diff --git a/server/workspace-service/src/service.ts b/server/workspace-service/src/service.ts index 43d4014550..d7e56138db 100644 --- a/server/workspace-service/src/service.ts +++ b/server/workspace-service/src/service.ts @@ -14,13 +14,13 @@ // import { getClient as getAccountClient } from '@hcengineering/account-client' import { + DOMAIN_BLOB, getBranding, isArchivingMode, isMigrationMode, isRestoringMode, systemAccountUuid, type BrandingMap, - DOMAIN_BLOB, type Data, type MeasureContext, type Tx, @@ -53,7 +53,13 @@ import { shutdownPostgres } from '@hcengineering/postgres' import { doBackupWorkspace, doRestoreWorkspace } from '@hcengineering/server-backup' -import type { PipelineFactory, StorageAdapter } from '@hcengineering/server-core' +import { + workspaceEvents, + type PipelineFactory, + type PlatformQueueProducer, + type QueueWorkspaceMessage, + type StorageAdapter +} from '@hcengineering/server-core' import { createBackupPipeline, getConfig, @@ -101,6 +107,7 @@ export class WorkspaceWorker { resolveBusy: (() => void) | null = null constructor ( + readonly workspaceQueue: PlatformQueueProducer, readonly version: Data, readonly txes: Tx[], readonly migrationOperation: [string, MigrateOperation][], @@ -270,6 +277,8 @@ export class WorkspaceWorker { region: this.region, time: Date.now() - t }) + + await this.workspaceQueue.send(ws.uuid, [workspaceEvents.created()]) } catch (err: any) { await opt.errorHandler(ws, err) @@ -285,6 +294,7 @@ export class WorkspaceWorker { region: this.region, time: Date.now() - t }) + await this.workspaceQueue.send(ws.uuid, [workspaceEvents.createFailed()]) } finally { if (!opt.console) { ;(logger as FileModelLogger).close() @@ -365,6 +375,7 @@ export class WorkspaceWorker { region: this.region, time: Date.now() - t }) + await this.workspaceQueue.send(ws.uuid, [workspaceEvents.upgraded()]) } catch (err: any) { await opt.errorHandler(ws, err) @@ -381,6 +392,7 @@ export class WorkspaceWorker { region: this.region, time: Date.now() - t }) + await this.workspaceQueue.send(ws.uuid, [workspaceEvents.upgradeFailed()]) } finally { if (!opt.console) { ;(logger as FileModelLogger).close() @@ -396,32 +408,7 @@ export class WorkspaceWorker { const adapter = getWorkspaceDestroyAdapter(dbUrl) await adapter.deleteWorkspace(ctx, sharedPipelineContextVars, workspace.uuid, workspace.dataId) - await this.doReindexFulltext(ctx, workspace, cleanIndexes) - } - - private async doReindexFulltext ( - ctx: MeasureContext, - workspace: WorkspaceInfoWithStatus, - cleanIndexes: boolean - ): Promise { - if (this.fulltextUrl !== undefined) { - const token = generateToken(systemAccountUuid, workspace.uuid, { service: 'workspace' }) - - try { - const res = await fetch(this.fulltextUrl + '/api/v1/reindex', { - method: 'PUT', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ token, onlyDrop: cleanIndexes }) - }) - if (!res.ok) { - throw new Error(`HTTP Error ${res.status} ${res.statusText}`) - } - } catch (err: any) { - ctx.error('failed to reset index', { err }) - } - } + await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.clearIndex()]) } async sendTransactorMaitenance (token: string, ws: WorkspaceUuid): Promise { @@ -490,6 +477,7 @@ export class WorkspaceWorker { return } await sendEvent('archiving-clean-done', 100) + await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.archived()]) break } case 'pending-deletion': @@ -504,6 +492,7 @@ export class WorkspaceWorker { return } await sendEvent('delete-done', 100) + await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.deleted()]) break } @@ -537,8 +526,8 @@ export class WorkspaceWorker { await sendEvent('restore-started', 0) if (await this.doRestore(ctx, workspace, opt)) { // We should reindex fulltext - await this.doReindexFulltext(ctx, workspace, false) await sendEvent('restore-done', 100) + await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.restored()]) } break default: diff --git a/server/ws/src/__tests__/server.test.ts b/server/ws/src/__tests__/server.test.ts index 9b19433dab..6252892351 100644 --- a/server/ws/src/__tests__/server.test.ts +++ b/server/ws/src/__tests__/server.test.ts @@ -41,7 +41,7 @@ import { type WorkspaceUuid } from '@hcengineering/core' import { ClientSession, startSessionManager } from '@hcengineering/server' -import { createDummyStorageAdapter } from '@hcengineering/server-core' +import { createDummyQueue, createDummyStorageAdapter } from '@hcengineering/server-core' import { startHttpServer } from '../server_http' import { genMinModel } from './minmodel' @@ -101,7 +101,8 @@ describe('server', () => { brandingMap: {}, serverFactory: startHttpServer, accountsUrl: '', - externalStorage: createDummyStorageAdapter() + externalStorage: createDummyStorageAdapter(), + queue: createDummyQueue() }) function connect (): WebSocket { @@ -212,7 +213,8 @@ describe('server', () => { brandingMap: {}, serverFactory: startHttpServer, accountsUrl: '', - externalStorage: createDummyStorageAdapter() + externalStorage: createDummyStorageAdapter(), + queue: createDummyQueue() }) async function findClose (token: string, timeoutPromise: Promise, code: number): Promise { diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index f292cce060..543d33425b 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -193,7 +193,7 @@ export function startHttpServer ( app.get('/api/v1/profiling', (req, res) => { try { - const token = req.query.token as string + const token = (req.query.token as string) ?? (req.headers.authorization ?? '').split(' ')[1] decodeToken(token) const jsonData = { profiling @@ -407,7 +407,7 @@ export function startHttpServer ( app.put('/api/v1/broadcast', (req, res) => { try { - const token = req.query.token as string + const token = (req.query.token as string) ?? (req.headers.authorization ?? '').split(' ')[1] decodeToken(token) const ws = sessions.workspaces.get(req.query.workspace as WorkspaceUuid) if (ws !== undefined) { diff --git a/services/ai-bot/pod-ai-bot/package.json b/services/ai-bot/pod-ai-bot/package.json index 82ff8389ec..06cd8586d3 100644 --- a/services/ai-bot/pod-ai-bot/package.json +++ b/services/ai-bot/pod-ai-bot/package.json @@ -89,6 +89,7 @@ "mongodb": "^6.12.0", "openai": "^4.56.0", "uuid": "^8.3.2", - "ws": "^8.18.0" + "ws": "^8.18.0", + "@hcengineering/analytics-service": "^0.6.0" } } diff --git a/services/ai-bot/pod-ai-bot/src/start.ts b/services/ai-bot/pod-ai-bot/src/start.ts index dce724bbf9..64dddd1629 100644 --- a/services/ai-bot/pod-ai-bot/src/start.ts +++ b/services/ai-bot/pod-ai-bot/src/start.ts @@ -14,17 +14,19 @@ // import { setMetadata } from '@hcengineering/platform' import serverClient, { withRetry } from '@hcengineering/server-client' -import serverToken, { generateToken } from '@hcengineering/server-token' import { initStatisticsContext } from '@hcengineering/server-core' +import serverToken, { generateToken } from '@hcengineering/server-token' -import config from './config' -import { getAccountUuid } from './utils/account' -import { registerLoaders } from './loaders' -import { getDbStorage } from './storage' -import { AIControl } from './controller' -import { createServer, listen } from './server/server' -import type { SocialId } from '@hcengineering/core' import { getClient as getAccountClient } from '@hcengineering/account-client' +import { SplitLogger } from '@hcengineering/analytics-service' +import { MeasureMetricsContext, newMetrics, type SocialId } from '@hcengineering/core' +import { join } from 'path' +import config from './config' +import { AIControl } from './controller' +import { registerLoaders } from './loaders' +import { createServer, listen } from './server/server' +import { getDbStorage } from './storage' +import { getAccountUuid } from './utils/account' export const start = async (): Promise => { setMetadata(serverToken.metadata.Secret, config.ServerSecret) @@ -33,7 +35,19 @@ export const start = async (): Promise => { registerLoaders() - const ctx = initStatisticsContext('ai-bot-service', {}) + const ctx = initStatisticsContext('ai-bot-service', { + factory: () => + new MeasureMetricsContext( + 'ai-bot-service', + {}, + {}, + newMetrics(), + new SplitLogger('ai-bot-service', { + root: join(process.cwd(), 'logs'), + enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true' + }) + ) + }) ctx.info('AI Bot Service started', { firstName: config.FirstName, lastName: config.LastName }) const personUuid = await withRetry( diff --git a/services/calendar/pod-calendar/src/workspaceClient.ts b/services/calendar/pod-calendar/src/workspaceClient.ts index d4578e1fbf..998d32239b 100644 --- a/services/calendar/pod-calendar/src/workspaceClient.ts +++ b/services/calendar/pod-calendar/src/workspaceClient.ts @@ -359,7 +359,7 @@ export class WorkspaceClient { }) const extracted = txes.filter((p) => p._id !== tx._id) const ev = TxProcessor.buildDoc2Doc(extracted) - if (ev !== undefined) { + if (ev != null) { const oldClient = await this.getCalendarClientByCalendar(ev.calendar as Ref) if (oldClient !== undefined) { const oldCalendar = this.calendars.byId.get(ev.calendar as Ref) @@ -414,7 +414,7 @@ export class WorkspaceClient { objectId: tx.objectId }) const ev = TxProcessor.buildDoc2Doc(txes) - if (ev === undefined) return + if (ev == null) return if (ev.access !== 'owner' && ev.access !== 'writer') return const client = await this.getCalendarClientByCalendar(ev?.calendar as Ref) if (client === undefined) { diff --git a/services/github/pod-github/src/sync/pullrequests.ts b/services/github/pod-github/src/sync/pullrequests.ts index 3c41450e6b..2391429894 100644 --- a/services/github/pod-github/src/sync/pullrequests.ts +++ b/services/github/pod-github/src/sync/pullrequests.ts @@ -701,7 +701,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS for (const [, txes] of groupedByTodo) { const todo = TxProcessor.buildDoc2Doc(txes) - if (todo !== undefined && h.hasMixin(todo, github.mixin.GithubTodo)) { + if (todo != null && h.hasMixin(todo, github.mixin.GithubTodo)) { removedTodos.push(h.as(todo, github.mixin.GithubTodo)) } } diff --git a/services/gmail/pod-gmail/package.json b/services/gmail/pod-gmail/package.json index 57ddd94a48..ac7f8a8b6c 100644 --- a/services/gmail/pod-gmail/package.json +++ b/services/gmail/pod-gmail/package.json @@ -76,6 +76,7 @@ "gaxios": "^5.0.1", "jwt-simple": "^0.5.6", "mongodb": "^6.12.0", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "@hcengineering/analytics-service": "^0.6.0" } } diff --git a/services/gmail/pod-gmail/src/main.ts b/services/gmail/pod-gmail/src/main.ts index cb7a4178da..aae9f66128 100644 --- a/services/gmail/pod-gmail/src/main.ts +++ b/services/gmail/pod-gmail/src/main.ts @@ -15,12 +15,15 @@ // limitations under the License. // +import { SplitLogger } from '@hcengineering/analytics-service' +import { MeasureMetricsContext, newMetrics } from '@hcengineering/core' import { setMetadata } from '@hcengineering/platform' import serverClient from '@hcengineering/server-client' import { initStatisticsContext, type StorageConfiguration } from '@hcengineering/server-core' import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' -import serverToken, { decodeToken } from '@hcengineering/server-token' +import serverToken from '@hcengineering/server-token' import { type IncomingHttpHeaders } from 'http' +import { join } from 'path' import { decode64 } from './base64' import config from './config' import { GmailController } from './gmailController' @@ -37,7 +40,19 @@ const extractToken = (header: IncomingHttpHeaders): any => { } export const main = async (): Promise => { - const ctx = initStatisticsContext('gmail', {}) + const ctx = initStatisticsContext('gmail', { + factory: () => + new MeasureMetricsContext( + 'gmail', + {}, + {}, + newMetrics(), + new SplitLogger('gmail', { + root: join(process.cwd(), 'logs'), + enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true' + }) + ) + }) setMetadata(serverClient.metadata.Endpoint, config.AccountsURL) setMetadata(serverClient.metadata.UserAgent, config.ServiceID) diff --git a/services/love/package.json b/services/love/package.json index 46b1ceb25b..e60a60bea7 100644 --- a/services/love/package.json +++ b/services/love/package.json @@ -74,6 +74,7 @@ "dotenv": "~16.0.0", "cors": "^2.8.5", "express": "^4.21.2", - "ws": "^8.18.0" + "ws": "^8.18.0", + "@hcengineering/analytics-service": "^0.6.0" } } diff --git a/services/love/src/main.ts b/services/love/src/main.ts index ff350b2d41..16df0c779f 100644 --- a/services/love/src/main.ts +++ b/services/love/src/main.ts @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -import { MeasureContext, Ref, WorkspaceIds } from '@hcengineering/core' +import { MeasureContext, MeasureMetricsContext, newMetrics, Ref, WorkspaceIds } from '@hcengineering/core' import { setMetadata } from '@hcengineering/platform' import serverClient from '@hcengineering/server-client' import { initStatisticsContext, StorageConfig, StorageConfiguration } from '@hcengineering/server-core' @@ -40,6 +40,8 @@ import config from './config' import { saveLiveKitEgressBilling, saveLiveKitSessionBilling } from './billing' import { getS3UploadParams, saveFile } from './storage' import { WorkspaceClient } from './workspaceClient' +import { join } from 'path' +import { SplitLogger } from '@hcengineering/analytics-service' const extractToken = (header: IncomingHttpHeaders): any => { try { @@ -62,7 +64,19 @@ export const main = async (): Promise => { const s3StorageConfigs: StorageConfiguration | undefined = config.S3StorageConfig !== undefined ? storageConfigFromEnv(config.S3StorageConfig) : undefined - const ctx = initStatisticsContext('love', {}) + const ctx = initStatisticsContext('love', { + factory: () => + new MeasureMetricsContext( + 'love', + {}, + {}, + newMetrics(), + new SplitLogger('love', { + root: join(process.cwd(), 'logs'), + enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true' + }) + ) + }) const storageConfig = storageConfigs.storages.findLast((p) => p.name === config.StorageProviderName) const s3storageConfig = s3StorageConfigs?.storages.findLast((p) => p.kind === 's3') diff --git a/services/print/pod-print/package.json b/services/print/pod-print/package.json index 0ff52598eb..636a5cf9e9 100644 --- a/services/print/pod-print/package.json +++ b/services/print/pod-print/package.json @@ -65,6 +65,7 @@ "express": "^4.21.2", "puppeteer": "^22.6.1", "mammoth": "^1.9.0", - "ws": "^8.18.0" + "ws": "^8.18.0", + "@hcengineering/analytics-service": "^0.6.0" } } diff --git a/services/print/pod-print/src/server.ts b/services/print/pod-print/src/server.ts index 4675d7ddda..214a62852c 100644 --- a/services/print/pod-print/src/server.ts +++ b/services/print/pod-print/src/server.ts @@ -14,7 +14,7 @@ // limitations under the License. // -import { generateId, type WorkspaceIds } from '@hcengineering/core' +import { generateId, MeasureMetricsContext, newMetrics, type WorkspaceIds } from '@hcengineering/core' import { StorageConfiguration, initStatisticsContext } from '@hcengineering/server-core' import { buildStorageFromConfig } from '@hcengineering/server-storage' import { getClient as getAccountClientRaw, AccountClient, isWorkspaceLoginInfo } from '@hcengineering/account-client' @@ -26,6 +26,8 @@ import { convertToHtml } from './convert' import { ApiError } from './error' import { PrintOptions, print, validKinds } from './print' import config from './config' +import { join } from 'path' +import { SplitLogger } from '@hcengineering/analytics-service' function getAccountClient (token: string): AccountClient { return getAccountClientRaw(config.AccountsUrl, token) @@ -129,7 +131,19 @@ export function createServer ( allowedHostnames: string[] ): { app: Express, close: () => void } { const storageAdapter = buildStorageFromConfig(storageConfig) - const measureCtx = initStatisticsContext('print', {}) + const measureCtx = initStatisticsContext('print', { + factory: () => + new MeasureMetricsContext( + 'print', + {}, + {}, + newMetrics(), + new SplitLogger('print', { + root: join(process.cwd(), 'logs'), + enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true' + }) + ) + }) const whitelistedHostnames = allowedHostnames.length > 0 ? new Set(allowedHostnames) : null const app = express() diff --git a/services/rekoni/package.json b/services/rekoni/package.json index f7e16c452e..569041a560 100644 --- a/services/rekoni/package.json +++ b/services/rekoni/package.json @@ -88,7 +88,8 @@ "morgan": "^1.10.0", "@hcengineering/server-core": "^0.6.1", "@hcengineering/server-token": "^0.6.11", - "@hcengineering/platform": "^0.6.11" + "@hcengineering/platform": "^0.6.11", + "@hcengineering/analytics-service": "^0.6.0" }, "description": "Document recognition service" } diff --git a/services/rekoni/src/server.ts b/services/rekoni/src/server.ts index 544e4fbaf5..a3796ec56b 100644 --- a/services/rekoni/src/server.ts +++ b/services/rekoni/src/server.ts @@ -31,6 +31,9 @@ import { extractDocument } from './process' import { type ReconiDocument } from './types' import serverToken from '@hcengineering/server-token' import { setMetadata } from '@hcengineering/platform' +import { MeasureMetricsContext, newMetrics } from '@hcengineering/core' +import { join } from 'path' +import { SplitLogger } from '@hcengineering/analytics-service' // eslint-disable-next-line @typescript-eslint/no-var-requires const extractToken = (header: IncomingHttpHeaders): any => { @@ -45,7 +48,19 @@ export const startServer = async (): Promise => { const app = express() setMetadata(serverToken.metadata.Secret, process.env.SECRET) - const ctx = initStatisticsContext('rekoni', {}) + const ctx = initStatisticsContext('rekoni', { + factory: () => + new MeasureMetricsContext( + 'rekoni', + {}, + {}, + newMetrics(), + new SplitLogger('rekoni', { + root: join(process.cwd(), 'logs'), + enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true' + }) + ) + }) class MyStream { write (text: string): void { diff --git a/services/sign/pod-sign/package.json b/services/sign/pod-sign/package.json index 26657a019d..2b3a6b7238 100644 --- a/services/sign/pod-sign/package.json +++ b/services/sign/pod-sign/package.json @@ -72,6 +72,7 @@ "@signpdf/utils": "^3.2.4", "node-forge": "^1.3.1", "pdf-lib": "^1.12.0", - "mongodb": "^6.12.0" + "mongodb": "^6.12.0", + "@hcengineering/analytics-service": "^0.6.0" } } diff --git a/services/sign/pod-sign/src/server.ts b/services/sign/pod-sign/src/server.ts index f753b63394..759072a045 100644 --- a/services/sign/pod-sign/src/server.ts +++ b/services/sign/pod-sign/src/server.ts @@ -14,7 +14,7 @@ // limitations under the License. // -import { generateId, type WorkspaceIds } from '@hcengineering/core' +import { generateId, MeasureMetricsContext, newMetrics, type WorkspaceIds } from '@hcengineering/core' import { initStatisticsContext, StorageConfiguration } from '@hcengineering/server-core' import { buildStorageFromConfig } from '@hcengineering/server-storage' import { getClient as getAccountClientRaw, AccountClient, isWorkspaceLoginInfo } from '@hcengineering/account-client' @@ -27,6 +27,8 @@ import config from './config' import { ApiError } from './error' import { signPDF } from './sign' import { extractToken } from './token' +import { join } from 'path' +import { SplitLogger } from '@hcengineering/analytics-service' function getAccountClient (token: string): AccountClient { return getAccountClientRaw(config.AccountsUrl, token) @@ -73,7 +75,19 @@ const wrapRequest = export function createServer (storageConfig: StorageConfiguration, brandings: BrandingMap): Express { const storageAdapter = buildStorageFromConfig(storageConfig) - const measureCtx = initStatisticsContext('sign', {}) + const measureCtx = initStatisticsContext('sign', { + factory: () => + new MeasureMetricsContext( + 'sign', + {}, + {}, + newMetrics(), + new SplitLogger('sign', { + root: join(process.cwd(), 'logs'), + enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true' + }) + ) + }) const app = express() app.use(cors()) diff --git a/services/telegram/pod-telegram/package.json b/services/telegram/pod-telegram/package.json index 8a302aff10..e8ddbcbda7 100644 --- a/services/telegram/pod-telegram/package.json +++ b/services/telegram/pod-telegram/package.json @@ -79,6 +79,7 @@ "mongodb": "^6.12.0", "telegram": "2.22.2", "uuid": "^8.3.2", - "ws": "^8.18.0" + "ws": "^8.18.0", + "@hcengineering/analytics-service": "^0.6.0" } } diff --git a/services/telegram/pod-telegram/src/main.ts b/services/telegram/pod-telegram/src/main.ts index 36d4d56e2f..056cb85a43 100644 --- a/services/telegram/pod-telegram/src/main.ts +++ b/services/telegram/pod-telegram/src/main.ts @@ -1,16 +1,17 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { IncomingHttpHeaders } from 'http' -import { SocialIdType } from '@hcengineering/core' +import { MeasureMetricsContext, newMetrics } from '@hcengineering/core' import { setMetadata } from '@hcengineering/platform' import serverClient from '@hcengineering/server-client' import { initStatisticsContext, type StorageConfiguration } from '@hcengineering/server-core' import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import serverToken, { decodeToken, type Token } from '@hcengineering/server-token' +import { IncomingHttpHeaders } from 'http' +import { SplitLogger } from '@hcengineering/analytics-service' +import { join } from 'path' +import config from './config' import { PlatformWorker } from './platform' import { createServer, Handler, listen } from './server' -import { telegram } from './telegram' -import config from './config' const extractTokenRaw = (headers: IncomingHttpHeaders): string | undefined => { return headers.authorization?.slice(7) @@ -25,7 +26,19 @@ const extractToken = (headers: IncomingHttpHeaders): Token | undefined => { } export const main = async (): Promise => { - const ctx = initStatisticsContext('telegram', {}) + const ctx = initStatisticsContext('telegram', { + factory: () => + new MeasureMetricsContext( + 'telegram', + {}, + {}, + newMetrics(), + new SplitLogger('telegram', { + root: join(process.cwd(), 'logs'), + enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true' + }) + ) + }) setMetadata(serverClient.metadata.Endpoint, config.AccountsURL) setMetadata(serverClient.metadata.UserAgent, config.ServiceID) diff --git a/tests/.env b/tests/.env index a3b3081a02..36efe35dcd 100644 --- a/tests/.env +++ b/tests/.env @@ -1,3 +1,8 @@ STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" -BACKUP_BUCKET_NAME=dev-backups \ No newline at end of file +BACKUP_BUCKET_NAME=dev-backups + +DB_URL=mongodb://mongodb:27018 +MONGO_URL=mongodb://mongodb:27018 +DB_PG_URL=postgresql://root@cockroach:26257/defaultdb?sslmode=disable +QUEUE_CONFIG='redpanda:9093;-staging' diff --git a/tests/docker-compose.override.yaml b/tests/docker-compose.override.yaml index 2b04ded180..c16574744b 100644 --- a/tests/docker-compose.override.yaml +++ b/tests/docker-compose.override.yaml @@ -1,13 +1,13 @@ services: account: environment: - - DB_URL=postgresql://root@cockroach:26257/defaultdb?sslmode=disable + - DB_URL=${DB_PG_URL} transactor: environment: - - DB_URL=postgresql://root@cockroach:26257/defaultdb?sslmode=disable + - DB_URL=${DB_PG_URL} workspace: environment: - - DB_URL=postgresql://root@cockroach:26257/defaultdb?sslmode=disable + - DB_URL=${DB_PG_URL} fulltext: environment: - - DB_URL=postgresql://root@cockroach:26257/defaultdb?sslmode=disable + - DB_URL=${DB_PG_URL} diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index d98bb64d2b..6682ffdc06 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -1,7 +1,26 @@ services: + stream: + image: 'hardcoreeng/stream' + pull_policy: always + environment: + - STREAM_ENDPOINT_URL=s3://localhost:9002 + - STREAM_INSECURE=true + - STREAM_SERVER_SECRET=secret + - AWS_ACCESS_KEY_ID=minioadmin + - AWS_SECRET_ACCESS_KEY=minioadmin + ports: + - 1081:1080 + restart: unless-stopped mongodb: image: 'mongo:7-jammy' command: mongod --port 27018 + healthcheck: + test: echo "try { db.currentOp().ok } catch (err) { }" | mongosh --port 27018 --quiet + interval: 5s + timeout: 30s + start_period: 0s + start_interval: 1s + retries: 30 environment: - PUID=1000 - PGID=1000 @@ -12,8 +31,54 @@ services: image: cockroachdb/cockroach:latest-v24.2 ports: - '26258:26257' + - '18089:8080' command: start-single-node --insecure restart: unless-stopped + redpanda: + image: docker.redpanda.com/redpandadata/redpanda:v24.3.6 + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9093,external://0.0.0.0:19093 + - --advertise-kafka-addr internal://redpanda:9093,external://localhost:19093 + - --pandaproxy-addr internal://0.0.0.0:8084,external://0.0.0.0:18084 + - --advertise-pandaproxy-addr internal://redpanda:8084,external://localhost:18084 + - --schema-registry-addr internal://0.0.0.0:8083,external://0.0.0.0:18083 + - --rpc-addr redpanda:33145 + - --advertise-rpc-addr redpanda:33145 + - --mode dev-container + - --smp 1 + - --default-log-level=info + ports: + - 18083:18083 + - 18084:18084 + - 19093:19093 + - 19645:9644 + healthcheck: + test: ['CMD', 'rpk', 'cluster', 'info', '-X', 'user=superuser', '-X', 'pass=secretpassword'] + interval: 10s + timeout: 5s + retries: 10 + redpanda_console: + image: docker.redpanda.com/redpandadata/console:v2.8.3 + entrypoint: /bin/sh + command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console' + environment: + CONFIG_FILEPATH: /tmp/config.yml + CONSOLE_CONFIG_FILE: | + kafka: + brokers: ["redpanda:9093"] + schemaRegistry: + enabled: true + urls: ["http://redpanda:8083"] + redpanda: + adminApi: + enabled: true + urls: ["http://redpanda:9644"] + ports: + - 8001:8080 + depends_on: + - redpanda minio: image: 'minio/minio' command: server /data --address ":9000" --console-address ":9001" @@ -51,9 +116,12 @@ services: - ./branding-test.json:/var/cfg/branding.json environment: - ACCOUNT_PORT=3003 + - QUEUE_CONFIG=${QUEUE_CONFIG} - SERVER_SECRET=secret + - ADMIN_EMAILS=admin,${PLATFORM_ADMIN_EMAILS} - WORKSPACE_LIMIT_PER_USER=100 - - DB_URL=mongodb://mongodb:27018 + - DB_URL=${MONGO_URL} + - REGION_INFO=|America - TRANSACTOR_URL=ws://transactor:3334;ws://localhost:3334 - STORAGE_CONFIG=${STORAGE_CONFIG} - MODEL_ENABLED=* @@ -65,14 +133,17 @@ services: - mongodb - cockroach - minio + - redpanda volumes: - ./branding-test.json:/var/cfg/branding.json environment: - SERVER_SECRET=secret - DB_URL=mongodb://mongodb:27018 - TRANSACTOR_URL=ws://transactor:3334;ws://localhost:3334 + - QUEUE_CONFIG=${QUEUE_CONFIG} - STORAGE_CONFIG=${STORAGE_CONFIG} - MODEL_ENABLED=* + - REGION= - ACCOUNTS_URL=http://account:3003 - BRANDING_PATH=/var/cfg/branding.json - STATS_URL=http://stats:4901 @@ -86,6 +157,7 @@ services: - minio - elastic - collaborator + - stats - transactor ports: - 8083:8083 @@ -117,13 +189,17 @@ services: - rekoni - cockroach - account + - stats + - redpanda ports: - 3334:3334 volumes: - ./branding-test.json:/var/cfg/branding.json environment: + - QUEUE_CONFIG=${QUEUE_CONFIG} + - REGION= - SERVER_PORT=3334 - - SERVER_SECRET=secret + - SERVER_SECRET=secret - DB_URL=mongodb://mongodb:27018 - METRICS_CONSOLE=false - METRICS_FILE=metrics.txt @@ -159,7 +235,7 @@ services: - 4007:4004 environment: - STATS_URL=http://stats:4901 - + fulltext: image: hardcoreeng/fulltext restart: unless-stopped @@ -168,6 +244,7 @@ services: - mongodb - cockroach - rekoni + - redpanda ports: - 4710:4710 environment: @@ -175,6 +252,7 @@ services: - SERVER_SECRET=secret - DB_URL=mongodb://mongodb:27018 - FULLTEXT_DB_URL=http://elastic:9200 + - QUEUE_CONFIG=${QUEUE_CONFIG} - ELASTIC_INDEX_NAME=local_storage_index - STORAGE_CONFIG=${STORAGE_CONFIG} - REKONI_URL=http://rekoni:4007 @@ -187,4 +265,4 @@ services: environment: - PORT=4901 - SERVER_SECRET=secret - restart: unless-stopped \ No newline at end of file + restart: unless-stopped diff --git a/tests/prepare-pg.sh b/tests/prepare-pg.sh index cc08244793..319ffc35c2 100755 --- a/tests/prepare-pg.sh +++ b/tests/prepare-pg.sh @@ -21,8 +21,10 @@ fi # Create user record in accounts ./tool-pg.sh create-account user1 -f John -l Appleseed -p 1234 ./tool-pg.sh create-account user2 -f Kainin -l Dirak -p 1234 +./tool-pg.sh create-account admin -f Super -l User -p 1234 # Create workspace record in accounts ./tool-pg.sh create-workspace sanity-ws email:user1 ./restore-pg.sh +rm -rf ./sanity/.auth diff --git a/tests/prepare.sh b/tests/prepare.sh index 1247f40b38..18a1d13cc6 100755 --- a/tests/prepare.sh +++ b/tests/prepare.sh @@ -21,9 +21,11 @@ fi # Create user record in accounts ./tool.sh create-account user1 -f John -l Appleseed -p 1234 ./tool.sh create-account user2 -f Kainin -l Dirak -p 1234 -./tool.sh create-account super -f Super -l User -p 1234 +./tool.sh create-account admin -f Super -l User -p 1234 # Create workspace record in accounts ./tool.sh create-workspace sanity-ws email:user1 ./restore-workspace.sh + +rm -rf ./sanity/.auth diff --git a/tests/sanity/package.json b/tests/sanity/package.json index 7f6be57bc6..725fb25cc4 100644 --- a/tests/sanity/package.json +++ b/tests/sanity/package.json @@ -50,6 +50,7 @@ "dotenv": "~16.0.0", "cross-env": "~7.0.3", "@hcengineering/core": "^0.6.32", - "@hcengineering/client-resources": "^0.6.27" + "@hcengineering/client-resources": "^0.6.27", + "@hcengineering/account": "^0.6.0" } } diff --git a/tests/sanity/tests/API/Api.ts b/tests/sanity/tests/API/Api.ts index ad565987d7..862ec5054b 100644 --- a/tests/sanity/tests/API/Api.ts +++ b/tests/sanity/tests/API/Api.ts @@ -1,5 +1,6 @@ +import type { WorkspaceInfoWithStatus, WorkspaceLoginInfo } from '@hcengineering/account' import { APIRequestContext } from '@playwright/test' -import { PlatformURI, LocalUrl, DevUrl } from '../utils' +import { DevUrl, LocalUrl, PlatformURI } from '../utils' export class ApiEndpoint { private readonly request: APIRequestContext @@ -45,11 +46,7 @@ export class ApiEndpoint { workspaceName: string, username: string, password: string - ): Promise<{ - workspace: string - workspaceId: string - workspaceName: string - }> { + ): Promise { const token = await this.loginAndGetToken(username, password) const url = this.baseUrl const payload = { @@ -58,7 +55,53 @@ export class ApiEndpoint { } const headers = this.getDefaultHeaders(token) const response = await this.request.post(url, { data: payload, headers }) - return (await response.json()).result + + const wsResult: WorkspaceLoginInfo = (await response.json()).result + + await this.waitWorkspaceReady(token, wsResult.workspaceUrl) + + return wsResult + } + + async waitWorkspaceReady (token: string, workspaceUrl: string): Promise { + // We need to wait for workspace to be created before we will continue. + const headers = this.getDefaultHeaders(token) + const url = this.baseUrl + const selectWorkspaceResponse: WorkspaceLoginInfo = ( + await ( + await this.request.post(url, { + data: { + method: 'selectWorkspace', + params: { workspaceUrl } + }, + headers + }) + ).json() + ).result + + const wsToken = selectWorkspaceResponse.token + if (wsToken === undefined) { + throw new Error('Workspace token is undefined') + } + + const headersInfo = this.getDefaultHeaders(wsToken) + while (true) { + const wsInfo: WorkspaceInfoWithStatus = ( + await ( + await this.request.post(url, { + data: { + method: 'getWorkspaceInfo', + params: { updateLastVisit: false } + }, + headers: headersInfo + }) + ).json() + ).result + await new Promise((resolve) => setTimeout(resolve, 1000)) + if (wsInfo.status.mode === 'active') { + break + } + } } async createAccount (email: string, password: string, firstName: string, lastName: string): Promise { diff --git a/tests/sanity/tests/auth/auth.setup.ts b/tests/sanity/tests/auth/auth.setup.ts index 3d1d52a24f..1e91d25693 100644 --- a/tests/sanity/tests/auth/auth.setup.ts +++ b/tests/sanity/tests/auth/auth.setup.ts @@ -4,6 +4,7 @@ import path from 'path' import { PlatformUser, PlatformUserSecond, PlatformWs, PlatformURI, setTestOptions } from '../utils' import { LoginPage } from '../model/login-page' import { SelectWorkspacePage } from '../model/select-workspace-page' +import { existsSync } from 'fs' const authFile = path.join(__dirname, '../../.auth/storage.json') const authFileSecond = path.join(__dirname, '../../.auth/storageSecond.json') @@ -19,16 +20,20 @@ async function authenticate (page: Page, user: string, password: string): Promis }) } -setup('auth user1', async ({ page }) => { - await authenticate(page, PlatformUser, '1234') - await setTestOptions(page) +if (!existsSync(authFile)) { + setup('auth user1', async ({ page }) => { + await authenticate(page, PlatformUser, '1234') + await setTestOptions(page) - await page.context().storageState({ path: authFile }) -}) + await page.context().storageState({ path: authFile }) + }) +} -setup('auth user2', async ({ page }) => { - await authenticate(page, PlatformUserSecond, '1234') - await setTestOptions(page) +if (!existsSync(authFileSecond)) { + setup('auth user2', async ({ page }) => { + await authenticate(page, PlatformUserSecond, '1234') + await setTestOptions(page) - await page.context().storageState({ path: authFileSecond }) -}) + await page.context().storageState({ path: authFileSecond }) + }) +} diff --git a/tests/sanity/tests/chat/chat.spec.ts b/tests/sanity/tests/chat/chat.spec.ts index 14dd7ba0a6..75f8e746e5 100644 --- a/tests/sanity/tests/chat/chat.spec.ts +++ b/tests/sanity/tests/chat/chat.spec.ts @@ -282,7 +282,7 @@ test.describe('Channel tests', () => { }) test('Check if user can copy message', async ({ page }) => { - const baseURL = process.env.PLATFORM_URI ?? 'http://localhost:8083' + const baseURL = process.env.PLATFORM_URI ?? 'http://huly.local:8083' const expectedUrl = `${baseURL}/workbench/${data.workspaceName}/chunter/chunter%3Aspace%3ARandom%7Cchunter%3Aclass%3AChannel?message=` await leftSideMenuPage.clickChunter() await channelPage.clickChannel('random') diff --git a/tests/sanity/tests/indexer.spec.ts b/tests/sanity/tests/indexer.spec.ts index 073054d7bc..a7f229175b 100644 --- a/tests/sanity/tests/indexer.spec.ts +++ b/tests/sanity/tests/indexer.spec.ts @@ -1,4 +1,4 @@ -import { test } from '@playwright/test' +import { expect, test } from '@playwright/test' import { NewDocument } from './model/documents/types' import { LeftSideMenuPage } from './model/left-side-menu-page' import { DocumentsPage } from './model/documents/documents-page' @@ -13,6 +13,7 @@ import { LoginPage } from './model/login-page' import { SignUpPage } from './model/signup-page' import { SelectWorkspacePage } from './model/select-workspace-page' +const retryOptions = { intervals: [1000, 1500, 2500], timeout: 60000 } test.use({ storageState: PlatformSetting }) @@ -173,7 +174,9 @@ test.describe('Fulltext index', () => { await test.step('search by title', async () => { await spotlight.open() await spotlight.fillSearchInput(titleId) - await spotlight.checkSearchResult(newDocument.title, 0) + await expect(async () => { + await spotlight.checkSearchResult(newDocument.title, 0) + }).toPass(retryOptions) await spotlight.close() }) }) diff --git a/tests/tool-local.sh b/tests/tool-local.sh index 699ccd9fd3..d534f4d0ca 100755 --- a/tests/tool-local.sh +++ b/tests/tool-local.sh @@ -9,6 +9,7 @@ export ACCOUNTS_URL=http://localhost:3000 export TRANSACTOR_URL=ws://localhost:3333 export ELASTIC_URL=http://localhost:9200 export SERVER_SECRET=secret +export QUEUE_CONFIG=localhost:19092 # Restore workspace contents in mongo/elastic node ../dev/tool/bundle/bundle.js $@ \ No newline at end of file diff --git a/tests/tool-pg.sh b/tests/tool-pg.sh index bad45b41e5..efb9e1df6b 100755 --- a/tests/tool-pg.sh +++ b/tests/tool-pg.sh @@ -11,5 +11,6 @@ export MONGO_URL=mongodb://localhost:27018 export ELASTIC_URL=http://localhost:9201 export SERVER_SECRET=secret export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable +export QUEUE_CONFIG='localhost:19093;-staging' node ${TOOL_OPTIONS} ../dev/tool/bundle/bundle.js $@ \ No newline at end of file diff --git a/tests/tool.sh b/tests/tool.sh index 0a78fac240..b10f652c84 100755 --- a/tests/tool.sh +++ b/tests/tool.sh @@ -11,5 +11,6 @@ export MONGO_URL=mongodb://localhost:27018 export DB_URL=mongodb://localhost:27018 export ELASTIC_URL=http://localhost:9201 export SERVER_SECRET=secret +export QUEUE_CONFIG='localhost:19093;-staging' node ../dev/tool/bundle/bundle.js $@ \ No newline at end of file diff --git a/ws-tests/.env b/ws-tests/.env index 564ac6bec2..57f3cbfcd8 100644 --- a/ws-tests/.env +++ b/ws-tests/.env @@ -2,4 +2,6 @@ STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" BACKUP_BUCKET_NAME=ws-dev-backups DB_URL=mongodb://huly.local:27018 +MONGO_URL=mongodb://huly.local:27018 DB_EU_URL=postgresql://root@huly.local:26258/defaultdb?sslmode=disable +QUEUE_CONFIG=redpanda:9093 diff --git a/ws-tests/api-tests/src/__tests__/rest.test.ts b/ws-tests/api-tests/src/__tests__/rest.test.ts index 0662090475..b29215113e 100644 --- a/ws-tests/api-tests/src/__tests__/rest.test.ts +++ b/ws-tests/api-tests/src/__tests__/rest.test.ts @@ -43,10 +43,10 @@ describe('rest-api-server', () => { let apiWorkspace2: WorkspaceToken beforeAll(async () => { - const config = await loadServerConfig('http://localhost:8083') + const config = await loadServerConfig('http://huly.local:8083') apiWorkspace1 = await getWorkspaceToken( - 'http://localhost:8083', + 'http://huly.local:8083', { email: 'user1', password: '1234', @@ -56,7 +56,7 @@ describe('rest-api-server', () => { ) apiWorkspace2 = await getWorkspaceToken( - 'http://localhost:8083', + 'http://huly.local:8083', { email: 'user1', password: '1234', diff --git a/ws-tests/docker-compose.yaml b/ws-tests/docker-compose.yaml index 7d34079cc2..32179b1948 100644 --- a/ws-tests/docker-compose.yaml +++ b/ws-tests/docker-compose.yaml @@ -1,9 +1,30 @@ services: + stream: + image: 'hardcoreeng/stream' + pull_policy: always + extra_hosts: + - 'huly.local:host-gateway' + environment: + - STREAM_ENDPOINT_URL=s3://huly.local:9000 + - STREAM_INSECURE=true + - STREAM_SERVER_SECRET=secret + - AWS_ACCESS_KEY_ID=minioadmin + - AWS_SECRET_ACCESS_KEY=minioadmin + ports: + - 1081:1080 + restart: unless-stopped mongodb: image: 'mongo:7-jammy' command: mongod --port 27018 extra_hosts: - 'huly.local:host-gateway' + healthcheck: + test: echo "try { db.currentOp().ok } catch (err) { }" | mongosh --port 27018 --quiet + interval: 5s + timeout: 30s + start_period: 0s + start_interval: 1s + retries: 30 environment: - PUID=1000 - PGID=1000 @@ -16,16 +37,66 @@ services: - 'huly.local:host-gateway' ports: - '26258:26257' + - '18089:8080' command: start-single-node --insecure restart: unless-stopped + redpanda: + image: docker.redpanda.com/redpandadata/redpanda:v24.3.6 + extra_hosts: + - 'huly.local:host-gateway' + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9093,external://0.0.0.0:19093 + - --advertise-kafka-addr internal://redpanda:9093,external://huly.local:19093 + - --pandaproxy-addr internal://0.0.0.0:8084,external://0.0.0.0:18084 + - --advertise-pandaproxy-addr internal://redpanda:8084,external://huly.local:18084 + - --schema-registry-addr internal://0.0.0.0:8083,external://0.0.0.0:18083 + - --rpc-addr redpanda:33145 + - --advertise-rpc-addr redpanda:33145 + - --mode dev-container + - --smp 1 + - --default-log-level=info + ports: + - 18083:18083 + - 18084:18084 + - 19093:19093 + - 19645:9644 + healthcheck: + test: ['CMD', 'rpk', 'cluster', 'info', '-X', 'user=superuser', '-X', 'pass=secretpassword'] + interval: 10s + timeout: 5s + retries: 10 + redpanda_console: + image: docker.redpanda.com/redpandadata/console:v2.8.3 + extra_hosts: + - 'huly.local:host-gateway' + entrypoint: /bin/sh + command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console' + environment: + CONFIG_FILEPATH: /tmp/config.yml + CONSOLE_CONFIG_FILE: | + kafka: + brokers: ["redpanda:9093"] + schemaRegistry: + enabled: true + urls: ["http://redpanda:8083"] + redpanda: + adminApi: + enabled: true + urls: ["http://redpanda:9644"] + ports: + - 8001:8080 + depends_on: + - redpanda minio: image: 'minio/minio' command: server /data --address ":9000" --console-address ":9001" extra_hosts: - 'huly.local:host-gateway' expose: - - 9000 - - 9001 + - 9002 + - 9003 ports: - 9002:9000 - 9003:9001 @@ -45,33 +116,50 @@ services: healthcheck: interval: 20s retries: 10 - test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"' + test: curl -s http://huly.local:9200/_cluster/health | grep -vq '"status":"red"' account: image: hardcoreeng/account pull_policy: never + extra_hosts: + - 'huly.local:host-gateway' links: - mongodb - minio - cockroach - extra_hosts: - - 'huly.local:host-gateway' ports: - 3003:3003 volumes: - ./branding-test.json:/var/cfg/branding.json environment: - - REGION_INFO=|America;europe| # Europe without name will not be available for creation of new workspaces. - # - REGION_INFO=|America;europe|Europe - - ADMIN_EMAILS=admin - - WORKSPACE_LIMIT_PER_USER=100 - ACCOUNT_PORT=3003 + - QUEUE_CONFIG=${QUEUE_CONFIG} - SERVER_SECRET=secret - - DB_URL=mongodb://huly.local:27018 - - TRANSACTOR_URL=ws://huly.local:3334;ws://huly.local:3334,ws://huly.local:3335;ws://huly.local:3335;europe, - - STORAGE_CONFIG=${STORAGE_CONFIG} - - MODEL_ENABLED=* - - BRANDING_PATH=/var/cfg/branding.json + - ADMIN_EMAILS=admin,${PLATFORM_ADMIN_EMAILS} - STATS_URL=http://huly.local:4901 + - WORKSPACE_LIMIT_PER_USER=100 + - DB_URL=${MONGO_URL} + # - REGION_INFO=|America;europe| # Europe without name will not be available for creation of new workspaces. + - REGION_INFO=|America;europe|Europe + - TRANSACTOR_URL=ws://huly.local:3334;ws://huly.local:3334,ws://huly.local:3335;ws://huly.local:3335;europe + - STORAGE_CONFIG=${STORAGE_CONFIG} + - FRONT_URL=http://huly.local:8083 + - MODEL_ENABLED=* + # - LAST_NAME_FIRST=true + # - WS_LIVENESS_DAYS=1 + - ACCOUNTS_URL=http://huly.local:3003 + - BRANDING_PATH=/var/cfg/branding.json + # - DISABLE_SIGNUP=true + restart: unless-stopped + stats: + image: hardcoreeng/stats + extra_hosts: + - 'huly.local:host-gateway' + ports: + - 4901:4901 + environment: + - PORT=4901 + - SERVER_SECRET=secret + restart: unless-stopped workspace: image: hardcoreeng/workspace extra_hosts: @@ -79,18 +167,20 @@ services: links: - mongodb - minio + - stats volumes: - ./branding-test.json:/var/cfg/branding.json environment: - WS_OPERATION=all+backup - SERVER_SECRET=secret + - QUEUE_CONFIG=${QUEUE_CONFIG} - DB_URL=${DB_URL} + - STATS_URL=http://huly.local:4901 - STORAGE_CONFIG=${STORAGE_CONFIG} - MODEL_ENABLED=* - REGION= - ACCOUNTS_URL=http://huly.local:3003 - BRANDING_PATH=/var/cfg/branding.json - - STATS_URL=http://huly.local:4901 - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} # - INIT_WORKSPACE=huly @@ -102,32 +192,53 @@ services: links: - cockroach - minio + - stats volumes: - ./branding-test.json:/var/cfg/branding.json environment: - WS_OPERATION=all+backup - REGION=europe - SERVER_SECRET=secret + - QUEUE_CONFIG=${QUEUE_CONFIG} - DB_URL=${DB_EU_URL} + - STATS_URL=http://huly.local:4901 - STORAGE_CONFIG=${STORAGE_CONFIG} - MODEL_ENABLED=* - ACCOUNTS_URL=http://huly.local:3003 - BRANDING_PATH=/var/cfg/branding.json - - STATS_URL=http://huly.local:4901 - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} + # - INIT_WORKSPACE=huly + restart: unless-stopped + collaborator: + image: hardcoreeng/collaborator + extra_hosts: + - 'huly.local:host-gateway' + links: + - mongodb + - minio + - transactor + - stats + ports: + - 3079:3079 + environment: + - COLLABORATOR_PORT=3079 + - SECRET=secret + - ACCOUNTS_URL=http://huly.local:3003 + - STORAGE_CONFIG=${STORAGE_CONFIG} + - STATS_URL=http://huly.local:4901 restart: unless-stopped front: image: hardcoreeng/front extra_hosts: - 'huly.local:host-gateway' - pull_policy: never links: - account - mongodb - minio - elastic - collaborator + - stats - transactor ports: - 8083:8083 @@ -138,102 +249,89 @@ services: - SERVER_SECRET=secret - ACCOUNTS_URL_INTERNAL=http://huly.local:3003 - ACCOUNTS_URL=http://huly.local:3003 + - STATS_URL=http://huly.local:4901 - UPLOAD_URL=/files - - ELASTIC_URL=http://huly.local:9201 - GMAIL_URL=http://huly.local:8088 - CALENDAR_URL=http://huly.local:8095 - - REKONI_URL=http://huly.local:4007 - TELEGRAM_URL=http://huly.local:8086 + - REKONI_URL=http://huly.local:4007 - COLLABORATOR_URL=ws://huly.local:3079 - STORAGE_CONFIG=${STORAGE_CONFIG} + - GITHUB_URL=http://huly.local:3501 + - PRINT_URL=http://huly.local:4006 + - SIGN_URL=http://huly.local:4007 - BRANDING_URL=http://huly.local:8083/branding-test.json - - STATS_URL=http://huly.local:4901 - STATS_API=http://huly.local:4901 + - STREAM_URL=http://huly.local:1081/recording + # - DISABLE_SIGNUP=true + restart: unless-stopped transactor: image: hardcoreeng/transactor extra_hosts: - 'huly.local:host-gateway' - pull_policy: never links: - mongodb - - elastic - minio - - rekoni - account + - stats ports: - 3334:3334 volumes: - ./branding-test.json:/var/cfg/branding.json environment: + - QUEUE_CONFIG=${QUEUE_CONFIG} - REGION= - SERVER_PORT=3334 - SERVER_SECRET=secret + - ENABLE_COMPRESSION=true + - STATS_URL=http://huly.local:4901 + - FULLTEXT_URL=http://huly.local:4710 - DB_URL=${DB_URL} - METRICS_CONSOLE=false - METRICS_FILE=metrics.txt - STORAGE_CONFIG=${STORAGE_CONFIG} - FRONT_URL=http://huly.local:8083 - ACCOUNTS_URL=http://huly.local:3003 - - COLLABORATOR_URL=http://huly.local:3079 - LAST_NAME_FIRST=true - BRANDING_PATH=/var/cfg/branding.json - - FULLTEXT_URL=http://huly.local:4710 - - STATS_URL=http://huly.local:4901 - - ENABLE_COMPRESSION=true + - AI_BOT_URL=http://huly.local:4011 transactor-europe: image: hardcoreeng/transactor extra_hosts: - 'huly.local:host-gateway' pull_policy: never links: - - elastic - - minio - - rekoni - cockroach + - minio - account + - stats ports: - 3335:3335 volumes: - ./branding-test.json:/var/cfg/branding.json environment: + - QUEUE_CONFIG=${QUEUE_CONFIG} - SERVER_PORT=3335 - REGION=europe - SERVER_SECRET=secret + - ENABLE_COMPRESSION=true + - FULLTEXT_URL=http://huly.local:4710 + - STATS_URL=http://huly.local:4901 - DB_URL=${DB_EU_URL} - METRICS_CONSOLE=false - METRICS_FILE=metrics.txt - STORAGE_CONFIG=${STORAGE_CONFIG} - FRONT_URL=http://huly.local:8083 - ACCOUNTS_URL=http://huly.local:3003 - - COLLABORATOR_URL=http://hult-local:3079 - LAST_NAME_FIRST=true - BRANDING_PATH=/var/cfg/branding.json - - FULLTEXT_URL=http://huly.local:4710 - - STATS_URL=http://huly.local:4901 - - ENABLE_COMPRESSION=true - collaborator: - image: hardcoreeng/collaborator - extra_hosts: - - 'huly.local:host-gateway' - links: - - mongodb - - minio - - transactor - ports: - - 3079:3079 - environment: - - COLLABORATOR_PORT=3079 - - SECRET=secret - - ACCOUNTS_URL=http://huly.local:3003 - - STORAGE_CONFIG=${STORAGE_CONFIG} - - FULLTEXT_URL=http://huly.local:4710 - - STATS_URL=http://huly.local:4901 + - AI_BOT_URL=http://huly.local:4011 restart: unless-stopped rekoni: image: hardcoreeng/rekoni-service extra_hosts: - 'huly.local:host-gateway' - restart: on-failure + restart: unless-stopped ports: - 4007:4004 environment: @@ -246,28 +344,26 @@ services: links: - elastic - mongodb - - cockroach - rekoni ports: - 4710:4710 environment: - PORT=4710 - SERVER_SECRET=secret + - QUEUE_CONFIG=${QUEUE_CONFIG} - DB_URL=${DB_URL} - FULLTEXT_DB_URL=http://huly.local:9201 - ELASTIC_INDEX_NAME=local_storage_index - STORAGE_CONFIG=${STORAGE_CONFIG} + - STATS_URL=http://huly.local:4901 - REKONI_URL=http://huly.local:4007 - ACCOUNTS_URL=http://huly.local:3003 - - STATS_URL=http://huly.local:4901 fulltext-europe: image: hardcoreeng/fulltext extra_hosts: - 'huly.local:host-gateway' - restart: unless-stopped links: - elastic - - mongodb - cockroach - rekoni ports: @@ -276,20 +372,32 @@ services: - PORT=4711 - REGION=europe - SERVER_SECRET=secret + - QUEUE_CONFIG=${QUEUE_CONFIG} - DB_URL=${DB_EU_URL} - FULLTEXT_DB_URL=http://huly.local:9201 - ELASTIC_INDEX_NAME=local_eu_storage_index # Use different one to be able to check migration is working - STORAGE_CONFIG=${STORAGE_CONFIG} + - STATS_URL=http://huly.local:4901 - REKONI_URL=http://huly.local:4007 - ACCOUNTS_URL=http://huly.local:3003 - - STATS_URL=http://huly.local:4901 - stats: - image: hardcoreeng/stats - extra_hosts: - - 'huly.local:host-gateway' - ports: - - 4901:4901 - environment: - - PORT=4901 - SERVER_SECRET=secret restart: unless-stopped + aiBot: + image: hardcoreeng/ai-bot + ports: + - 4011:4010 + extra_hosts: + - 'huly.local:host-gateway' + restart: unless-stopped + environment: + - SERVER_SECRET=secret + - MONGO_URL=${MONGO_URL} + - QUEUE_CONFIG=${QUEUE_CONFIG} + - ACCOUNTS_URL=http://huly.local:3000 + - STORAGE_CONFIG=${STORAGE_CONFIG} + - FIRST_NAME=Jolie + - LAST_NAME=AI + - PASSWORD=password + - AVATAR_PATH=./avatar.png + - AVATAR_CONTENT_TYPE=.png + - STATS_URL=http://huly.local:4900 diff --git a/ws-tests/prepare.sh b/ws-tests/prepare.sh index 9f0ea14804..9e5ee6b99e 100755 --- a/ws-tests/prepare.sh +++ b/ws-tests/prepare.sh @@ -37,3 +37,5 @@ fi ./tool-europe.sh create-workspace api-tests-cr email:user1 --region 'europe' ./tool.sh assign-workspace user1 api-tests ./tool.sh assign-workspace user1 api-tests-cr + +rm -rf ./sanity/.auth diff --git a/ws-tests/profile-download.sh b/ws-tests/profile-download.sh index 91d8f5454f..4cfe3ca63c 100755 --- a/ws-tests/profile-download.sh +++ b/ws-tests/profile-download.sh @@ -2,4 +2,4 @@ echo "Downloading profile" token=$(./tool.sh generate-token --admin anticrm@hc.engineering sanity-ws) current=$(date +%Y%m%d%H%M%S) mkdir -p ./profiles -curl -X PUT "http://localhost:3334/api/v1/manage?token=${token}&operation=profile-stop" -o "./profiles/profile-${current}".cpuprofile +curl -X PUT "http://huly.local:3334/api/v1/manage?token=${token}&operation=profile-stop" -o "./profiles/profile-${current}".cpuprofile diff --git a/ws-tests/profile-start.sh b/ws-tests/profile-start.sh index 8ab5b509e6..70723e9ca4 100755 --- a/ws-tests/profile-start.sh +++ b/ws-tests/profile-start.sh @@ -1,3 +1,3 @@ echo "Start profiling on server" token=$(./tool.sh generate-token --admin anticrm@hc.engineering sanity-ws) -curl -X PUT "http://localhost:3334/api/v1/manage?token=${token}&operation=profile-start" +curl -X PUT "http://huly.local:3334/api/v1/manage?token=${token}&operation=profile-start" diff --git a/ws-tests/sanity/package.json b/ws-tests/sanity/package.json index d379fa21e6..90252f990d 100644 --- a/ws-tests/sanity/package.json +++ b/ws-tests/sanity/package.json @@ -19,14 +19,14 @@ "format": "format tests", "ci": "playwright install --with-deps chromium", "test": "", - "uitest": "cross-env LOCAL_URL=http://localhost:3003/ DEV_URL= playwright test -c ./tests/playwright.config.ts", + "uitest": "cross-env LOCAL_URL=http://huly.local:3003/ DEV_URL= playwright test -c ./tests/playwright.config.ts", "staging-uitest": "cross-env PLATFORM_URI=https://front.hc.engineering/ playwright test -c ./tests/playwright.config.ts --grep @staging", - "dev-uitest": "cross-env PLATFORM_URI=http://localhost:8080 PLATFORM_TRANSACTOR=ws://localhost:3333 DEV_URL=http://localhost:8080/account playwright test -c ./tests/playwright.config.ts", - "debug": "cross-env LOCAL_URL=http://localhost:3003/ DEV_URL= playwright test -c ./tests/playwright.config.ts --debug --headed", - "dev-debug": "cross-env PLATFORM_URI=http://localhost:8080 PLATFORM_TRANSACTOR=ws://localhost:3333 playwright test -c ./tests/playwright.config.ts --debug --headed", - "codegen": "playwright codegen --load-storage .auth/storage.json http://localhost:8083/login/selectWorkspace", - "codegen-admin": "playwright codegen --load-storage .auth/adminStorage.json http://localhost:8083/login/admin", - "dev-codegen": "cross-env playwright codegen --load-storage storage-dev.json http://localhost:8080/workbench/sanity-ws/", + "dev-uitest": "cross-env PLATFORM_URI=http://huly.local:8080 PLATFORM_TRANSACTOR=ws://huly.local:3333 DEV_URL=http://huly.local:8080/account playwright test -c ./tests/playwright.config.ts", + "debug": "cross-env LOCAL_URL=http://huly.local:3003/ DEV_URL= playwright test -c ./tests/playwright.config.ts --debug --headed", + "dev-debug": "cross-env PLATFORM_URI=http://huly.local:8080 PLATFORM_TRANSACTOR=ws://huly.local:3333 playwright test -c ./tests/playwright.config.ts --debug --headed", + "codegen": "playwright codegen --load-storage .auth/storage.json http://huly.local:8083/login/selectWorkspace", + "codegen-admin": "playwright codegen --load-storage .auth/adminStorage.json http://huly.local:8083/login/admin", + "dev-codegen": "cross-env playwright codegen --load-storage storage-dev.json http://huly.local:8080/workbench/sanity-ws/", "allure:generate": "allure generate allure-results -o allure-report --clean" }, "devDependencies": { diff --git a/ws-tests/sanity/tests/workspace/archive.spec.ts b/ws-tests/sanity/tests/workspace/archive.spec.ts index 1d7e1c2d26..972f057ba4 100644 --- a/ws-tests/sanity/tests/workspace/archive.spec.ts +++ b/ws-tests/sanity/tests/workspace/archive.spec.ts @@ -81,7 +81,7 @@ test.describe('Workspace Archive tests', () => { await page2.locator('div:nth-child(4) > .checkbox-container > .checkSVG').click() await page2.getByRole('button', { name: 'America', exact: true }).click() - await page2.getByRole('button', { name: 'europe (hidden)' }).click() + await page2.getByRole('button', { name: 'europe' }).click() await page2.getByPlaceholder('Search').click() await page2.getByPlaceholder('Search').fill(workspaceInfo.workspace) await page2.locator(`[id="${workspaceInfo.workspace}"]`).getByRole('button', { name: 'Archive' }).click() diff --git a/ws-tests/sanity/tests/workspace/create.spec.ts b/ws-tests/sanity/tests/workspace/create.spec.ts index dd70fb22b8..99a5283334 100644 --- a/ws-tests/sanity/tests/workspace/create.spec.ts +++ b/ws-tests/sanity/tests/workspace/create.spec.ts @@ -198,6 +198,10 @@ test.describe('Workspace tests', () => { return url.pathname.startsWith('/login/createWorkspace') }) + await page2.waitForURL((url) => { + return url.pathname.startsWith('/login/createWorkspace') + }) + // Ok we signed in, and no workspace present. await page2.goto(linkText ?? '') const joinPage = new SignInJoinPage(page2) diff --git a/ws-tests/sanity/tests/workspace/migrate.spec.ts b/ws-tests/sanity/tests/workspace/migrate.spec.ts index e4a4e8ef25..92757b99fd 100644 --- a/ws-tests/sanity/tests/workspace/migrate.spec.ts +++ b/ws-tests/sanity/tests/workspace/migrate.spec.ts @@ -67,7 +67,6 @@ test.describe('Workspace Migration tests', () => { await loginPage2.goto() await loginPage2.login('admin', '1234') await page.waitForURL((url) => { - console.log(url.pathname) return url.pathname.startsWith('/login/selectWorkspace') || url.pathname.startsWith('/workbench/') }) @@ -83,7 +82,7 @@ test.describe('Workspace Migration tests', () => { await page2.locator('div:nth-child(4) > .checkbox-container > .checkSVG').click() await page2.getByRole('button', { name: 'America', exact: true }).click() - await page2.getByRole('button', { name: 'europe (hidden)' }).click() + await page2.getByRole('button', { name: 'europe' }).click() await page2.getByPlaceholder('Search').click() await page2.getByPlaceholder('Search').fill(workspaceInfo.workspace) await page2.locator(`[id="${workspaceInfo.workspace}"]`).getByRole('button', { name: 'Migrate' }).click() diff --git a/ws-tests/tool-europe.sh b/ws-tests/tool-europe.sh index 7a68bb17ad..cd39c2d5fd 100755 --- a/ws-tests/tool-europe.sh +++ b/ws-tests/tool-europe.sh @@ -3,18 +3,19 @@ export MODEL_VERSION=$(node ../common/scripts/show_version.js) export MINIO_ACCESS_KEY=minioadmin export MINIO_SECRET_KEY=minioadmin -export MINIO_ENDPOINT=localhost:9002 -export ACCOUNTS_URL=http://localhost:3003 -export TRANSACTOR_URL=ws://localhost:3334 -# export ACCOUNT_DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable -export ACCOUNT_DB_URL=mongodb://localhost:27018 -export MONGO_URL=mongodb://localhost:27018 -export ELASTIC_URL=http://localhost:9201 +export MINIO_ENDPOINT=huly.local:9002 +export ACCOUNTS_URL=http://huly.local:3003 +export TRANSACTOR_URL=ws://huly.local:3334 +# export ACCOUNT_DB_URL=postgresql://root@huly.local:26258/defaultdb?sslmode=disable +export ACCOUNT_DB_URL=mongodb://huly.local:27018 +export MONGO_URL=mongodb://huly.local:27018 +export ELASTIC_URL=http://huly.local:9201 export SERVER_SECRET=secret -export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable +export DB_URL=postgresql://root@huly.local:26258/defaultdb?sslmode=disable export REGION_INFO="|America;europe|" # Europe without name will not be available for creation of new workspaces. -export TRANSACTOR_URL="ws://transactor:3334;ws://localhost:3334,ws://transactor-europe:3335;ws://localhost:3335;europe," +export TRANSACTOR_URL="ws://huly.local:3334;ws://huly.localv:3334,ws://huly.local:3335;ws://huly.local:3335;europe," +export QUEUE_CONFIG=huly.local:19093 # Check if local bundle.js exists and use it if available BUNDLE_PATH="../dev/tool/bundle/bundle.js" diff --git a/ws-tests/tool-local.sh b/ws-tests/tool-local.sh index 699ccd9fd3..a12abf336d 100755 --- a/ws-tests/tool-local.sh +++ b/ws-tests/tool-local.sh @@ -1,14 +1,15 @@ export MODEL_VERSION=$(node ../common/scripts/show_version.js) export MINIO_ACCESS_KEY=minioadmin export MINIO_SECRET_KEY=minioadmin -export MINIO_ENDPOINT=localhost:9000 -export MONGO_URL=mongodb://localhost:27017 -export DB_URL=mongodb://localhost:27017 -export ACCOUNT_DB_URL=mongodb://localhost:27017 -export ACCOUNTS_URL=http://localhost:3000 -export TRANSACTOR_URL=ws://localhost:3333 -export ELASTIC_URL=http://localhost:9200 +export MINIO_ENDPOINT=huly.local:9000 +export MONGO_URL=mongodb://huly.local:27017 +export DB_URL=mongodb://huly.local:27017 +export ACCOUNT_DB_URL=mongodb://huly.local:27017 +export ACCOUNTS_URL=http://huly.local:3000 +export TRANSACTOR_URL=ws://huly.local:3333 +export ELASTIC_URL=http://huly.local:9200 export SERVER_SECRET=secret +export QUEUE_CONFIG=huly.local:19093 # Restore workspace contents in mongo/elastic node ../dev/tool/bundle/bundle.js $@ \ No newline at end of file diff --git a/ws-tests/tool.sh b/ws-tests/tool.sh index 7795c05efe..e9f208df09 100755 --- a/ws-tests/tool.sh +++ b/ws-tests/tool.sh @@ -3,15 +3,16 @@ export MODEL_VERSION=$(node ../common/scripts/show_version.js) export MINIO_ACCESS_KEY=minioadmin export MINIO_SECRET_KEY=minioadmin -export MINIO_ENDPOINT=localhost:9002 -export ACCOUNTS_URL=http://localhost:3003 -export TRANSACTOR_URL=ws://localhost:3334 -# export ACCOUNT_DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable -export ACCOUNT_DB_URL=mongodb://localhost:27018 -export MONGO_URL=mongodb://localhost:27018 -export ELASTIC_URL=http://localhost:9201 +export MINIO_ENDPOINT=huly.local:9002 +export ACCOUNTS_URL=http://huly.local:3003 +export TRANSACTOR_URL=ws://huly.local:3334 +# export ACCOUNT_DB_URL=postgresql://root@huly.local:26258/defaultdb?sslmode=disable +export ACCOUNT_DB_URL=mongodb://huly.local:27018 +export MONGO_URL=mongodb://huly.local:27018 +export ELASTIC_URL=http://huly.local:9201 export SERVER_SECRET=secret export DB_URL=$MONGO_URL +export QUEUE_CONFIG=huly.local:19093 # Check if local bundle.js exists and use it if available BUNDLE_PATH="../dev/tool/bundle/bundle.js" diff --git a/ws-tests/wait-elastic.sh b/ws-tests/wait-elastic.sh index bd032456f5..752854f80a 100755 --- a/ws-tests/wait-elastic.sh +++ b/ws-tests/wait-elastic.sh @@ -3,7 +3,7 @@ port=$1 echo "Warning Elastic to up and running with attachment processor... ${port}" for i in `seq 1 30`; do - res=$(curl -s http://localhost:${port}/_cluster/health ) + res=$(curl -s http://huly.local:${port}/_cluster/health ) echo "$res" if [[ $res = *"yellow"* ]]; then echo "Elastic up and running..."