WEBDIENSTE/huly-platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2026-08-17 18:05:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

use redpanda queue (#8291)

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev
2025-03-22 13:55:46 +05:00
committed by GitHub
parent 1f82104933
commit 58eb205688
143 changed files with 2288 additions and 2018 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/main.yml
+6
View File
@@ -228,6 +228,9 @@ jobs:
run: node common/scripts/install-run-rush.js docker
env:
DOCKER_CLI_HINTS: false
- name: Configure /etc/hosts
run: |
sudo echo "127.0.0.1 huly.local" | sudo tee -a /etc/hosts
- name: Prepare server
run: |
cd ./tests
@@ -343,6 +346,9 @@ jobs:
run: node common/scripts/install-run-rush.js docker
env:
DOCKER_CLI_HINTS: false
- name: Configure /etc/hosts
run: |
sudo echo "127.0.0.1 huly.local" | sudo tee -a /etc/hosts
- name: Prepare server
run: |
cd ./tests
.vscode/launch.json
Vendored
+59 -26
View File
@@ -11,9 +11,7 @@
"port": 9229,
"request": "attach",
"sourceMaps": true,
"skipFiles": [
"<node_internals>/**"
],
"skipFiles": ["<node_internals>/**"],
"type": "node"
},
{
@@ -25,7 +23,7 @@
"console": "integratedTerminal",
"justMyCode": true,
"env": {
"PYTORCH_ENABLE_MPS_FALLBACK":"1"
"PYTORCH_ENABLE_MPS_FALLBACK": "1"
}
},
{
@@ -54,7 +52,7 @@
"ACCOUNTS_URL": "http://localhost:3000",
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
// "SERVER_PROVIDER":"uweb"
"SERVER_PROVIDER":"ws",
"SERVER_PROVIDER": "ws",
"MODEL_VERSION": "0.7.1",
// "VERSION": "0.6.289",
"ELASTIC_INDEX_NAME": "local_storage_index",
@@ -97,13 +95,14 @@
"ACCOUNTS_URL": "http://localhost:3000",
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
// "SERVER_PROVIDER":"uweb"
"SERVER_PROVIDER":"ws",
"MODEL_VERSION": "0.7.1",
"SERVER_PROVIDER": "ws",
"MODEL_VERSION": "0.7.24",
// "VERSION": "0.6.289",
"ELASTIC_INDEX_NAME": "local_storage_index",
"UPLOAD_URL": "/files",
"AI_BOT_URL": "http://localhost:4010",
"STATS_URL": "http://huly.local:4900"
"STATS_URL": "http://huly.local:4900",
"QUEUE_CONFIG": "localhost:19092"
},
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"runtimeVersion": "20",
@@ -163,8 +162,10 @@
"REKONI_URL": "http://localhost:4004",
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
"ELASTIC_INDEX_NAME": "local_storage_index",
"STATS_URL":"http://huly.local:4900",
"REGION": "cockroach",
"STATS_URL": "http://huly.local:4900",
"ACCOUNTS_URL": "http://localhost:3000",
"QUEUE_CONFIG": "localhost:19092"
},
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"runtimeVersion": "20",
@@ -184,7 +185,7 @@
"DB_URL": "mongodb://localhost:27017",
// "DB_URL": "postgresql://root@huly.local:26257/defaultdb?sslmode=disable",
"SERVER_SECRET": "secret",
"REGION_INFO":"|Mongo;cockroach|CockroachDB",
"REGION_INFO": "|Mongo;cockroach|CockroachDB",
"TRANSACTOR_URL": "ws://huly.local:3333,ws://huly.local:3332;;cockroach",
"ACCOUNTS_URL": "http://localhost:3000",
"ACCOUNT_PORT": "3000",
@@ -215,8 +216,8 @@
"DB_URL": "mongodb://localhost:27018",
// "DB_URL": "postgresql://postgres:example@localhost:5432",
"SERVER_SECRET": "secret",
"REGION_INFO":"|Mongo;pg|Postgres;cockroach|CockroachDB",
"TRANSACTOR_URL": "ws://huly.local:3334;;,ws://huly.local:3335;;europe",
"REGION_INFO": "|Mongo;pg|Postgres;cockroach|CockroachDB",
"TRANSACTOR_URL": "ws://huly.local:3334;;,ws://huly.local:3335;;europe",
"ACCOUNTS_URL": "http://localhost:3003",
"ACCOUNT_PORT": "3003",
"FRONT_URL": "http://localhost:8083",
@@ -261,7 +262,7 @@
"DB_URL": "mongodb://localhost:27017",
// "DB_URL": "postgresql://postgres:example@localhost:5432",
// "DB_URL": "postgresql://root@huly.local:26257/defaultdb?sslmode=disable",
"REGION": "",
"SERVER_SECRET": "secret",
"TRANSACTOR_URL": "ws://localhost:3333",
@@ -274,7 +275,7 @@
"MODEL_VERSION": "v0.7.1",
"WS_OPERATION": "all+backup",
"BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
"BACKUP_BUCKET": "dev-backups",
"BACKUP_BUCKET": "dev-backups"
// "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init",
// "INIT_WORKSPACE": "staging-dev"
},
@@ -307,7 +308,7 @@
"MODEL_VERSION": "0.7.1",
"WS_OPERATION": "all+backup",
"BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
"BACKUP_BUCKET": "dev-backups",
"BACKUP_BUCKET": "dev-backups"
// "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init",
// "INIT_WORKSPACE": "staging-dev"
},
@@ -447,7 +448,7 @@
"name": "Debug tool upgrade PG(Cockroach)",
"type": "node",
"request": "launch",
"args": ["src/__start.ts", "upgrade-workspace", "platform" ],
"args": ["src/__start.ts", "upgrade-workspace", "platform"],
"env": {
"SERVER_SECRET": "secret",
"MINIO_ACCESS_KEY": "minioadmin",
@@ -505,8 +506,8 @@
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
"SECRET": "secret",
"REGION": "cockroach",
"BUCKET_NAME":"backups",
"INTERVAL":"0"
"BUCKET_NAME": "backups",
"INTERVAL": "0"
},
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"showAsyncStacks": true,
@@ -528,7 +529,7 @@
"DB_URL": "mongodb://localhost:27017",
"SERVER_SECRET": "secret",
"SECRET": "secret",
"BUCKET_NAME":"backups"
"BUCKET_NAME": "backups"
},
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"showAsyncStacks": true,
@@ -548,7 +549,7 @@
// "DB_URL": "mongodb://localhost:27017",
"DB_URL": "postgresql://root@host.docker.internal:26258/defaultdb?sslmode=disable",
"SERVER_SECRET": "secret",
"BUCKET_NAME":"dev-backups"
"BUCKET_NAME": "dev-backups"
},
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"showAsyncStacks": true,
@@ -714,7 +715,7 @@
"AVATAR_PATH": "./assets/avatar.png",
"AVATAR_CONTENT_TYPE": ".png",
"STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
"LOVE_ENDPOINT": "http://localhost:8096",
"LOVE_ENDPOINT": "http://localhost:8096"
},
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"sourceMaps": true,
@@ -771,14 +772,26 @@
"urlFilter": "http://localhost:8080/*",
"port": 9222,
"sourceMapPathOverrides": {
"webpack://*": "${workspaceFolder}/*",
"webpack://*": "${workspaceFolder}/*"
}
},
{
"name": "Debug Notion import",
"type": "node",
"request": "launch",
"args": ["src/__start.ts", "import-notion-to-teamspace", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-u", "user1", "-pw", "1234", "-ws", "ws5", "-ts", "notion", ],
"args": [
"src/__start.ts",
"import-notion-to-teamspace",
"/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743",
"-u",
"user1",
"-pw",
"1234",
"-ws",
"ws5",
"-ts",
"notion"
],
// "args": ["src/__start.ts", "import-notion-with-teamspaces", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-u", "user1", "-pw", "1234", "-ws", "ws1"],
"env": {
"FRONT_URL": "http://localhost:8087"
@@ -793,9 +806,19 @@
"name": "Debug ClickUp import",
"type": "node",
"request": "launch",
"args": ["src/__start.ts", "import-clickup-tasks", "/home/anna/work/clickup/aleksandr/debug/mentions.csv", "-u", "user1", "-pw", "1234", "-ws", "ws10"],
"args": [
"src/__start.ts",
"import-clickup-tasks",
"/home/anna/work/clickup/aleksandr/debug/mentions.csv",
"-u",
"user1",
"-pw",
"1234",
"-ws",
"ws10"
],
"env": {
"FRONT_URL": "http://localhost:8087",
"FRONT_URL": "http://localhost:8087"
},
"runtimeVersion": "20",
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
@@ -807,7 +830,17 @@
"name": "Debug Huly import",
"type": "node",
"request": "launch",
"args": ["src/__start.ts", "import", "/home/anna/huly/platform/dev/import-tool/docs/huly/example-workspace", "-u", "user1", "-pw", "1234", "-ws", "ws1"],
"args": [
"src/__start.ts",
"import",
"/home/anna/huly/platform/dev/import-tool/docs/huly/example-workspace",
"-u",
"user1",
"-pw",
"1234",
"-ws",
"ws1"
],
"env": {
"FRONT_URL": "http://localhost:8087"
},
README.md
+1
View File
@@ -159,6 +159,7 @@ Add the following line to your /etc/hosts file
```plain
127.0.0.1 huly.local
::1 huly.local
```
Accessing the URL <http://huly.local:8087> will lead you to the app in development mode.
common/config/rush/pnpm-lock.yaml
Generated
+108 -151
View File
@@ -316,6 +316,9 @@ importers:
'@rush-temp/inventory-resources':
specifier: file:./projects/inventory-resources.tgz
version: file:projects/inventory-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/kafka':
specifier: file:./projects/kafka.tgz
version: file:projects/kafka.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/kanban':
specifier: file:./projects/kanban.tgz
version: file:projects/kanban.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
@@ -508,9 +511,6 @@ importers:
'@rush-temp/model-server-drive':
specifier: file:./projects/model-server-drive.tgz
version: file:projects/model-server-drive.tgz
'@rush-temp/model-server-fulltext':
specifier: file:./projects/model-server-fulltext.tgz
version: file:projects/model-server-fulltext.tgz
'@rush-temp/model-server-gmail':
specifier: file:./projects/model-server-gmail.tgz
version: file:projects/model-server-gmail.tgz
@@ -922,12 +922,6 @@ importers:
'@rush-temp/server-drive-resources':
specifier: file:./projects/server-drive-resources.tgz
version: file:projects/server-drive-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/server-fulltext':
specifier: file:./projects/server-fulltext.tgz
version: file:projects/server-fulltext.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/server-fulltext-resources':
specifier: file:./projects/server-fulltext-resources.tgz
version: file:projects/server-fulltext-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/server-github':
specifier: file:./projects/server-github.tgz
version: file:projects/server-github.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
@@ -1786,6 +1780,9 @@ importers:
jwt-simple:
specifier: ^0.5.6
version: 0.5.6
kafkajs:
specifier: ^2.2.4
version: 2.2.4
koa:
specifier: ^2.15.4
version: 2.16.0
@@ -3830,7 +3827,7 @@ packages:
version: 0.0.0
'@rush-temp/ai-bot-resources@file:projects/ai-bot-resources.tgz':
resolution: {integrity: sha512-6E8sBtpn6OP/Vpgmf02OPnglOUBfDvB0iZFmvu6Dp1NhTir70tm8kV8IBCqU4MadnD4f636sZwKEItk4NXv+Rw==, tarball: file:projects/ai-bot-resources.tgz}
resolution: {integrity: sha512-F9ehCKoqDjVOCtNkYkhL5J8f2a6LVuZZlFjnlNllq3cdEv9D5s8q17KREQnVjEwdXK6mPrUbsWaDjNDpyd8Bug==, tarball: file:projects/ai-bot-resources.tgz}
version: 0.0.0
'@rush-temp/ai-bot@file:projects/ai-bot.tgz':
@@ -4153,6 +4150,10 @@ packages:
resolution: {integrity: sha512-FVCeKLDYhO53Xnhz0Q/64nU6cMcGaQWrF8jvheuF14ZhPevDUaQfRMwnLR4RGBzIhiTc9jvUIUM54VI1FSkoAA==, tarball: file:projects/inventory.tgz}
version: 0.0.0
'@rush-temp/kafka@file:projects/kafka.tgz':
resolution: {integrity: sha512-Q5+NPNp08oBRTGukVK1HY4Bx/XI3TSnbIY2TIhi4HhU9etcWhHL2EGcsmWhFRfG0RC2gcXC9LI3prx9HDc2kEw==, tarball: file:projects/kafka.tgz}
version: 0.0.0
'@rush-temp/kanban@file:projects/kanban.tgz':
resolution: {integrity: sha512-a1aAx7zBxeZjaejI86Ho69hHcEm7F4dkdD9rVCv2BVHzjG/1C8mq7pv+QWHFGoxkiQwwm9MatVV4IMCSjDjAZw==, tarball: file:projects/kanban.tgz}
version: 0.0.0
@@ -4222,7 +4223,7 @@ packages:
version: 0.0.0
'@rush-temp/model-all@file:projects/model-all.tgz':
resolution: {integrity: sha512-G7I5APyAySX8V9/mzUjuXX3LGtRNZ+uYQhHTl4/7ulGaAsuSagzDJIvClO2akY+GhplEfRrKBiI2fb0R0/i30g==, tarball: file:projects/model-all.tgz}
resolution: {integrity: sha512-UxCRw3u19dTizt2hYLWPNCJ468ntnLnwEtP/8mAEb1Ta93SbPgbV9RfCK+hBBM/vp843HVwdNrab1fwqFXb+3Q==, tarball: file:projects/model-all.tgz}
version: 0.0.0
'@rush-temp/model-analytics-collector@file:projects/model-analytics-collector.tgz':
@@ -4405,10 +4406,6 @@ packages:
resolution: {integrity: sha512-MQUIWrqoJZkFbbKhT569m2pfvCJsK7l3rVWhO9DlNfxFmItei5ZSsMzC7+4qpI7iyQsb620y23RkhEl17/9YKQ==, tarball: file:projects/model-server-drive.tgz}
version: 0.0.0
'@rush-temp/model-server-fulltext@file:projects/model-server-fulltext.tgz':
resolution: {integrity: sha512-qOs1G3E2P64j6xsyD1w2+ZPLclBeAyx1xmGEI19kneyA8IB8NNRcpI5GIB9vIrYJ2lJUi0jyI+sHkLoxmgtBaA==, tarball: file:projects/model-server-fulltext.tgz}
version: 0.0.0
'@rush-temp/model-server-gmail@file:projects/model-server-gmail.tgz':
resolution: {integrity: sha512-vdwW7KKXUWLji9gEb2uEGwbjyJfeQr2g4G/vhZH15sy87rI5f7+M7x5p7zZq5IOF9dk6YArAbILD/yl7u+E3Xg==, tarball: file:projects/model-server-gmail.tgz}
version: 0.0.0
@@ -4614,7 +4611,7 @@ packages:
version: 0.0.0
'@rush-temp/pod-ai-bot@file:projects/pod-ai-bot.tgz':
resolution: {integrity: sha512-sv9r+DDafpUyjHUabAnULCjQC2xnZ8t4AfhIhvXBM6vl500IrhBVkeAGcNeIk0WYepfAnRn4mQVM3yX+0e0Ygw==, tarball: file:projects/pod-ai-bot.tgz}
resolution: {integrity: sha512-iYWt2dpdc5lKElhJEc89Zar3zq7+qXEKsTAlVqFxMtAxwCgDIQqTvds5b/W5FEvgfJsmXUN/Oib9zBr2urn6Fg==, tarball: file:projects/pod-ai-bot.tgz}
version: 0.0.0
'@rush-temp/pod-analytics-collector@file:projects/pod-analytics-collector.tgz':
@@ -4646,7 +4643,7 @@ packages:
version: 0.0.0
'@rush-temp/pod-fulltext@file:projects/pod-fulltext.tgz':
resolution: {integrity: sha512-rDotFbiaTJUlFISEWa7+hd74cQWXGijcNrq3qeAItyw4A2PVKLSQusomUp5Mo1oP0wIurREXjiKCN1I7I27eAA==, tarball: file:projects/pod-fulltext.tgz}
resolution: {integrity: sha512-za9dXz8LTdV3Mwyr3Lbno6Ys7c0tKu2F9+QH0D75THvniEMYUK1mG4Pt/7hByNzTB1VqzIaozs6Id+DfXYqS2Q==, tarball: file:projects/pod-fulltext.tgz}
version: 0.0.0
'@rush-temp/pod-github@file:projects/pod-github.tgz':
@@ -4654,7 +4651,7 @@ packages:
version: 0.0.0
'@rush-temp/pod-gmail@file:projects/pod-gmail.tgz':
resolution: {integrity: sha512-aYecRf97Yj23bi2BdWXg4DrH4oy3AB2UfrpbwNo6nvLhHJLsrcRgOe9zCTakZ7Yb+nDzsxYuOwZiDIr9aWtsyw==, tarball: file:projects/pod-gmail.tgz}
resolution: {integrity: sha512-HyyHadfmgT1OP7pdln0WJvK6cGPTYVVvxq21JBHKshu40WojCB6dLdAch/YrTO0aSS7UzluuLmYnnCn5JGvfcg==, tarball: file:projects/pod-gmail.tgz}
version: 0.0.0
'@rush-temp/pod-hook@file:projects/pod-hook.tgz':
@@ -4662,7 +4659,7 @@ packages:
version: 0.0.0
'@rush-temp/pod-love@file:projects/pod-love.tgz':
resolution: {integrity: sha512-m+9dPMR45FJpQB+M5ka4PiJcTpV8COlQA5F36is2LkMLwnsDjcB94Sx+TFdIyjhVFpfB8l+t886mSPveHql4ag==, tarball: file:projects/pod-love.tgz}
resolution: {integrity: sha512-K9Hvwm8QesWBTSx/A+swmRDUOy0RY8y4tNJkrW85Zh5wMUxHzeYrX/tQL+usk+cRrvPCw9C8yag1gxsKqPkWIQ==, tarball: file:projects/pod-love.tgz}
version: 0.0.0
'@rush-temp/pod-mail@file:projects/pod-mail.tgz':
@@ -4670,11 +4667,11 @@ packages:
version: 0.0.0
'@rush-temp/pod-print@file:projects/pod-print.tgz':
resolution: {integrity: sha512-i0Y9ShIeieNqCNMCXax1CZTStiDTw9M156m7fCDSrNS6VV4v2e5jy/HyNLtpyXz/ZRlOKndBdyiKsls+5pDL8g==, tarball: file:projects/pod-print.tgz}
resolution: {integrity: sha512-KNZrBeofhGFWQHMd1aPU7smmL50h/fGw4tXy7bEV2PcO14vphe2MyMfuWcTw1ksmeXFP6V3C2XxmgKFk+DT0dg==, tarball: file:projects/pod-print.tgz}
version: 0.0.0
'@rush-temp/pod-server@file:projects/pod-server.tgz':
resolution: {integrity: sha512-MJ5TQobQy4kv67qy7qpQgz056fhaZw8Zs/NaXniFafG/xjstgWeIJCXNHDkKvdBm+wT6EdHgNLIN3BaRpMSDsA==, tarball: file:projects/pod-server.tgz}
resolution: {integrity: sha512-toWgesqQWTYgkwA6VPDiaJJc7ZmDkHFzNg/MbDjSACtmy0/endv3ioglyXYdoDo/m4IbFZOghMUJOZbQgQvRUA==, tarball: file:projects/pod-server.tgz}
version: 0.0.0
'@rush-temp/pod-ses@file:projects/pod-ses.tgz':
@@ -4682,7 +4679,7 @@ packages:
version: 0.0.0
'@rush-temp/pod-sign@file:projects/pod-sign.tgz':
resolution: {integrity: sha512-CmOWryDSvF5qg0NTlkxT29gZP0qj0DimD25X2U6sQGtowtEytNvVmmg3cuSNTpwN+WR6+eli4JWyK6gathi+fQ==, tarball: file:projects/pod-sign.tgz}
resolution: {integrity: sha512-7ERrvIDW3r+NlglIYXW8kK50EiYLL9za+/EON9pW/8u0gkIaPIjEfL6yvdAh3euZ+khgPkzH+jrV57tu+jxT4w==, tarball: file:projects/pod-sign.tgz}
version: 0.0.0
'@rush-temp/pod-stats@file:projects/pod-stats.tgz':
@@ -4694,11 +4691,11 @@ packages:
version: 0.0.0
'@rush-temp/pod-telegram@file:projects/pod-telegram.tgz':
resolution: {integrity: sha512-FAYbl5GmRCS3fOR5eUXKNS8KmwnwB+7ZZ68vDi5lfeH4hkJky/VdR5yq4dX5SzOB35e7hz9JWriDGxfjUcx6Wg==, tarball: file:projects/pod-telegram.tgz}
resolution: {integrity: sha512-CtJJaVgL0u4w9JUOAXdnAJqUnpSIlIpa2aY7OU3vZPR6hLqXt5pmDFvq3siQzkZ+27mJ6vc+IOBAaqbmumOkXg==, tarball: file:projects/pod-telegram.tgz}
version: 0.0.0
'@rush-temp/pod-workspace@file:projects/pod-workspace.tgz':
resolution: {integrity: sha512-6wRqrWl0oZUzQ2CZBpPK09doQpYepraNAwCO3OcPDaRnDfQo0WBwLvfWOdDVW3REunCCh/DHMxGpuiLnaQX4KQ==, tarball: file:projects/pod-workspace.tgz}
resolution: {integrity: sha512-QrThk+6HwkGIdmj2tiPv9LTIOe5+QJChsG9DtIzmxaeyOjnhtpX2ROEuaVXL3oXl97EBoXX8fBsVpDMQ4i+pRw==, tarball: file:projects/pod-workspace.tgz}
version: 0.0.0
'@rush-temp/postgres@file:projects/postgres.tgz':
@@ -4818,7 +4815,7 @@ packages:
version: 0.0.0
'@rush-temp/rekoni-service@file:projects/rekoni-service.tgz':
resolution: {integrity: sha512-drU6icafREaUYcSZfNn/CEYQLJC1ha6FkF7QrRcQ/TsuFaruapqmET0Ttqcr/vQaYemKF6SnshiKRGmeg+tcDA==, tarball: file:projects/rekoni-service.tgz}
resolution: {integrity: sha512-O8bBBE0Bw2m69G/KVLXgHnfa2WzWn9MG1CwklzSf6rkyVT/ltUA7WXwdPEA/ZFjLbkRpgd2RmcVRSERlAXDmRw==, tarball: file:projects/rekoni-service.tgz}
version: 0.0.0
'@rush-temp/rekoni@file:projects/rekoni.tgz':
@@ -4957,14 +4954,6 @@ packages:
resolution: {integrity: sha512-5zQ2vbqHHFW1gdEWDqV8aDGq5u2I4z9qoD4TH0QbBaE5XITQrvJoZfK9WW+CceePEM2O+1clTu9w1KTBIEd5Hg==, tarball: file:projects/server-drive.tgz}
version: 0.0.0
'@rush-temp/server-fulltext-resources@file:projects/server-fulltext-resources.tgz':
resolution: {integrity: sha512-NebNMPNgIeG8N0qA5Q4ROcs5Jn/dmE0WX92uxw/qVUO/k8DLMxRsrFvyKrrMGtYgeivbxJPe6PEd3FijUAMfiw==, tarball: file:projects/server-fulltext-resources.tgz}
version: 0.0.0
'@rush-temp/server-fulltext@file:projects/server-fulltext.tgz':
resolution: {integrity: sha512-QYDVer6XKHqKih+CXofNkysDl9TafFIlN5sw4GdJjLT3/AJt7L2FeCKMuPgMtYQhihj2kodtBfNUbnSeLA8qJw==, tarball: file:projects/server-fulltext.tgz}
version: 0.0.0
'@rush-temp/server-github-model@file:projects/server-github-model.tgz':
resolution: {integrity: sha512-U7kynRwConx+S1nLfFluVkWJtFKCEMJRIHx/oN1OFy2zmIGSw1o7z1NIgEYAiYogj8zP1SPYoVKPQTSkSosKqw==, tarball: file:projects/server-github-model.tgz}
version: 0.0.0
@@ -5038,7 +5027,7 @@ packages:
version: 0.0.0
'@rush-temp/server-pipeline@file:projects/server-pipeline.tgz':
resolution: {integrity: sha512-egC06Gj9kZodIHe8m7g6K99KtOTXZyZJgPymrkDj+GGZX7+7Pd/5Nck0jT4i/Ufk1c0zOxEBXJE4+KTuvo/aew==, tarball: file:projects/server-pipeline.tgz}
resolution: {integrity: sha512-LWUTB3SnKZRoKrdx1ywH5lu/7jjYyD71rffj73ITM9s0Y6G9rSVNF7R4otEwOEmD+ZniJlJTz8h2vqfmEuALEg==, tarball: file:projects/server-pipeline.tgz}
version: 0.0.0
'@rush-temp/server-preference@file:projects/server-preference.tgz':
@@ -5318,7 +5307,7 @@ packages:
version: 0.0.0
'@rush-temp/tool@file:projects/tool.tgz':
resolution: {integrity: sha512-1/Aodec9LySBThOZAeA3XJfP83lSVTVRFdlb8uNTORq1sHJxCADx9BliCh2dTzdhydW46wcAxMPeArhopkcN1Q==, tarball: file:projects/tool.tgz}
resolution: {integrity: sha512-zqh0++u5ETaUNlwkipP+8CeICqrFYEDBO4sAjFRyLPd1SHZbanUpQRAg58t4C+Z7q4Owqr149ee/hCGWPult3w==, tarball: file:projects/tool.tgz}
version: 0.0.0
'@rush-temp/tracker-assets@file:projects/tracker-assets.tgz':
@@ -10075,6 +10064,10 @@ packages:
resolution: {integrity: sha512-40aUybvhH9t2h71ncA1/1SbtTNCVZHgsTsTgqPUxGWDmUDrXyDf2wMNQKEbdBjbf4AI+fQhbECNTV6lWxQKUzg==}
engines: {node: '>= 0.4.0'}
kafkajs@2.2.4:
resolution: {integrity: sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==}
engines: {node: '>=14.0.0'}
katex@0.16.15:
resolution: {integrity: sha512-yE9YJIEAk2aZ+FL/G8r+UGw0CTUzEA8ZFy6E+8tc3spHUKq3qBnzCkI1CQwGoI9atJhVyFPEypQsTY7mJ1Pi9w==}
hasBin: true
@@ -11170,8 +11163,8 @@ packages:
resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==}
engines: {node: '>=8'}
parse5@7.1.2:
resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==}
parse5@7.2.1:
resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==}
parseley@0.12.1:
resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==}
@@ -11481,8 +11474,8 @@ packages:
resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==}
engines: {node: '>=4'}
postgres-array@3.0.4:
resolution: {integrity: sha512-nAUSGfSDGOaOAEGwqsRY27GPOea7CNipJPOA7lPbdEpx5Kg3qzdP0AaWC5MlhTWV9s4hFX39nomVZ+C4tnGOJQ==}
postgres-array@3.0.2:
resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==}
engines: {node: '>=12'}
postgres-bytea@1.0.0:
@@ -16059,7 +16052,7 @@ snapshots:
'@puppeteer/browsers@2.3.0':
dependencies:
debug: 4.4.0
debug: 4.3.5
extract-zip: 2.0.1
progress: 2.0.3
proxy-agent: 6.4.0
@@ -18853,6 +18846,32 @@ snapshots:
- supports-color
- ts-node
'@rush-temp/kafka@file:projects/kafka.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
kafkajs: 2.2.4
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3)
typescript: 5.7.3
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
- babel-jest
- babel-plugin-macros
- esbuild
- node-notifier
- supports-color
- ts-node
'@rush-temp/kanban@file:projects/kanban.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
@@ -19179,17 +19198,17 @@ snapshots:
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3)
typescript: 5.7.3
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.6.2
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
@@ -19203,10 +19222,10 @@ snapshots:
'@rush-temp/mail-resources@file:projects/mail-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
@@ -19219,9 +19238,9 @@ snapshots:
svelte-check: 3.6.9(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)
svelte-eslint-parser: 0.33.1(svelte@4.2.19)
svelte-loader: 3.2.0(svelte@4.2.19)
svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.7.3)
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3)
typescript: 5.7.3
svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.6.2)
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.6.2
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
@@ -19243,17 +19262,17 @@ snapshots:
'@rush-temp/mail@file:projects/mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3)
typescript: 5.7.3
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.6.2
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
@@ -19650,29 +19669,29 @@ snapshots:
'@rush-temp/model-mail@file:projects/model-mail.tgz':
dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
prettier: 3.2.5
typescript: 5.7.3
typescript: 5.6.2
transitivePeerDependencies:
- supports-color
'@rush-temp/model-my-space@file:projects/model-my-space.tgz':
dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
prettier: 3.2.5
typescript: 5.7.3
typescript: 5.6.2
transitivePeerDependencies:
- supports-color
@@ -20017,20 +20036,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
'@rush-temp/model-server-fulltext@file:projects/model-server-fulltext.tgz':
dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
prettier: 3.2.5
typescript: 5.6.2
transitivePeerDependencies:
- supports-color
'@rush-temp/model-server-gmail@file:projects/model-server-gmail.tgz':
dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
@@ -20601,17 +20606,17 @@ snapshots:
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3)
typescript: 5.7.3
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.6.2
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
@@ -20625,10 +20630,10 @@ snapshots:
'@rush-temp/my-space-resources@file:projects/my-space-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
@@ -20642,9 +20647,9 @@ snapshots:
svelte-check: 3.6.9(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)
svelte-eslint-parser: 0.33.1(svelte@4.2.19)
svelte-loader: 3.2.0(svelte@4.2.19)
svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.7.3)
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3)
typescript: 5.7.3
svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.6.2)
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.6.2
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
@@ -20666,17 +20671,17 @@ snapshots:
'@rush-temp/my-space@file:projects/my-space.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3)
typescript: 5.7.3
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.6.2
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
@@ -23846,56 +23851,6 @@ snapshots:
- supports-color
- ts-node
'@rush-temp/server-fulltext-resources@file:projects/server-fulltext-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.6.2
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
- '@types/node'
- babel-jest
- babel-plugin-macros
- esbuild
- node-notifier
- supports-color
- ts-node
'@rush-temp/server-fulltext@file:projects/server-fulltext.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.6.2
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
- babel-jest
- babel-plugin-macros
- esbuild
- node-notifier
- supports-color
- ts-node
'@rush-temp/server-github-model@file:projects/server-github-model.tgz':
dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
@@ -28120,7 +28075,7 @@ snapshots:
dependencies:
'@types/node': 20.11.19
'@types/tough-cookie': 4.0.5
parse5: 7.1.2
parse5: 7.2.1
'@types/json-schema@7.0.15': {}
@@ -32966,7 +32921,7 @@ snapshots:
https-proxy-agent: 5.0.1
is-potential-custom-element-name: 1.0.1
nwsapi: 2.2.13
parse5: 7.1.2
parse5: 7.2.1
saxes: 6.0.0
symbol-tree: 3.2.4
tough-cookie: 4.1.4
@@ -33076,6 +33031,8 @@ snapshots:
jwt-simple@0.5.6: {}
kafkajs@2.2.4: {}
katex@0.16.15:
dependencies:
commander: 8.3.0
@@ -34249,7 +34206,7 @@ snapshots:
json-parse-even-better-errors: 2.3.1
lines-and-columns: 1.2.4
parse5@7.1.2:
parse5@7.2.1:
dependencies:
entities: 4.5.0
@@ -34369,7 +34326,7 @@ snapshots:
dependencies:
pg-int8: 1.0.1
pg-numeric: 1.0.2
postgres-array: 3.0.4
postgres-array: 3.0.2
postgres-bytea: 3.0.0
postgres-date: 2.1.0
postgres-interval: 3.0.0
@@ -34536,7 +34493,7 @@ snapshots:
postgres-array@2.0.0: {}
postgres-array@3.0.4: {}
postgres-array@3.0.2: {}
postgres-bytea@1.0.0: {}
@@ -34784,7 +34741,7 @@ snapshots:
dependencies:
'@puppeteer/browsers': 2.3.0
chromium-bidi: 0.6.2(devtools-protocol@0.0.1312386)
debug: 4.4.0
debug: 4.3.5
devtools-protocol: 0.0.1312386
ws: 8.18.0(bufferutil@4.0.8)(utf-8-validate@6.0.4)
transitivePeerDependencies:
@@ -35671,7 +35628,7 @@ snapshots:
dependencies:
component-emitter: 1.3.1
cookiejar: 2.1.4
debug: 4.4.0
debug: 4.3.5
fast-safe-stringify: 2.1.1
form-data: 4.0.0
formidable: 2.1.2
dev/.env
+2
View File
@@ -3,3 +3,5 @@ MONGO_URL=mongodb://mongodb:27017?compressors=snappy
DB_URL_PG=postgresql://postgres:example@postgres:5432
BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
BACKUP_BUCKET_NAME=dev-backups
DB_CR_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable
QUEUE_CONFIG=redpanda:9092
dev/docker-compose.yaml
+131 -82
View File
@@ -1,5 +1,5 @@
services:
stream:
stream:
image: 'hardcoreeng/stream'
extra_hosts:
- 'huly.local:host-gateway'
@@ -44,9 +44,66 @@ services:
volumes:
- cockroach_db:/cockroach/cockroach-data
restart: unless-stopped
redpanda:
image: docker.redpanda.com/redpandadata/redpanda:v24.3.6
extra_hosts:
- 'huly.local:host-gateway'
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
- --advertise-kafka-addr internal://redpanda:9092,external://localhost:19092
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
- --advertise-pandaproxy-addr internal://redpanda:8082,external://localhost:18082
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
- --rpc-addr redpanda:33145
- --advertise-rpc-addr redpanda:33145
- --mode dev-container
- --smp 1
- --default-log-level=info
container_name: redpanda
volumes:
- redpanda:/var/lib/redpanda/data
ports:
- 18081:18081
- 18082:18082
- 19092:19092
- 19644:9644
healthcheck:
test: ['CMD', 'rpk', 'cluster', 'info', '-X', 'user=superuser', '-X', 'pass=secretpassword']
interval: 10s
timeout: 5s
retries: 10
redpanda_console:
image: docker.redpanda.com/redpandadata/console:v2.8.3
extra_hosts:
- 'huly.local:host-gateway'
entrypoint: /bin/sh
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
environment:
CONFIG_FILEPATH: /tmp/config.yml
CONSOLE_CONFIG_FILE: |
kafka:
brokers: ["redpanda:9092"]
schemaRegistry:
enabled: true
urls: ["http://redpanda:8081"]
redpanda:
adminApi:
enabled: true
urls: ["http://redpanda:9644"]
ports:
- 8000:8080
depends_on:
- redpanda
minio:
image: 'minio/minio'
command: server /data --address ":9000" --console-address ":9001"
extra_hosts:
- 'huly.local:host-gateway'
expose:
- 9000
- 9001
ports:
- 9000:9000
- 9001:9001
@@ -55,9 +112,10 @@ services:
restart: unless-stopped
elastic:
image: 'elasticsearch:7.14.2'
command: |
/bin/sh -c "./bin/elasticsearch-plugin list | grep -q ingest-attachment || yes | ./bin/elasticsearch-plugin install --silent ingest-attachment;
/usr/local/bin/docker-entrypoint.sh eswrapper"
expose:
- 9200
extra_hosts:
- 'huly.local:host-gateway'
volumes:
- elastic:/usr/share/elasticsearch/data
ports:
@@ -67,8 +125,6 @@ services:
- BITNAMI_DEBUG=true
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- http.cors.enabled=true
- http.cors.allow-origin=http://localhost:8082
healthcheck:
interval: 20s
retries: 10
@@ -88,21 +144,20 @@ services:
- ./branding.json:/var/cfg/branding.json
environment:
- ACCOUNT_PORT=3000
- QUEUE_CONFIG=${QUEUE_CONFIG}
- SERVER_SECRET=secret
- ADMIN_EMAILS=admin
- ADMIN_EMAILS=admin,${PLATFORM_ADMIN_EMAILS}
- STATS_URL=http://huly.local:4900
- WORKSPACE_LIMIT_PER_USER=10000
- DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable
- DB_URL=${DB_CR_URL}
# - DB_URL=${MONGO_URL}
# - DB_NS=account-2
# Pass only one region to disallow selection for new workspaces.Ø
# Pass only one region to disallow selection for new workspaces.
- REGION_INFO=|Mongo;cockroach|CockroachDB
# - REGION_INFO=cockroach|CockroachDB
- TRANSACTOR_URL=ws://huly.local:3333,ws://huly.local:3332;;cockroach,
- MAIL_URL=
- STORAGE_CONFIG=${STORAGE_CONFIG}
- FRONT_URL=http://huly.local:8087
- RESERVED_DB_NAMES=telegram,gmail,github
- MODEL_ENABLED=*
- LAST_NAME_FIRST=true
# - WS_LIVENESS_DAYS=1
@@ -133,15 +188,15 @@ services:
environment:
- WS_OPERATION=all+backup
- SERVER_SECRET=secret
- QUEUE_CONFIG=${QUEUE_CONFIG}
- DB_URL=${MONGO_URL}
- STATS_URL=http://huly.local:4900
- MAIL_URL=
- STORAGE_CONFIG=${STORAGE_CONFIG}
- RESERVED_DB_NAMES=telegram,gmail,github
- MODEL_ENABLED=*
- REGION=
- ACCOUNTS_URL=http://huly.local:3000
- BRANDING_PATH=/var/cfg/branding.json
# - PARALLEL=2
- BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
- BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
# - INIT_WORKSPACE=staging-dev
@@ -158,19 +213,18 @@ services:
- ./branding.json:/var/cfg/branding.json
environment:
- WS_OPERATION=all+backup
- SERVER_SECRET=secret
- DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable
- STATS_URL=http://huly.local:4900
- MAIL_URL=
- REGION=cockroach
- SERVER_SECRET=secret
- QUEUE_CONFIG=${QUEUE_CONFIG}
- DB_URL=${DB_CR_URL}
- STATS_URL=http://huly.local:4900
- STORAGE_CONFIG=${STORAGE_CONFIG}
- RESERVED_DB_NAMES=telegram,gmail,github
- MODEL_ENABLED=*
- ACCOUNTS_URL=http://huly.local:3000
- BRANDING_PATH=/var/cfg/branding.json
# - PARALLEL=2
- BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
- BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
- MAIL_URL=
# - INIT_WORKSPACE=staging-dev
restart: unless-stopped
collaborator:
@@ -236,21 +290,19 @@ services:
- minio
- account
- stats
# - apm-server
ports:
- 3333:3333
volumes:
- ./branding.json:/var/cfg/branding.json
environment:
# - SERVER_PROVIDER=uweb
# - UWS_HTTP_MAX_HEADERS_SIZE="32768"
- UV_THREADPOOL_SIZE=10
- QUEUE_CONFIG=${QUEUE_CONFIG}
- REGION=
- SERVER_PORT=3333
- SERVER_SECRET=secret
- ENABLE_COMPRESSION=false
- ENABLE_COMPRESSION=true
- STATS_URL=http://huly.local:4900
- FULLTEXT_URL=http://huly.local:4700
# - DB_URL=postgresql://postgres:example@postgres:5432
# - DB_URL=${DB_CR_URL}
- DB_URL=${MONGO_URL}
- MONGO_URL=${MONGO_URL}
- 'MONGO_OPTIONS={"appName": "transactor", "maxPoolSize": 10}'
@@ -258,7 +310,6 @@ services:
- METRICS_FILE=metrics.txt
- STORAGE_CONFIG=${STORAGE_CONFIG}
- FRONT_URL=http://huly.local:8087
# - APM_SERVER_URL=http://apm-server:8200
- MAIL_URL=''
- ACCOUNTS_URL=http://huly.local:3000
- LAST_NAME_FIRST=true
@@ -274,52 +325,38 @@ services:
- minio
- account
- stats
# - apm-server
ports:
- 3332:3332
volumes:
- ./branding.json:/var/cfg/branding.json
environment:
# - SERVER_PROVIDER=uweb
# - UWS_HTTP_MAX_HEADERS_SIZE="32768"
# - UV_THREADPOOL_SIZE=10
- QUEUE_CONFIG=${QUEUE_CONFIG}
- SERVER_PORT=3332
- REGION=cockroach
- SERVER_SECRET=secret
- ENABLE_COMPRESSION=false
- ENABLE_COMPRESSION=true
- FULLTEXT_URL=http://huly.local:4702
- STATS_URL=http://huly.local:4900
- DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable
- DB_URL=${DB_CR_URL}
- METRICS_CONSOLE=false
- METRICS_FILE=metrics.txt
- STORAGE_CONFIG=${STORAGE_CONFIG}
- FRONT_URL=http://huly.local:8087
# - APM_SERVER_URL=http://apm-server:8200
- MAIL_URL=''
- ACCOUNTS_URL=http://huly.local:3000
- LAST_NAME_FIRST=true
- BRANDING_PATH=/var/cfg/branding.json
- AI_BOT_URL=http://huly.local:4010
restart: unless-stopped
green:
image: hardcoreeng/green
extra_hosts:
- 'huly.local:host-gateway'
links:
- cockroach
- stats
ports:
- 6767:6767
environment:
- PORT=6767
- AUTH_TOKEN=secret
- STATS_URL=http://huly.local:4900
- DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable
restart: unless-stopped
rekoni:
image: hardcoreeng/rekoni-service
extra_hosts:
- 'huly.local:host-gateway'
restart: unless-stopped
ports:
- 4004:4004
environment:
- STATS_URL=http://huly.local:4901
fulltext:
image: hardcoreeng/fulltext
extra_hosts:
@@ -328,10 +365,13 @@ services:
links:
- elastic
- mongodb
- rekoni
ports:
- 4700:4700
environment:
- PORT=4700
- SERVER_SECRET=secret
- QUEUE_CONFIG=${QUEUE_CONFIG}
- DB_URL=${MONGO_URL}
- FULLTEXT_DB_URL=http://huly.local:9200
- ELASTIC_INDEX_NAME=local_storage_index
@@ -347,18 +387,22 @@ services:
links:
- elastic
- cockroach
- rekoni
ports:
- 4702:4702
environment:
- PORT=4702
- REGION=cockroach
- SERVER_SECRET=secret
- DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable
- QUEUE_CONFIG=${QUEUE_CONFIG}
- DB_URL=${DB_CR_URL}
- FULLTEXT_DB_URL=http://huly.local:9200
- ELASTIC_INDEX_NAME=local_storage_index
- ELASTIC_INDEX_NAME=local_storage_index # Same index for simplicity
- STORAGE_CONFIG=${STORAGE_CONFIG}
- STATS_URL=http://huly.local:4900
- REKONI_URL=http://huly.local:4004
- ACCOUNTS_URL=http://huly.local:3000
- SERVER_SECRET=secret
print:
image: hardcoreeng/print
extra_hosts:
@@ -368,6 +412,7 @@ services:
- 4005:4005
environment:
- SECRET=secret
- QUEUE_CONFIG=${QUEUE_CONFIG}
- STORAGE_CONFIG=${STORAGE_CONFIG}
- STATS_URL=http://huly.local:4900
- ACCOUNTS_URL=http://huly.local:3000
@@ -384,6 +429,7 @@ services:
environment:
- SECRET=secret
- MINIO_ENDPOINT=minio
- QUEUE_CONFIG=${QUEUE_CONFIG}
- MINIO_ACCESS_KEY=minioadmin
- ACCOUNTS_URL=http://huly.local:3000
- MINIO_SECRET_KEY=minioadmin
@@ -391,21 +437,21 @@ services:
- SERVICE_ID=sign-service
- BRANDING_PATH=/var/cfg/branding.json
- STATS_URL=http://huly.local:4900
# analytics:
# image: hardcoreeng/analytics-collector
# extra_hosts:
# - 'huly.local:host-gateway'
# restart: unless-stopped
# ports:
# - 4017:4017
# environment:
# - SECRET=secret
# - PORT=4017
# - MONGO_URL=${MONGO_URL}
# - 'MONGO_OPTIONS={"appName":"analytics","maxPoolSize":1}'
# - SERVICE_ID=analytics-collector-service
# - ACCOUNTS_URL=http://huly.local:3000
# - STATS_URL=http://huly.local:4900
# analytics:
# image: hardcoreeng/analytics-collector
# extra_hosts:
# - 'huly.local:host-gateway'
# restart: unless-stopped
# ports:
# - 4017:4017
# environment:
# - SECRET=secret
# - PORT=4017
# - MONGO_URL=${MONGO_URL}
# - 'MONGO_OPTIONS={"appName":"analytics","maxPoolSize":1}'
# - SERVICE_ID=analytics-collector-service
# - ACCOUNTS_URL=http://huly.local:3000
# - STATS_URL=http://huly.local:4900
aiBot:
image: hardcoreeng/ai-bot
ports:
@@ -416,6 +462,7 @@ services:
environment:
- SERVER_SECRET=secret
- MONGO_URL=${MONGO_URL}
- QUEUE_CONFIG=${QUEUE_CONFIG}
- ACCOUNTS_URL=http://huly.local:3000
- STORAGE_CONFIG=${STORAGE_CONFIG}
- FIRST_NAME=Jolie
@@ -424,23 +471,24 @@ services:
- AVATAR_PATH=./avatar.png
- AVATAR_CONTENT_TYPE=.png
- STATS_URL=http://huly.local:4900
# - LOVE_ENDPOINT=http://huly.local:8096
# - OPENAI_API_KEY=token
# telegram-bot:
# image: hardcoreeng/telegram-bot
# extra_hosts:
# - "huly.local:host-gateway"
# restart: unless-stopped
# environment:
# - PORT=4020
# - BOT_TOKEN=token
# - MONGO_URL=${MONGO_URL}
# - MONGO_DB=telegram-bot
# - SECRET=secret
# - DOMAIN=domain
# - ACCOUNTS_URL=http://huly.local:3000
# - SERVICE_ID=telegram-bot-service
# - STATS_URL=http://huly.local:4900
# - LOVE_ENDPOINT=http://huly.local:8096
# - OPENAI_API_KEY=token
# telegram-bot:
# image: hardcoreeng/telegram-bot
# extra_hosts:
# - "huly.local:host-gateway"
# restart: unless-stopped
# environment:
# - PORT=4020
# - QUEUE_CONFIG=${QUEUE_CONFIG}
# - BOT_TOKEN=token
# - MONGO_URL=${MONGO_URL}
# - MONGO_DB=telegram-bot
# - SECRET=secret
# - DOMAIN=domain
# - ACCOUNTS_URL=http://huly.local:3000
# - SERVICE_ID=telegram-bot-service
# - STATS_URL=http://huly.local:4900
export:
image: hardcoreeng/export
extra_hosts:
@@ -465,3 +513,4 @@ volumes:
files:
elastic:
cockroach_db:
redpanda:
dev/tool/package.json
+3 -2
View File
@@ -20,7 +20,7 @@
"docker:abuild": "docker build -t hardcoreeng/tool . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool",
"docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging",
"docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool",
"run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4700 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js",
"run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4700 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:9092' node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js",
"run-local-cr": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 $TOOL_OPT ./bundle/bundle.js",
"run-local-brk": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js",
"run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js",
@@ -171,6 +171,7 @@
"bufferutil": "^4.0.8",
"utf-8-validate": "^6.0.4",
"msgpackr": "^1.11.2",
"msgpackr-extract": "^3.0.3"
"msgpackr-extract": "^3.0.3",
"@hcengineering/kafka": "^0.6.0"
}
}
dev/tool/src/clean.ts
-12
View File
@@ -26,7 +26,6 @@ import core, {
type CollaborativeDoc,
type Client as CoreClient,
DOMAIN_BENCHMARK,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MIGRATION,
DOMAIN_MODEL,
DOMAIN_STATUS,
@@ -1218,17 +1217,6 @@ export async function removeDuplicateIds (
// await updateArrRefs(txop, newId, doc)
// // update docIndexState
// const docIndexState = await client.findOne(core.class.DocIndexState, { doc: doc._id })
// if (docIndexState !== undefined) {
// const { _id, space, modifiedBy, modifiedOn, createdBy, createdOn, _class, ...data } = docIndexState
// await txop.createDoc(docIndexState._class, docIndexState.space, {
// ...data,
// removed: false
// })
// await txop.update(docIndexState, { removed: true, needIndex: true })
// }
// if (domain !== DOMAIN_MODEL) {
// const raw = await db.collection(domain).findOne({ _id: doc._id })
// await db.collection(domain).insertOne({
dev/tool/src/elastic.ts
-13
View File
@@ -14,8 +14,6 @@
//
import { Client as ElasticClient } from '@elastic/elasticsearch'
import core, { DOMAIN_DOC_INDEX_STATE } from '@hcengineering/core'
import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo'
import { type StorageAdapter } from '@hcengineering/server-core'
export async function rebuildElastic (
@@ -24,17 +22,6 @@ export async function rebuildElastic (
storageAdapter: StorageAdapter,
elasticUrl: string
): Promise<void> {
const client = getMongoClient(mongoUrl)
try {
const _client = await client.getClient()
const db = getWorkspaceMongoDB(_client, dataId)
await db
.collection(DOMAIN_DOC_INDEX_STATE)
.updateMany({ _class: core.class.DocIndexState }, { $set: { elastic: false } })
} finally {
client.close()
}
await dropElastic(elasticUrl, dataId)
}
dev/tool/src/fulltext.ts
-33
View File
@@ -1,33 +0,0 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type MeasureContext } from '@hcengineering/core'
export async function reindexWorkspace (ctx: MeasureContext, fulltextUrl: string, token: string): Promise<void> {
try {
const res = await fetch(fulltextUrl + '/api/v1/reindex', {
method: 'PUT',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ token })
})
if (!res.ok) {
throw new Error(`HTTP Error ${res.status} ${res.statusText}`)
}
} catch (err: any) {
ctx.error('failed to reset index', { err })
}
}
dev/tool/src/index.ts
+41 -3
View File
@@ -47,6 +47,7 @@ import {
import serverToken, { generateToken } from '@hcengineering/server-token'
import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
import { getPlatformQueue } from '@hcengineering/kafka'
import { buildStorageFromConfig, createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import { program, type Command } from 'commander'
import { updateField } from './workspace'
@@ -81,11 +82,15 @@ import {
createPostgresTxAdapter,
shutdownPostgres
} from '@hcengineering/postgres'
import type { StorageAdapter } from '@hcengineering/server-core'
import {
QueueTopic,
workspaceEvents,
type QueueWorkspaceMessage,
type StorageAdapter
} from '@hcengineering/server-core'
import { getAccountDBUrl, getMongoDBUrl } from './__start'
// import { fillGithubUsers, fixAccountEmails, renameAccount } from './account'
import { changeConfiguration } from './configuration'
import { reindexWorkspace } from './fulltext'
import { moveAccountDbFromMongoToPG } from './db'
import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
@@ -363,6 +368,13 @@ export function devTool (
progress: 100
})
const wsProducer = getPlatformQueue('tool', cmd.region).createProducer<QueueWorkspaceMessage>(
toolCtx,
QueueTopic.Workspace
)
await wsProducer.send(res.workspaceUuid, [workspaceEvents.created()])
await wsProducer.close()
console.log('create-workspace done')
})
})
@@ -442,6 +454,12 @@ export function devTool (
})
console.log(metricsToString(measureCtx.metrics, 'upgrade', 60))
const wsProducer = getPlatformQueue('tool', info.region).createProducer<QueueWorkspaceMessage>(
toolCtx,
QueueTopic.Workspace
)
await wsProducer.send(info.uuid, [workspaceEvents.upgraded()])
await wsProducer.close()
console.log('upgrade-workspace done')
})
})
@@ -1121,6 +1139,12 @@ export function devTool (
storageAdapter: workspaceStorage,
historyFile: cmd.historyFile
})
const wsProducer = getPlatformQueue('tool', ws.region).createProducer<QueueWorkspaceMessage>(
toolCtx,
QueueTopic.Workspace
)
await wsProducer.send(ws.uuid, [workspaceEvents.fullReindex()])
await wsProducer.close()
await workspaceStorage?.close()
})
}
@@ -2063,7 +2087,12 @@ export function devTool (
}
console.log('reindex workspace', workspace)
await reindexWorkspace(toolCtx, fulltextUrl, getToolToken(ws?.uuid))
const wsProducer = getPlatformQueue('tool', ws.region).createProducer<QueueWorkspaceMessage>(
toolCtx,
QueueTopic.Workspace
)
await wsProducer.send(ws.uuid, [workspaceEvents.fullReindex()])
await wsProducer.close()
console.log('done', workspace)
})
})
@@ -2367,6 +2396,15 @@ export function devTool (
// })
// })
program
.command('queue-init-topics')
.description('create required kafka topics')
.option('-tx <tx>', 'Number of TX partitions', '5')
.action(async (cmd: { tx: string }) => {
const queue = getPlatformQueue('tool')
await queue.createTopics(parseInt(cmd.tx ?? '1'))
})
extendProgram?.(program)
program.parse(process.argv)
models/all/package.json
-1
View File
@@ -112,7 +112,6 @@
"@hcengineering/model-server-ai-bot": "^0.6.0",
"@hcengineering/model-ai-bot": "^0.6.0",
"@hcengineering/model-recorder": "^0.6.0",
"@hcengineering/model-server-fulltext": "^0.6.0",
"@hcengineering/model-test-management": "^0.6.0",
"@hcengineering/model-survey": "^0.6.0",
"@hcengineering/model-presence": "^0.6.0",
models/all/src/index.ts
-2
View File
@@ -110,7 +110,6 @@ import {
serverDocumentsId,
createModel as serverDocumentsModel
} from '@hcengineering/model-server-controlled-documents'
import { serverFulltextId, createModel as serverFulltextModel } from '@hcengineering/model-server-fulltext'
import { surveyId, createModel as surveyModel } from '@hcengineering/model-survey'
import { presenceId, createModel as presenceModel } from '@hcengineering/model-presence'
import processes, { processId, createModel as processModel } from '@hcengineering/model-process'
@@ -493,7 +492,6 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[
[serverTrainingModel, serverTrainingId],
[serverDocumentsModel, serverDocumentsId],
[serverAiBotModel, serverAiBotId],
[serverFulltextModel, serverFulltextId],
[serverProcessModel, serverProcessId]
]
models/controlled-documents/src/migration.ts
-10
View File
@@ -26,8 +26,6 @@ import {
type Class,
type Data,
type Doc,
type DocIndexState,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_SEQUENCE,
DOMAIN_TX,
generateId,
@@ -446,14 +444,6 @@ async function migrateDocumentMetaInternalCode (client: MigrationClient): Promis
}
await client.bulk(DOMAIN_DOCUMENTS, operations)
await client.update<DocIndexState>(
DOMAIN_DOC_INDEX_STATE,
{
_id: { $in: Array.from(updatedIds) as any },
objectClass: documents.class.DocumentMeta
},
{ $set: { needIndex: true } }
)
}
export const documentsOperation: MigrateOperation = {
models/core/src/core.ts
-18
View File
@@ -26,11 +26,9 @@ import {
type Configuration,
type ConfigurationElement,
type Doc,
type DocIndexState,
type Domain,
DOMAIN_BLOB,
DOMAIN_CONFIGURATION,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MIGRATION,
DOMAIN_MODEL,
DOMAIN_RELATION,
@@ -339,22 +337,6 @@ export class TPluginConfiguration extends TDoc implements PluginConfiguration {
beta!: boolean
}
@Model(core.class.DocIndexState, core.class.Doc, DOMAIN_DOC_INDEX_STATE)
export class TDocIndexState extends TDoc implements DocIndexState {
@Prop(TypeRef(core.class.Class), core.string.Class)
@Index(IndexKind.Indexed)
@Hidden()
objectClass!: Ref<Class<Doc>>
@Prop(TypeBoolean(), getEmbeddedLabel('Removed'))
@Hidden()
removed!: boolean
@Prop(TypeBoolean(), getEmbeddedLabel('NeedIndexing'))
@Hidden()
needIndex!: boolean
}
@Model(core.class.FullTextSearchContext, core.class.Doc, DOMAIN_MODEL)
export class TFullTextSearchContext extends TDoc implements FullTextSearchContext {
toClass!: Ref<Class<Doc<Space>>>
models/core/src/index.ts
-27
View File
@@ -17,7 +17,6 @@ import {
DOMAIN_BENCHMARK,
DOMAIN_BLOB,
DOMAIN_CONFIGURATION,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MIGRATION,
DOMAIN_SPACE,
DOMAIN_STATUS,
@@ -38,7 +37,6 @@ import {
TConfiguration,
TConfigurationElement,
TDoc,
TDocIndexState,
TDomainIndexConfiguration,
TEnum,
TEnumOf,
@@ -149,7 +147,6 @@ export function createModel (builder: Builder): void {
TEnum,
TTypeAny,
TTypeRelatedDocument,
TDocIndexState,
TFullTextSearchContext,
TConfiguration,
TConfigurationElement,
@@ -262,30 +259,6 @@ export function createModel (builder: Builder): void {
]
})
builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
domain: DOMAIN_DOC_INDEX_STATE,
indexes: [
{
keys: { needIndex: 1, objectClass: 1 }
}
],
disabled: [
{ attachedToClass: 1 },
{ stages: 1 },
{ space: 1 },
{ _class: 1 },
{ needIndex: 1 },
{ objectClass: 1 },
{ _class: 1 },
{ attachedTo: 1 },
{ modifiedBy: 1 },
{ modifiedOn: 1 },
{ createdBy: 1 },
{ createdBy: -1 },
{ createdOn: -1 }
]
})
builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
toClass: core.class.Space,
childProcessingAllowed: false
models/core/src/migration.ts
+1 -1
View File
@@ -306,7 +306,7 @@ export function getAccountsFromTxes (accTxes: TxCUD<Doc>[]): any {
return Object.values(byAccounts)
.map((txes) => TxProcessor.buildDoc2Doc(txes))
.filter((it) => it !== undefined)
.filter((it) => it != null)
}
export async function getSocialKeyByOldAccount (client: MigrationClient): Promise<Record<string, string>> {
models/server-fulltext/package.json
-37
View File
@@ -1,37 +0,0 @@
{
"name": "@hcengineering/model-server-fulltext",
"version": "0.6.0",
"main": "lib/index.js",
"svelte": "src/index.ts",
"types": "types/index.d.ts",
"author": "Anticrm Platform Contributors",
"template": "@hcengineering/model-package",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"format": "format src",
"_phase:build": "compile transpile src",
"_phase:format": "format src",
"_phase:validate": "compile validate"
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3"
},
"dependencies": {
"@hcengineering/core": "^0.6.32",
"@hcengineering/model": "^0.6.11",
"@hcengineering/platform": "^0.6.11",
"@hcengineering/server-fulltext": "^0.6.0",
"@hcengineering/server-core": "^0.6.1"
}
}
models/server-fulltext/src/index.ts
-26
View File
@@ -1,26 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type Builder } from '@hcengineering/model'
import core from '@hcengineering/core'
import serverCore from '@hcengineering/server-core'
import serverFulltext from '@hcengineering/server-fulltext'
export { serverFulltextId } from '@hcengineering/server-fulltext'
export function createModel (builder: Builder): void {
builder.createDoc(serverCore.class.Trigger, core.space.Model, {
trigger: serverFulltext.trigger.OnChange
})
}
packages/core/src/classes.ts
-13
View File
@@ -421,8 +421,6 @@ export interface TransientConfiguration extends Class<Doc> {
*/
export const DOMAIN_BLOB = 'blob' as Domain
export const DOMAIN_DOC_INDEX_STATE = 'doc-index-state' as Domain
/**
* @public
*/
@@ -581,17 +579,6 @@ export function versionToString (version: Version | Data<Version>): string {
return `${version?.major}.${version?.minor}.${version?.patch}`
}
/**
* @public
*
* Define status for full text indexing
*/
export interface DocIndexState extends Doc {
objectClass: Ref<Class<Doc>>
needIndex: boolean
removed: boolean
}
/**
* @public
*/
packages/core/src/component.ts
-2
View File
@@ -28,7 +28,6 @@ import type {
Configuration,
ConfigurationElement,
Doc,
DocIndexState,
DomainIndexConfiguration,
Enum,
EnumOf,
@@ -149,7 +148,6 @@ export default plugin(coreId, {
PluginConfiguration: '' as Ref<Class<PluginConfiguration>>,
UserStatus: '' as Ref<Class<UserStatus>>,
TypeRelatedDocument: '' as Ref<Class<Type<RelatedDocument>>>,
DocIndexState: '' as Ref<Class<DocIndexState>>,
DomainIndexConfiguration: '' as Ref<Class<DomainIndexConfiguration>>,
Configuration: '' as Ref<Class<Configuration>>,
packages/core/src/measurements/metrics.ts
+10 -7
View File
@@ -198,6 +198,7 @@ function printMetricsChildren (params: Record<string, Metrics>, offset: number,
if (Object.keys(params).length > 0) {
r += '\n' + toLen('', ' ', offset)
r += Object.entries(params)
.filter((it) => it[1].value > 0.1)
.map(([k, vv]) => toString(k, vv, offset, length))
.join('\n' + toLen('', ' ', offset))
}
@@ -211,11 +212,13 @@ function printMetricsParams (
): string {
let r = ''
const joinP = (key: string, data: Record<string, MetricsData>): string[] => {
return Object.entries(data).map(([k, vv]) =>
`${toLen('', ' ', offset)}${toLen(key + '=' + k, '-', length - offset)}: avg ${
vv.value / (vv.operations > 0 ? vv.operations : 1)
} total: ${vv.value} ops: ${vv.operations}`.trim()
)
return Object.entries(data)
.filter((it) => it[1].value >= 0.1)
.map(([k, vv]) =>
`${toLen('', ' ', offset)}${toLen(key + '=' + k, '-', length - offset)}: avg ${
Math.round((vv.value / (vv.operations > 0 ? vv.operations : 1)) * 100) / 100
} total: ${Math.round(vv.value * 100) / 100} ops: ${vv.operations}`.trim()
)
}
const joinParams = Object.entries(params).reduce<string[]>((p, c) => [...p, ...joinP(c[0], c[1])], [])
if (Object.keys(joinParams).length > 0) {
@@ -227,8 +230,8 @@ function printMetricsParams (
function toString (name: string, m: Metrics, offset: number, length: number): string {
let r = `${toLen('', ' ', offset)}${toLen(name, '-', length - offset)}: avg ${
m.value / (m.operations > 0 ? m.operations : 1)
} total: ${m.value} ops: ${m.operations}`.trim()
Math.round((m.value / (m.operations > 0 ? m.operations : 1)) * 100) / 100
} total: ${Math.round(m.value * 100) / 100} ops: ${m.operations}`.trim()
r += printMetricsParams(m.params, offset + 4, length)
r += printMetricsChildren(m.measurements, offset + 4, length)
return r
packages/core/src/server.ts
+1 -2
View File
@@ -13,7 +13,7 @@
// limitations under the License.
//
import type { Account, AccountUuid, Doc, DocIndexState, Domain, PersonId, Ref } from './classes'
import type { Account, AccountUuid, Doc, Domain, PersonId, Ref } from './classes'
import { MeasureContext } from './measurements'
import { DocumentQuery, FindOptions } from './storage'
import type { DocumentUpdate, Tx } from './tx'
@@ -52,7 +52,6 @@ export interface SessionData {
workspace: WorkspaceIds
branding: Branding | null
socialStringsToUsers: Map<PersonId, AccountUuid>
fulltextUpdates?: Map<Ref<DocIndexState>, DocIndexState>
asyncRequests?: (() => Promise<void>)[]
}
packages/core/src/tx.ts
+9 -3
View File
@@ -15,7 +15,6 @@
import type { KeysByType } from 'simplytyped'
import type {
PersonId,
Arr,
AttachedDoc,
Class,
@@ -23,6 +22,7 @@ import type {
Doc,
Domain,
Mixin,
PersonId,
PropertyType,
Ref,
Space,
@@ -399,10 +399,16 @@ export abstract class TxProcessor implements WithTx {
return rawDoc
}
static buildDoc2Doc<D extends Doc>(txes: Tx[]): D | undefined {
static buildDoc2Doc<D extends Doc>(txes: Tx[]): D | undefined | null {
let doc: Doc
const deleteTx = txes.find((tx) => tx._class === core.class.TxRemoveDoc)
if (deleteTx !== undefined) {
return null
}
const createTx = txes.find((tx) => tx._class === core.class.TxCreateDoc)
if (createTx === undefined) return
if (createTx === undefined) {
return
}
doc = TxProcessor.createDoc2Doc(createTx as TxCreateDoc<Doc>)
for (const tx of txes) {
if (tx._class === core.class.TxUpdateDoc) {
packages/core/src/utils.ts
-2
View File
@@ -27,7 +27,6 @@ import {
Doc,
DocData,
DOMAIN_BLOB,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MODEL,
DOMAIN_TRANSIENT,
FullTextSearchContext,
@@ -673,7 +672,6 @@ export function isClassIndexable (
}
if (
domain === DOMAIN_DOC_INDEX_STATE ||
domain === DOMAIN_TX ||
domain === DOMAIN_MODEL ||
domain === DOMAIN_BLOB ||
plugins/board-resources/src/components/editor/CardActions.svelte
+1 -1
View File
@@ -42,7 +42,7 @@
const txes = await client.findAll(core.class.Tx, { objectId: value._id })
while (txes.length) {
const doc = TxProcessor.buildDoc2Doc<Card>(txes)
if (doc === undefined) {
if (doc == null) {
break
}
targetState = store.byId.get(doc.status)
plugins/view-resources/src/inference.ts
-33
View File
@@ -1,33 +0,0 @@
import core, { type Class, type Client, type Doc, type DocIndexState, type Ref } from '@hcengineering/core'
import { writable } from 'svelte/store'
export interface InferenceFocus {
objectId: Ref<Doc>
objectClass: Ref<Class<Doc>>
state: DocIndexState
}
/**
* @public
*/
export const inferenceFocusStore = writable<InferenceFocus | undefined>(undefined)
/**
* @public
*/
export async function updateFocus (client: Client, doc?: { _id: Ref<Doc>, _class: Ref<Class<Doc>> }): Promise<void> {
if (doc === undefined) {
inferenceFocusStore.set(undefined)
return
}
const state = await client.findOne(core.class.DocIndexState, {
_id: doc._id as Ref<DocIndexState>,
objectClass: doc._class
})
if (state !== undefined) {
inferenceFocusStore.update(() => {
return { objectId: doc._id, objectClass: doc._class, state }
})
}
}
plugins/workbench-resources/src/components/ServerManagerGeneral.svelte
+8 -1
View File
@@ -35,7 +35,14 @@
let profiling = false
async function fetchStats (time: number): Promise<void> {
await fetch(endpoint + `/api/v1/profiling?token=${token}`, {})
await fetch(endpoint + '/api/v1/profiling', {
method: 'GET',
keepalive: true,
headers: {
'Content-Type': 'application/json',
Authorization: 'Bearer ' + token
}
})
.then(async (json) => {
data = await json.json()
profiling = data?.profiling ?? false
pods/fulltext/package.json
+2 -1
View File
@@ -75,6 +75,7 @@
"@hcengineering/server-client": "^0.6.0",
"@hcengineering/server-storage": "^0.6.0",
"@hcengineering/postgres": "^0.6.0",
"@hcengineering/mongo": "^0.6.1"
"@hcengineering/mongo": "^0.6.1",
"@hcengineering/kafka": "^0.6.0"
}
}
pods/fulltext/src/index.ts
+4
View File
@@ -24,6 +24,7 @@ import { createRekoniAdapter, type FulltextDBConfiguration } from '@hcengineerin
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import { readFileSync } from 'fs'
import { startIndexer } from './server'
import { getPlatformQueue } from '@hcengineering/kafka'
const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[]
@@ -103,7 +104,10 @@ if (accountsUrl === undefined) {
const storageConfig: StorageConfiguration = storageConfigFromEnv()
const externalStorage = buildStorageFromConfig(storageConfig)
const queue = getPlatformQueue('fulltext')
const onClose = startIndexer(metricsContext, {
queue,
model,
config,
externalStorage,
pods/fulltext/src/server.ts
+152 -71
View File
@@ -4,22 +4,24 @@ import type {
Class,
Doc,
DocumentQuery,
Domain,
IndexingUpdateEvent,
MeasureContext,
Ref,
SearchOptions,
SearchQuery,
Tx,
TxCUD,
TxWorkspaceEvent,
WorkspaceUuid,
WorkspaceIds
WorkspaceIds,
WorkspaceUuid
} from '@hcengineering/core'
import core, {
DOMAIN_DOC_INDEX_STATE,
generateId,
Hierarchy,
ModelDb,
systemAccountUuid,
TxProcessor,
WorkspaceEvent
} from '@hcengineering/core'
import {
@@ -44,15 +46,22 @@ import {
setDBExtraOptions,
shutdownPostgres
} from '@hcengineering/postgres'
import serverClientPlugin, { getTransactorEndpoint, getAccountClient } from '@hcengineering/server-client'
import serverClientPlugin, { getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client'
import serverCore, {
createContentAdapter,
createPipeline,
QueueTopic,
QueueWorkspaceEvent,
workspaceEvents,
type ConsumerControl,
type ContentTextAdapter,
type FullTextAdapter,
type MiddlewareCreator,
type Pipeline,
type PipelineContext,
type PlatformQueue,
type QueueWorkspaceMessage,
type QueueWorkspaceReindexMessage,
type StorageAdapter
} from '@hcengineering/server-core'
import { FullTextIndexPipeline, searchFulltext, type FulltextDBConfiguration } from '@hcengineering/server-indexer'
@@ -66,12 +75,31 @@ import {
setAdapterSecurity,
sharedPipelineContextVars
} from '@hcengineering/server-pipeline'
import serverToken, { decodeToken, generateToken, type Token } from '@hcengineering/server-token'
import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
import cors from '@koa/cors'
import Koa from 'koa'
import bodyParser from 'koa-bodyparser'
import Router from 'koa-router'
function fulltextModelFilter (h: Hierarchy, model: Tx[]): Tx[] {
const allowedClasess: Ref<Class<Doc>>[] = [
core.class.Class,
core.class.Attribute,
core.class.Mixin,
core.class.Type,
core.class.Status,
core.class.Permission,
core.class.Space,
core.class.Tx,
core.class.FullTextSearchContext
]
return model.filter(
(it) =>
TxProcessor.isExtendsCUD(it._class) &&
allowedClasess.some((cl) => h.isDerived((it as TxCUD<Doc>).objectClass, cl))
)
}
class WorkspaceIndexer {
fulltext!: FullTextIndexPipeline
pipeline!: Pipeline
@@ -100,7 +128,7 @@ class WorkspaceIndexer {
ContextNameMiddleware.create,
DomainFindMiddleware.create,
DBAdapterInitMiddleware.create,
ModelMiddleware.create(model),
ModelMiddleware.create(model, fulltextModelFilter), // TODO: Add filtration of only class structure and FullTextSearchContext
DBAdapterMiddleware.create(dbConf)
]
@@ -153,40 +181,37 @@ class WorkspaceIndexer {
}
// Send tx to pipeline
// TODO: Fix me
void fetch(transactorEndpoint + `/api/v1/broadcast?token=${token}&workspace=${workspace.uuid}`, {
void fetch(transactorEndpoint + `/api/v1/broadcast?workspace=${workspace.uuid}`, {
method: 'PUT',
keepalive: true,
headers: {
Authorization: `Bearer ${token}`
},
body: JSON.stringify(tx)
})
},
async () => {
const helper = result.pipeline.context.adapterManager?.domainHelper
if (helper !== undefined) {
const dhelper = result.pipeline.context.adapterManager?.getAdapter(DOMAIN_DOC_INDEX_STATE, true).helper?.()
if (dhelper !== undefined) {
// Force creation of indexes for search domain
await helper.checkDomain(ctx, DOMAIN_DOC_INDEX_STATE, 10000, dhelper)
}
}
}
)
await result.fulltext.startIndexing(() => {
result.lastUpdate = Date.now()
})
return result
}
async reindex (onlyDrop: boolean): Promise<void> {
await this.fulltext.cancel()
await this.fulltext.clearIndex(onlyDrop)
if (!onlyDrop) {
await this.fulltext.startIndexing(() => {
this.lastUpdate = Date.now()
})
}
async reindex (
ctx: MeasureContext,
domain: Domain,
classes: Ref<Class<Doc>>[],
control?: ConsumerControl
): Promise<void> {
await this.fulltext.reindex(ctx, domain, classes, control)
}
async dropWorkspace (): Promise<void> {
await this.fulltext.dropWorkspace()
}
async getIndexClassess (): Promise<{ domain: Domain, classes: Ref<Class<Doc>>[] }[]> {
return await this.fulltext.getIndexClassess()
}
async close (): Promise<void> {
await this.fulltext.cancel()
await this.pipeline.close()
}
}
@@ -229,6 +254,7 @@ process.on('exit', () => {
export async function startIndexer (
ctx: MeasureContext,
opt: {
queue: PlatformQueue
model: Tx[]
dbURL: string
config: FulltextDBConfiguration
@@ -292,16 +318,89 @@ export async function startIndexer (
}
}, closeTimeout) // Every 5 minutes we should close unused indexes.
const workspaceProducer = opt.queue.createProducer<QueueWorkspaceMessage>(ctx, QueueTopic.Workspace)
const workspaceConsumer = opt.queue.createConsumer<QueueWorkspaceMessage>(
ctx,
QueueTopic.Workspace,
opt.queue.getClientId(),
async (msg, control) => {
for (const m of msg) {
ctx.info('workspace message', { message: m })
const ws = m.id as WorkspaceUuid
for (const mm of m.value) {
if (
mm.type === QueueWorkspaceEvent.Created ||
mm.type === QueueWorkspaceEvent.Restored ||
mm.type === QueueWorkspaceEvent.FullReindex
) {
const indexer = await getIndexer(ctx, ws, generateToken(systemAccountUuid, ws), true)
if (indexer !== undefined) {
await indexer.dropWorkspace() // TODO: Add heartbeat
const classes = await indexer.getIndexClassess()
await workspaceProducer.send(
ws,
classes.map((it) => workspaceEvents.reindex(it.domain, it.classes))
)
}
} else if (
mm.type === QueueWorkspaceEvent.Deleted ||
mm.type === QueueWorkspaceEvent.Archived ||
mm.type === QueueWorkspaceEvent.ClearIndex
) {
const token = generateToken(systemAccountUuid, ws)
const accountClient = getAccountClient(token)
const workspaceInfo = await accountClient.getWorkspaceInfo(false)
if (workspaceInfo !== undefined) {
if (workspaceInfo.dataId != null) {
await fulltextAdapter.clean(ctx, workspaceInfo.dataId as unknown as WorkspaceUuid)
}
await fulltextAdapter.clean(ctx, workspaceInfo.uuid)
}
} else if (mm.type === QueueWorkspaceEvent.Reindex) {
const indexer = await getIndexer(ctx, ws, generateToken(systemAccountUuid, ws), true)
const mmd = mm as QueueWorkspaceReindexMessage
await indexer?.reindex(ctx, mmd.domain, mmd.classes, control)
}
}
}
}
)
let txInformer: any
let txMessages: number = 0
const txConsumer = opt.queue.createConsumer<TxCUD<Doc>>(
ctx,
QueueTopic.Tx,
opt.queue.getClientId(),
async (msg, control) => {
clearTimeout(txInformer)
txInformer = setTimeout(() => {
ctx.info('tx message', { count: txMessages })
txMessages = 0
}, 5000)
txMessages += msg.length
for (const m of msg) {
const ws = m.id as WorkspaceUuid
const indexer = await getIndexer(ctx, ws, generateToken(systemAccountUuid, ws), true)
await indexer?.fulltext.processDocuments(ctx, m.value, control)
}
}
)
async function getIndexer (
ctx: MeasureContext,
workspace: WorkspaceUuid,
token: string,
token: string | undefined,
create: boolean = false
): Promise<WorkspaceIndexer | undefined> {
const accountClient = getAccountClient(token)
const workspaceInfo = await accountClient.getWorkspaceInfo(false)
let idx = indexers.get(workspace)
if (idx === undefined && create) {
const accountClient = getAccountClient(token)
const workspaceInfo = await accountClient.getWorkspaceInfo(false)
if (workspaceInfo === undefined) {
ctx.error('Workspace not available for token')
return
@@ -339,7 +438,8 @@ export async function startIndexer (
router.put('/api/v1/search', async (req, res) => {
try {
const request = req.request.body as Search
const decoded = decodeToken(request.token) // Just to be safe
const token = request.token ?? req.headers.authorization?.split(' ')[1]
const decoded = decodeToken(token) // Just to be safe
ctx.info('search', { classes: request._classes, query: request.query, workspace: decoded.workspace })
await ctx.with('search', {}, async (ctx) => {
@@ -348,8 +448,6 @@ export async function startIndexer (
)
req.body = docs
})
void triggerIndexer(decoded, request.token)
} catch (err: any) {
Analytics.handleError(err)
console.error(err)
@@ -361,7 +459,8 @@ export async function startIndexer (
router.put('/api/v1/full-text-search', async (req, res) => {
try {
const request = req.request.body as FulltextSearch
const decoded = decodeToken(request.token) // Just to be safe
const token = request.token ?? req.headers.authorization?.split(' ')[1]
const decoded = decodeToken(token) // Just to be safe
ctx.info('fulltext-search', { ...request.query, workspace: decoded.workspace })
await ctx.with('full-text-search', {}, async (ctx) => {
const result = await ctx.with('searchFulltext', {}, (ctx) =>
@@ -369,24 +468,6 @@ export async function startIndexer (
)
req.body = result
})
void triggerIndexer(decoded, request.token)
} catch (err: any) {
Analytics.handleError(err)
console.error(err)
req.res.writeHead(404, {})
req.res.end()
}
})
router.put('/api/v1/warmup', async (req, res) => {
try {
const request = req.request.body as IndexDocuments
const decoded = decodeToken(request.token) // Just to be safe
req.body = {}
ctx.info('warm-up', { workspace: decoded.workspace })
void triggerIndexer(decoded, request.token, true)
} catch (err: any) {
Analytics.handleError(err)
console.error(err)
@@ -398,7 +479,8 @@ export async function startIndexer (
router.put('/api/v1/close', async (req, res) => {
try {
const request = req.request.body as IndexDocuments
const decoded = decodeToken(request.token) // Just to be safe
const token = request.token ?? req.headers.authorization?.split(' ')[1]
const decoded = decodeToken(token) // Just to be safe
req.body = {}
ctx.info('close', { workspace: decoded.workspace })
@@ -422,12 +504,14 @@ export async function startIndexer (
router.put('/api/v1/index-documents', async (req, res) => {
try {
const request = req.request.body as IndexDocuments
const decoded = decodeToken(request.token) // Just to be safe
const token = request.token ?? req.headers.authorization?.split(' ')[1]
const decoded = decodeToken(token) // Just to be safe
const indexer = await getIndexer(ctx, decoded.workspace, request.token)
const indexer = await getIndexer(ctx, decoded.workspace, token)
if (indexer !== undefined) {
indexer.lastUpdate = Date.now()
await ctx.with('index-documents', {}, (ctx) => indexer.fulltext.indexDocuments(ctx, request.requests))
// TODO: Fixme
// await ctx.with('index-documents', {}, (ctx) => indexer.fulltext.indexDocuments(ctx, request.requests))
}
req.body = {}
} catch (err: any) {
@@ -441,14 +525,19 @@ export async function startIndexer (
router.put('/api/v1/reindex', async (req, res) => {
try {
const request = req.request.body as Reindex
const decoded = decodeToken(request.token) // Just to be safe
const token = request.token ?? req.headers.authorization?.split(' ')[1]
const decoded = decodeToken(token) // Just to be safe
req.body = {}
ctx.info('reindex', { workspace: decoded.workspace })
const indexer = await getIndexer(ctx, decoded.workspace, request.token, true)
const indexer = await getIndexer(ctx, decoded.workspace, token, true)
if (indexer !== undefined) {
indexer.lastUpdate = Date.now()
await indexer.reindex(request?.onlyDrop ?? false)
if (request?.onlyDrop ?? false) {
await workspaceProducer.send(decoded.workspace, [workspaceEvents.clearIndex()])
} else {
await workspaceProducer.send(decoded.workspace, [workspaceEvents.fullReindex()])
}
}
} catch (err: any) {
Analytics.handleError(err)
@@ -465,19 +554,11 @@ export async function startIndexer (
})
const close = (): void => {
void txConsumer.shutdown()
void workspaceConsumer.shutdown()
clearInterval(shutdownInterval)
server.close()
}
return close
async function triggerIndexer (decoded: Token, token: string, trigger: boolean = false): Promise<void> {
const indexer = await getIndexer(ctx, decoded.workspace, token, trigger)
if (indexer !== undefined) {
indexer.lastUpdate = Date.now()
if (trigger) {
indexer.fulltext.triggerIndexing()
}
}
}
}
pods/server/package.json
+2 -1
View File
@@ -81,6 +81,7 @@
"msgpackr-extract": "^3.0.3",
"@hcengineering/postgres": "^0.6.0",
"snappy": "^7.2.2",
"@hcengineering/rpc": "^0.6.5"
"@hcengineering/rpc": "^0.6.5",
"@hcengineering/kafka": "^0.6.0"
}
}
pods/server/src/__start.ts
+20 -6
View File
@@ -4,22 +4,24 @@
// Add this to the VERY top of the first file loaded in your app
import { Analytics } from '@hcengineering/analytics'
import { SplitLogger, configureAnalytics } from '@hcengineering/analytics-service'
import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service'
import contactPlugin from '@hcengineering/contact'
import { MeasureMetricsContext, newMetrics, setOperationLogProfiling } from '@hcengineering/core'
import { getPlatformQueue } from '@hcengineering/kafka'
import { setMetadata } from '@hcengineering/platform'
import { setDBExtraOptions } from '@hcengineering/postgres'
import { serverConfigFromEnv } from '@hcengineering/server'
import serverAiBot from '@hcengineering/server-ai-bot'
import serverCalendar from '@hcengineering/server-calendar'
import serverCore, {
initStatisticsContext,
loadBrandingMap,
type ConnectionSocket,
type Session,
type StorageConfiguration,
type UserStatistics,
type Workspace,
type WorkspaceStatistics,
initStatisticsContext,
loadBrandingMap
type WorkspaceStatistics
} from '@hcengineering/server-core'
import serverNotification from '@hcengineering/server-notification'
import { storageConfigFromEnv } from '@hcengineering/server-storage'
@@ -29,7 +31,6 @@ import { startHttpServer } from '@hcengineering/server-ws'
import { join } from 'path'
import { start } from '.'
import { profileStart, profileStop } from './inspector'
import { setDBExtraOptions } from '@hcengineering/postgres'
configureAnalytics(process.env.SENTRY_DSN, {})
Analytics.setTag('application', 'transactor')
@@ -37,6 +38,18 @@ Analytics.setTag('application', 'transactor')
let getUsers: () => WorkspaceStatistics[] = () => {
return []
}
const queueConfig = process.env.QUEUE_CONFIG
if (queueConfig === undefined) {
throw new Error('Please provide queue config')
}
const queue = getPlatformQueue('transactor')
void queue.createTopics(10).catch((err) => {
console.error('Failed to create required topics', err)
})
// Force create server metrics context with proper logging
const metricsContext = initStatisticsContext('transactor', {
getUsers: (): WorkspaceStatistics[] => {
@@ -90,7 +103,8 @@ const { shutdown, sessionManager } = start(metricsContext, config.dbUrl, {
start: profileStart,
stop: profileStop
},
mongoUrl: config.mongoUrl
mongoUrl: config.mongoUrl,
queue
})
const entryToUserStats = (session: Session, socket: ConnectionSocket): UserStatistics => {
pods/server/src/__tests__/backup.spec.ts
-138
View File
@@ -1,138 +0,0 @@
import core, {
DOMAIN_TX,
generateId,
MeasureMetricsContext,
type WorkspaceDataId,
type WorkspaceUuid,
type Doc,
type LowLevelStorage,
type Ref,
type TxCreateDoc
} from '@hcengineering/core'
import builder from '@hcengineering/model-all'
import { wrapPipeline } from '@hcengineering/server-core'
import { getServerPipeline } from '@hcengineering/server-pipeline'
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
const model = builder().getTxes()
// const dbURL = 'postgresql://root@localhost:26257/defaultdb?sslmode=disable'
const dbURL = 'postgresql://postgres:example@localhost:5432'
const STORAGE_CONFIG = 'minio|localhost:9000?accessKey=minioadmin&secretKey=minioadmin&useSSL=false'
// jest.setTimeout(4500000)
describe.skip('test-backup-find', () => {
it('check create/load/clean', async () => {
const toolCtx = new MeasureMetricsContext('-', {})
// We should setup a DB with docuemnts and try to backup them.
const wsIds = { dataId: 'testdb-backup-test' as WorkspaceDataId, uuid: 'test' as WorkspaceUuid, url: 'test' }
const storageConfig = storageConfigFromEnv(STORAGE_CONFIG)
const storageAdapter = buildStorageFromConfig(storageConfig)
const pipeline = await getServerPipeline(toolCtx, model, dbURL, wsIds, storageAdapter, {
disableTriggers: true
})
try {
const client = wrapPipeline(toolCtx, pipeline, wsIds)
const lowLevel = pipeline.context.lowLevelStorage as LowLevelStorage
// We need to create a backup docs if they are missing.
await prepareTxes(lowLevel, toolCtx)
const docs: Doc[] = []
while (true) {
const chunk = await client.loadChunk(DOMAIN_TX, 0)
const part = await client.loadDocs(
DOMAIN_TX,
chunk.docs.map((doc) => doc.id as Ref<Doc>)
)
docs.push(...part)
if (chunk.finished) {
break
}
}
await client.closeChunk(0)
expect(docs.length).toBeGreaterThan(459)
await client.clean(
DOMAIN_TX,
docs.map((doc) => doc._id)
)
const findDocs = await client.findAll(core.class.Tx, {})
expect(findDocs.length).toBe(0)
//
} finally {
await pipeline.close()
await storageAdapter.close()
}
})
it('check traverse', async () => {
const toolCtx = new MeasureMetricsContext('-', {})
// We should setup a DB with docuemnts and try to backup them.
const wsIds = { dataId: 'testdb-backup-test' as WorkspaceDataId, uuid: 'test' as WorkspaceUuid, url: 'test' }
const storageConfig = storageConfigFromEnv(STORAGE_CONFIG)
const storageAdapter = buildStorageFromConfig(storageConfig)
const pipeline = await getServerPipeline(toolCtx, model, dbURL, wsIds, storageAdapter, {
disableTriggers: true
})
try {
const client = wrapPipeline(toolCtx, pipeline, wsIds)
const lowLevel = pipeline.context.lowLevelStorage as LowLevelStorage
// We need to create a backup docs if they are missing.
await prepareTxes(lowLevel, toolCtx, 1500)
const iter = await lowLevel.traverse(DOMAIN_TX, {})
const allDocs: Doc[] = []
while (true) {
const docs = await iter.next(50)
if (docs == null || docs?.length === 0) {
break
}
await client.clean(
DOMAIN_TX,
docs.map((doc) => doc._id)
)
allDocs.push(...docs)
}
expect(allDocs.length).toBeGreaterThan(1449)
const findDocs = await client.findAll(core.class.Tx, {})
expect(findDocs.length).toBe(0)
//
} finally {
await pipeline.close()
await storageAdapter.close()
}
})
})
async function prepareTxes (
lowLevel: LowLevelStorage,
toolCtx: MeasureMetricsContext,
count: number = 500
): Promise<void> {
const docs = await lowLevel.rawFindAll(DOMAIN_TX, {})
if ((docs?.length ?? 0) < count) {
// We need to fill some documents to be pressent
const docs: TxCreateDoc<Doc>[] = []
for (let i = 0; i < count; i++) {
docs.push({
_class: core.class.TxCreateDoc,
_id: generateId(),
space: core.space.Tx,
modifiedBy: core.account.ConfigUser,
modifiedOn: Date.now(),
attributes: {
qwe: generateId()
},
objectClass: core.class.DocIndexState,
objectId: generateId(),
objectSpace: core.space.Workspace
})
}
await lowLevel.upload(toolCtx, DOMAIN_TX, docs)
}
}
pods/server/src/server.ts
+6 -3
View File
@@ -23,7 +23,8 @@ import {
type Session,
type SessionManager,
type StorageConfiguration,
type Workspace
type Workspace,
type PlatformQueue
} from '@hcengineering/server-core'
import { type Token } from '@hcengineering/server-token'
@@ -76,6 +77,7 @@ export function start (
metrics: MeasureContext,
dbUrl: string,
opt: {
queue: PlatformQueue
fulltextUrl: string
storageConfig: StorageConfiguration
port: number
@@ -120,7 +122,7 @@ export function start (
metrics,
dbUrl,
model,
{ ...opt, externalStorage, adapterSecurity: isAdapterSecurity(dbUrl) },
{ ...opt, externalStorage, adapterSecurity: isAdapterSecurity(dbUrl), queue: opt.queue },
{}
)
const sessionFactory = (token: Token, workspace: Workspace, account: Account): Session => {
@@ -136,7 +138,8 @@ export function start (
enableCompression: opt.enableCompression,
accountsUrl: opt.accountsUrl,
externalStorage,
profiling: opt.profiling
profiling: opt.profiling,
queue: opt.queue
})
return {
shutdown: async () => {
pods/workspace/package.json
+2 -1
View File
@@ -56,6 +56,7 @@
"@hcengineering/server-core": "^0.6.1",
"@hcengineering/model-all": "^0.6.0",
"@hcengineering/analytics": "^0.6.0",
"@hcengineering/analytics-service": "^0.6.0"
"@hcengineering/analytics-service": "^0.6.0",
"@hcengineering/kafka": "^0.6.0"
}
}
pods/workspace/src/__start.ts
+4
View File
@@ -4,6 +4,7 @@
import { Analytics } from '@hcengineering/analytics'
import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service'
import { MeasureMetricsContext, newMetrics, type Tx } from '@hcengineering/core'
import { getPlatformQueue } from '@hcengineering/kafka'
import builder, { getModelVersion, migrateOperations } from '@hcengineering/model-all'
import { initStatisticsContext, loadBrandingMap } from '@hcengineering/server-core'
import { serveWorkspaceAccount } from '@hcengineering/workspace-service'
@@ -34,8 +35,11 @@ const metricsContext = initStatisticsContext('workspace', {
const brandingPath = process.env.BRANDING_PATH
const queue = getPlatformQueue('workspace')
serveWorkspaceAccount(
metricsContext,
queue,
getModelVersion(),
txes,
migrateOperations,
qms-tests/.env
+2 -1
View File
@@ -1,2 +1,3 @@
STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
MONGO_URL=mongodb://mongodb:27018?compressors=snappy
MONGO_URL=mongodb://mongodb:27018?compressors=snappy
QUEUE_CONFIG=redpanda:9093
qms-tests/docker-compose.yaml
+53
View File
@@ -11,6 +11,55 @@ services:
ports:
- 27018:27018
restart: unless-stopped
redpanda:
image: docker.redpanda.com/redpandadata/redpanda:v24.3.6
extra_hosts:
- 'huly.local:host-gateway'
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9093,external://0.0.0.0:19093
- --advertise-kafka-addr internal://redpanda:9093,external://localhost:19093
- --pandaproxy-addr internal://0.0.0.0:8084,external://0.0.0.0:18084
- --advertise-pandaproxy-addr internal://redpanda:8084,external://localhost:18084
- --schema-registry-addr internal://0.0.0.0:8083,external://0.0.0.0:18083
- --rpc-addr redpanda:33145
- --advertise-rpc-addr redpanda:33145
- --mode dev-container
- --smp 1
- --default-log-level=info
ports:
- 18083:18083
- 18084:18084
- 19093:19093
- 19645:9644
healthcheck:
test: ['CMD', 'rpk', 'cluster', 'info', '-X', 'user=superuser', '-X', 'pass=secretpassword']
interval: 10s
timeout: 5s
retries: 10
redpanda_console:
image: docker.redpanda.com/redpandadata/console:v2.8.3
extra_hosts:
- 'huly.local:host-gateway'
entrypoint: /bin/sh
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
environment:
CONFIG_FILEPATH: /tmp/config.yml
CONSOLE_CONFIG_FILE: |
kafka:
brokers: ["redpanda:9093"]
schemaRegistry:
enabled: true
urls: ["http://redpanda:8083"]
redpanda:
adminApi:
enabled: true
urls: ["http://redpanda:9644"]
ports:
- 8001:8080
depends_on:
- redpanda
minio:
image: 'minio/minio'
command: server /data --address ":9000"
@@ -51,6 +100,7 @@ services:
volumes:
- ./branding-test.json:/var/cfg/branding-test.json
environment:
- QUEUE_CONFIG=${QUEUE_CONFIG}
- ACCOUNT_PORT=3003
- SERVER_SECRET=secret
- WORKSPACE_LIMIT_PER_USER=100
@@ -69,6 +119,7 @@ services:
volumes:
- ./branding-test.json:/var/cfg/branding.json
environment:
- QUEUE_CONFIG=${QUEUE_CONFIG}
- SERVER_SECRET=secret
- DB_URL=mongodb://mongodb:27018
- MONGO_URL=mongodb://mongodb:27018
@@ -124,6 +175,7 @@ services:
volumes:
- ./branding-test.json:/var/cfg/branding-test.json
environment:
- QUEUE_CONFIG=${QUEUE_CONFIG}
- SERVER_PROVIDER=${SERVER_PROVIDER}
- SERVER_PORT=3334
- SERVER_SECRET=secret
@@ -212,6 +264,7 @@ services:
ports:
- 4710:4710
environment:
- QUEUE_CONFIG=${QUEUE_CONFIG}
- PORT=4710
- SERVER_SECRET=secret
- DB_URL=mongodb://mongodb:27018
qms-tests/prepare.sh
+2
View File
@@ -16,6 +16,8 @@ if [ "x$DO_CLEAN" == 'xtrue' ]; then
docker system prune -a -f
fi
../tests/wait-elastic.sh 9201
# Create user record in accounts
./tool.sh create-account user1 -f John -l Appleseed -p 1234
./tool.sh create-account user2 -f Kainin -l Dirak -p 1234
qms-tests/tool-local.sh
+1
View File
@@ -8,6 +8,7 @@ export ACCOUNT_DB_URL=mongodb://localhost:27017
export ACCOUNTS_URL=http://localhost:3000
export TRANSACTOR_URL=ws://localhost:3333
export SERVER_SECRET=secret
export QUEUE_CONFIG=huly.local:19092
# Restore workspace contents in mongo/elastic
node ../dev/tool/bundle/bundle.js $@
qms-tests/tool.sh
+1
View File
@@ -10,5 +10,6 @@ export ACCOUNT_DB_URL=mongodb://localhost:27018
export DB_URL=mongodb://localhost:27018
export SERVER_SECRET=secret
export STORAGE_CONFIG="minio|localhost:9002?accessKey=minioadmin&secretKey=minioadmin"
export QUEUE_CONFIG=huly.local:19093
node ../dev/tool/bundle/bundle.js $@
rush.json
+5 -15
View File
@@ -1518,21 +1518,6 @@
"projectFolder": "pods/fulltext",
"shouldPublish": false
},
{
"packageName": "@hcengineering/server-fulltext",
"projectFolder": "server-plugins/fulltext",
"shouldPublish": false
},
{
"packageName": "@hcengineering/server-fulltext-resources",
"projectFolder": "server-plugins/fulltext-resources",
"shouldPublish": false
},
{
"packageName": "@hcengineering/model-server-fulltext",
"projectFolder": "models/server-fulltext",
"shouldPublish": false
},
{
"packageName": "@hcengineering/hr",
"projectFolder": "plugins/hr",
@@ -2397,6 +2382,11 @@
"packageName": "@hcengineering/server-process-resources",
"projectFolder": "server-plugins/process-resources",
"shouldPublish": false
},
{
"packageName": "@hcengineering/kafka",
"projectFolder": "server/kafka",
"shouldPublish": false
}
]
}
server-plugins/ai-bot-resources/src/utils.ts
+1 -1
View File
@@ -38,7 +38,7 @@ export async function createAccountRequest (workspace: WorkspaceUuid, ctx: Measu
body: JSON.stringify({})
})
} catch (err) {
ctx.error('Could not send create ai account request', { err })
ctx.warn('Could not send create ai account request', { err })
}
}
server-plugins/fulltext-resources/.eslintrc.js
-7
View File
@@ -1,7 +0,0 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}
server-plugins/fulltext-resources/.npmignore
-4
View File
@@ -1,4 +0,0 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/
server-plugins/fulltext-resources/config/rig.json
-4
View File
@@ -1,4 +0,0 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}
server-plugins/fulltext-resources/src/index.ts
-103
View File
@@ -1,103 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import core, {
DocIndexState,
isClassIndexable,
isFullTextAttribute,
Tx,
TxProcessor,
type AnyAttribute,
type Class,
type Doc,
type FullTextSearchContext,
type Ref,
type TxCUD,
type TxUpdateDoc
} from '@hcengineering/core'
import { TriggerControl } from '@hcengineering/server-core'
const indexingCtx = 'indexing_contexts'
export async function OnChange (txes: Tx[], control: TriggerControl): Promise<Tx[]> {
let contexts = control.contextCache.get(indexingCtx) as Map<Ref<Class<Doc>>, FullTextSearchContext>
if (contexts === undefined) {
contexts = new Map()
control.contextCache.set(indexingCtx, contexts)
}
if (control.ctx.contextData.fulltextUpdates === undefined) {
control.ctx.contextData.fulltextUpdates = new Map()
}
const docsToUpsert = control.ctx.contextData.fulltextUpdates
for (const tx of txes) {
const attrs = new Map<string, AnyAttribute>()
if (TxProcessor.isExtendsCUD(tx._class)) {
const cud = tx as TxCUD<Doc>
if (!isClassIndexable(control.hierarchy, cud.objectClass, contexts)) {
// No need, since no indixable fields or attachments.
continue
}
// We need to check if operation has indexable attributes.
if (cud._class === core.class.TxUpdateDoc) {
let hasFulltext = false
// We could skip updates for a lot of transactions if they have non idexable attributes
const upd = cud as TxUpdateDoc<Doc>
for (const [k] of Object.entries(upd.operations)) {
if (k.startsWith('$') || k === 'modifiedOn') {
// Is operator
// Skip operator changes
} else {
const key = upd.objectClass + '.' + k
const attr = attrs.get(key) ?? control.hierarchy.findAttribute(upd.objectClass, k)
if (attr !== undefined) {
attrs.set(key, attr ?? null)
}
if (attr != null) {
if (isFullTextAttribute(attr)) {
hasFulltext = true
}
}
}
}
if (!hasFulltext) {
// No full text changes, no indexing is required
continue
}
}
docsToUpsert.set(cud.objectId as Ref<DocIndexState>, {
_id: cud.objectId as Ref<DocIndexState>,
_class: core.class.DocIndexState,
objectClass: cud.objectClass,
needIndex: true,
modifiedBy: tx.modifiedBy,
modifiedOn: tx.modifiedOn,
space: cud.objectSpace,
removed: cud._class === core.class.TxRemoveDoc
})
}
}
return []
}
// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
export default async () => ({
trigger: {
OnChange
}
})
server-plugins/fulltext-resources/tsconfig.json
-10
View File
@@ -1,10 +0,0 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"declarationDir": "./types",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}
server-plugins/fulltext/.eslintrc.js
-7
View File
@@ -1,7 +0,0 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}
server-plugins/fulltext/.npmignore
-4
View File
@@ -1,4 +0,0 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/
server-plugins/fulltext/config/rig.json
-4
View File
@@ -1,4 +0,0 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}
server-plugins/fulltext/jest.config.js
-7
View File
@@ -1,7 +0,0 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
roots: ["./src"],
coverageReporters: ["text-summary", "html"]
}
server-plugins/fulltext/package.json
-44
View File
@@ -1,44 +0,0 @@
{
"name": "@hcengineering/server-fulltext",
"version": "0.6.0",
"main": "lib/index.js",
"svelte": "src/index.ts",
"types": "types/index.d.ts",
"files": [
"lib/**/*",
"types/**/*",
"tsconfig.json"
],
"author": "Anticrm Platform Contributors",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"format": "format src",
"test": "jest --passWithNoTests --silent",
"_phase:build": "compile transpile src",
"_phase:test": "jest --passWithNoTests --silent",
"_phase:format": "format src",
"_phase:validate": "compile validate"
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@types/node": "~20.11.16",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5"
},
"dependencies": {
"@hcengineering/platform": "^0.6.11",
"@hcengineering/server-core": "^0.6.1"
}
}
server-plugins/fulltext/src/index.ts
-33
View File
@@ -1,33 +0,0 @@
//
// Copyright © 2020, 2021 Anticrm Platform Contributors.
// Copyright © 2021, 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import type { Plugin, Resource } from '@hcengineering/platform'
import { plugin } from '@hcengineering/platform'
import type { TriggerFunc } from '@hcengineering/server-core'
/**
* @public
*/
export const serverFulltextId = 'server-fulltext' as Plugin
/**
* @public
*/
export default plugin(serverFulltextId, {
trigger: {
OnChange: '' as Resource<TriggerFunc>
}
})
server-plugins/fulltext/tsconfig.json
-10
View File
@@ -1,10 +0,0 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"declarationDir": "./types",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}
server-plugins/recruit-resources/src/index.ts
+2 -2
View File
@@ -125,8 +125,8 @@ async function handleVacancyUpdate (control: TriggerControl, cud: TxCUD<Doc>, re
objectId: updateTx.objectId
})
).filter((it) => it._id !== updateTx._id)
const vacancy = TxProcessor.buildDoc2Doc(txes) as Vacancy
if (vacancy.company != null) {
const vacancy = TxProcessor.buildDoc2Doc<Vacancy>(txes)
if (vacancy?.company != null) {
// We have old value
res.push(
control.txFactory.createTxMixin(
server-plugins/tracker-resources/src/index.ts
+4 -4
View File
@@ -331,7 +331,7 @@ async function doTimeReportUpdate (cud: TxCUD<TimeSpendReport>, control: Trigger
objectId: cud.objectId
})
).filter((it) => it._id !== cud._id)
const doc: TimeSpendReport | undefined = TxProcessor.buildDoc2Doc(logTxes)
const doc = TxProcessor.buildDoc2Doc<TimeSpendReport>(logTxes)
const res: Tx[] = []
const [currentIssue] = await control.findAll(
@@ -340,7 +340,7 @@ async function doTimeReportUpdate (cud: TxCUD<TimeSpendReport>, control: Trigger
{ _id: attachedTo },
{ limit: 1 }
)
if (doc !== undefined) {
if (doc != null) {
res.push(
control.txFactory.createTxUpdateDoc<Issue>(
attachedToClass,
@@ -370,8 +370,8 @@ async function doTimeReportUpdate (cud: TxCUD<TimeSpendReport>, control: Trigger
objectId: cud.objectId
})
).filter((it) => it._id !== cud._id)
const doc: TimeSpendReport | undefined = TxProcessor.buildDoc2Doc(logTxes)
if (doc !== undefined) {
const doc = TxProcessor.buildDoc2Doc<TimeSpendReport>(logTxes)
if (doc != null) {
const [currentIssue] = await control.findAll(
control.ctx,
tracker.class.Issue,
server/account/src/utils.ts
+1 -1
View File
@@ -562,7 +562,7 @@ export async function selectWorkspace (
throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspaceUrl }))
}
if (accountUuid === systemAccountUuid) {
if (accountUuid === systemAccountUuid || extra?.admin === 'true') {
return {
account: accountUuid,
token: generateToken(accountUuid, workspace.uuid, extra),
server/backup/src/backup.ts
+2 -48
View File
@@ -21,7 +21,6 @@ import core, {
Doc,
Domain,
DOMAIN_BLOB,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MODEL,
DOMAIN_MODEL_TX,
DOMAIN_TRANSIENT,
@@ -36,7 +35,6 @@ import core, {
TxProcessor,
type BackupStatus,
type Blob,
type DocIndexState,
type Tx,
type TxCUD,
type WorkspaceIds,
@@ -623,22 +621,6 @@ export async function cloneWorkspace (
// )
}
// function prepareClonedDocuments (docs: Doc[]): Doc[] {
// docs = docs.map((p) => {
// // if full text is skipped, we need to clean stages for indexes.
// if (p._class === core.class.DocIndexState) {
// ;(p as DocIndexState).needIndex = true
// }
// return {
// ...p,
// modifiedOn: Date.now(),
// createdOn: Date.now()
// }
// })
// return docs
// }
// async function cleanDomain (ctx: MeasureContext, connection: CoreClient & BackupClient, domain: Domain): Promise<void> {
// // Load all digest from collection.
// let idx: number | undefined
@@ -1861,6 +1843,7 @@ export async function restore (
// We do not backup elastic anymore
domains.delete('fulltext-blob' as Domain)
domains.delete('doc-index-state' as Domain)
let uploadedMb = 0
let uploaded = 0
@@ -2000,11 +1983,6 @@ export async function restore (
tx.objectSpace = core.space.Workspace
}
}
if (opt.cleanIndexState === true) {
if (d._class === core.class.DocIndexState) {
;(d as DocIndexState).needIndex = true
}
}
}
if (opt.recheck === true) {
@@ -2220,34 +2198,10 @@ export async function restore (
}
}
}
async function performCleanDocIndexState (docsToRemove: Ref<Doc>[]): Promise<void> {
ctx.info('cleanup', { toRemove: docsToRemove.length, workspace: workspaceId, domain: c })
while (docsToRemove.length > 0) {
const part = docsToRemove.splice(0, 1000)
try {
const docs = (await connection.loadDocs(DOMAIN_DOC_INDEX_STATE, part)) as DocIndexState[]
docs.forEach((it) => {
it.needIndex = true
it.removed = true
it.modifiedOn = Date.now()
})
await connection.upload(DOMAIN_DOC_INDEX_STATE, docs)
} catch (err: any) {
ctx.error('failed to clean, will retry', { error: err, workspaceId })
docsToRemove.push(...part)
}
}
}
if (c !== DOMAIN_BLOB) {
// Clean domain documents if not blob
if (docsToRemove.length > 0 && opt.merge !== true) {
if (c === DOMAIN_DOC_INDEX_STATE) {
// We need to create a request to clean fulltext for selected docuemnts.
// We need o clean a FULLTEXT domain as well
await performCleanDocIndexState([...docsToRemove])
} else {
await performCleanOfDomain(docsToRemove, c)
}
await performCleanOfDomain(docsToRemove, c)
}
}
}
server/core/src/adapter.ts
+8 -1
View File
@@ -14,7 +14,6 @@
//
import {
type WorkspaceIds,
type Class,
type Doc,
type DocumentQuery,
@@ -28,6 +27,7 @@ import {
type Ref,
type Tx,
type TxResult,
type WorkspaceIds,
type WorkspaceUuid
} from '@hcengineering/core'
import { type StorageAdapter } from './storage'
@@ -61,6 +61,11 @@ export type DbAdapterHandler = (
count: number,
helper: DomainHelperOperations
) => void
export interface RawFindIterator {
find: (ctx: MeasureContext) => Promise<Doc[]>
close: () => Promise<void>
}
/**
* @public
*/
@@ -87,6 +92,8 @@ export interface DbAdapter extends LowLevelStorage {
// Allow to register a handler to listen for domain operations
on?: (handler: DbAdapterHandler) => void
rawFind: (ctx: MeasureContext, domain: Domain) => RawFindIterator
}
/**
server/core/src/base.ts
+4
View File
@@ -104,6 +104,10 @@ export abstract class BaseMiddleware implements Middleware {
return this.next?.handleBroadcast(ctx) ?? emptyBroadcastResult
}
provideBroadcast (ctx: MeasureContext<SessionData>): Promise<void> {
return this.next?.handleBroadcast(ctx) ?? emptyBroadcastResult
}
protected provideTx (ctx: MeasureContext<SessionData>, tx: Tx[]): Promise<TxMiddlewareResult> {
if (this.next !== undefined) {
return this.next.tx(ctx, tx)
server/core/src/index.ts
+1
View File
@@ -34,3 +34,4 @@ export * from './nullAdapter'
export * from './service'
export * from './stats'
export * from './triggers'
export * from './queue'
server/core/src/mem.ts
+8 -1
View File
@@ -35,7 +35,7 @@ import core, {
type TxResult,
type WorkspaceIds
} from '@hcengineering/core'
import { type DbAdapter, type DbAdapterHandler, type DomainHelperOperations } from './adapter'
import { type DbAdapter, type DbAdapterHandler, type DomainHelperOperations, type RawFindIterator } from './adapter'
/**
* @public
*/
@@ -62,6 +62,13 @@ export class DummyDbAdapter implements DbAdapter {
return toFindResult([])
}
rawFind (ctx: MeasureContext, domain: Domain): RawFindIterator {
return {
find: async () => [],
close: async () => {}
}
}
async init (): Promise<void> {}
helper (): DomainHelperOperations {
server/core/src/plugin.ts
+1 -4
View File
@@ -16,7 +16,7 @@
import { type Metadata, type Plugin, plugin } from '@hcengineering/platform'
import type { Class, Ref, Space, Mixin } from '@hcengineering/core'
import type { Class, Mixin, Ref } from '@hcengineering/core'
import type { ObjectDDParticipant, SearchPresenter, Trigger } from './types'
/**
@@ -35,9 +35,6 @@ const serverCore = plugin(serverCoreId, {
ObjectDDParticipant: '' as Ref<ObjectDDParticipant>,
SearchPresenter: '' as Ref<Mixin<SearchPresenter>>
},
space: {
DocIndexState: '' as Ref<Space>
},
metadata: {
FrontUrl: '' as Metadata<string>,
FilesUrl: '' as Metadata<string>,
server/core/src/queue/dummyQueue.ts
+65
View File
@@ -0,0 +1,65 @@
import type { MeasureContext, WorkspaceUuid } from '@hcengineering/core'
import { type ConsumerHandle, type PlatformQueue, type PlatformQueueProducer, type QueueTopic } from './types'
/**
* A dummy implementation of PlatformQueueProducer for testing and development
*/
class DummyQueueProducer<T> implements PlatformQueueProducer<T> {
async send (id: WorkspaceUuid | string, msgs: T[]): Promise<void> {
await Promise.resolve()
}
async close (): Promise<void> {
await Promise.resolve()
}
}
/**
* A dummy implementation of PlatformQueue for testing and development
*/
export class DummyQueue implements PlatformQueue {
createProducer<T>(ctx: MeasureContext, topic: QueueTopic): PlatformQueueProducer<T> {
return new DummyQueueProducer<T>()
}
getClientId (): string {
return 'dummy'
}
createConsumer<T>(
ctx: MeasureContext,
topic: QueueTopic,
groupId: string,
onMessage: (
msg: { id: WorkspaceUuid | string, value: T }[],
queue: {
pause: () => void
heartbeat: () => Promise<void>
}
) => Promise<void>,
options?: {
bulkSize?: number
fromBegining?: boolean
}
): ConsumerHandle {
return {
shutdown: async (): Promise<void> => {
await Promise.resolve()
},
isConnected: (): boolean => {
return false
}
}
}
async createTopics (tx: number): Promise<void> {
await Promise.resolve()
}
}
/**
* Creates a new instance of the dummy queue implementation
*/
export function createDummyQueue (): PlatformQueue {
return new DummyQueue()
}
server/core/src/queue/index.ts
+4
View File
@@ -0,0 +1,4 @@
export * from './types'
export * from './workspace'
export * from './dummyQueue'
export * from './users'
server/core/src/queue/types.ts
+59
View File
@@ -0,0 +1,59 @@
import type { MeasureContext, WorkspaceUuid } from '@hcengineering/core'
export enum QueueTopic {
// Topic with partitions to split workspace transactions into
Tx = 'tx',
// Topic to send workspace information into
Workspace = 'workspace',
// A fulltext topic to hold extra fulltext request, like re-indexing requests etc, private to fulltext service.
Fulltext = 'fulltext',
// A topic about user activity.
Users = 'users'
}
export interface ConsumerHandle {
shutdown: () => Promise<void>
isConnected: () => boolean
}
export interface ConsumerMessage<T> {
id: WorkspaceUuid | string
value: T[]
}
export interface ConsumerControl {
pause: () => void
heartbeat: () => Promise<void>
}
export interface PlatformQueue {
createProducer: <T>(ctx: MeasureContext, topic: QueueTopic) => PlatformQueueProducer<T>
/**
* Create a consumer for a topic.
* return a function to close the reciever.
*/
createConsumer: <T>(
ctx: MeasureContext,
topic: QueueTopic,
groupId: string,
onMessage: (msg: ConsumerMessage<T>[], queue: ConsumerControl) => Promise<void>,
options?: {
fromBegining?: boolean
}
) => ConsumerHandle
createTopics: (tx: number) => Promise<void>
getClientId: () => string
}
/**
* Create a producer for a topic.
*/
export interface PlatformQueueProducer<T> {
send: (id: WorkspaceUuid | string, msgs: T[]) => Promise<void>
close: () => Promise<void>
}
server/core/src/queue/users.ts
+32
View File
@@ -0,0 +1,32 @@
import type { AccountUuid, PersonId } from '@hcengineering/core'
export enum QueueUserEvent {
login = 'login',
logout = 'logout'
}
export interface QueueUserMessage {
type: QueueUserEvent
user: AccountUuid
}
export interface QueueUserLogin extends QueueUserMessage {
type: QueueUserEvent.login | QueueUserEvent.logout
sessions: number
socialIds: PersonId[]
}
export const userEvents = {
login: function userLogin (data: Omit<QueueUserLogin, 'type'>): QueueUserLogin {
return {
type: QueueUserEvent.login,
...data
}
},
logout: function userLogout (data: Omit<QueueUserLogin, 'type'>): QueueUserLogin {
return {
type: QueueUserEvent.logout,
...data
}
}
}
server/core/src/queue/workspace.ts
+46
View File
@@ -0,0 +1,46 @@
import type { Class, Doc, Domain, Ref } from '@hcengineering/core'
export enum QueueWorkspaceEvent {
Up = 'up',
Down = 'down',
Created = 'created',
CreateFailed = 'create-failed',
Upgraded = 'upgraded',
Upgradefailed = 'upgrade-failed',
Deleted = 'deleted',
Archived = 'archived',
Restored = 'restored',
FullReindex = 'full-fulltext-reindex',
Reindex = 'fulltext-reindex',
ClearIndex = 'clear-fulltext-index'
}
export interface QueueWorkspaceMessage {
type: QueueWorkspaceEvent
}
export interface QueueWorkspaceReindexMessage extends QueueWorkspaceMessage {
type: QueueWorkspaceEvent.Reindex
domain: Domain
classes: Ref<Class<Doc>>[]
}
export const workspaceEvents = {
open: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Up }),
down: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Down }),
created: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Created }),
upgraded: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Upgraded }),
upgradeFailed: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Upgradefailed }),
createFailed: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.CreateFailed }),
deleted: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Deleted }),
archived: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Archived }),
restored: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Restored }),
fullReindex: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.FullReindex }),
clearIndex: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.ClearIndex }),
reindex: (domain: Domain, classes: Ref<Class<Doc>>[]): QueueWorkspaceReindexMessage => ({
type: QueueWorkspaceEvent.Reindex,
domain,
classes
})
}
server/indexer/src/indexer/indexer.ts
+334 -580
View File
File diff suppressed because it is too large Load Diff
server/indexer/src/indexer/types.ts
-11
View File
@@ -16,8 +16,6 @@
import {
type Class,
type Doc,
type DocIndexState,
type DocumentUpdate,
type FullTextSearchContext,
type Hierarchy,
type ModelDb,
@@ -34,12 +32,3 @@ export interface FullTextPipeline {
cancelling: boolean
}
/**
* @public
*/
export type DocUpdateHandler = (doc: DocIndexState, update: DocumentUpdate<DocIndexState>) => Promise<void>
export const indexes = 'verify-indexes-v2'
export const fullReindex = 'full-text-indexer-v5'
export const docStructure = 'full-text-structure-v6'
server/indexer/src/indexer/utils.ts
+2 -24
View File
@@ -13,20 +13,17 @@
// limitations under the License.
//
import core, {
import {
type AnyAttribute,
type Class,
type Data,
type Doc,
type DocIndexState,
type FullTextSearchContext,
getFullTextContext,
type Hierarchy,
type Obj,
type Ref,
type Space
} from '@hcengineering/core'
import plugin, { type IndexedDoc } from '@hcengineering/server-core'
import { type IndexedDoc } from '@hcengineering/server-core'
import { type FullTextPipeline } from './types'
export { docKey, isFullTextAttribute } from '@hcengineering/core'
@@ -55,25 +52,6 @@ export function getContent (
return attrs
}
/**
* @public
*/
export function createStateDoc (
id: Ref<Doc>,
objectClass: Ref<Class<Obj>>,
data: Omit<Data<DocIndexState>, 'objectClass'> & { space?: Ref<Space> }
): DocIndexState {
return {
_class: core.class.DocIndexState,
_id: id as Ref<DocIndexState>,
space: data.space ?? plugin.space.DocIndexState,
objectClass,
modifiedBy: core.account.System,
modifiedOn: Date.now(),
...data
}
}
/**
* @public
*/
models/server-fulltext/.eslintrc.js → server/kafka/.eslintrc.js
+1 -1
View File
@@ -1,5 +1,5 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/model/eslint.config.json'],
extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
models/server-fulltext/.npmignore → server/kafka/.npmignore
View File
models/server-fulltext/config/rig.json → server/kafka/config/rig.json
+1 -1
View File
@@ -1,5 +1,5 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig",
"rigProfile": "model"
"rigProfile": "node"
}
server-plugins/fulltext-resources/jest.config.js → server/kafka/jest.config.js
View File
server-plugins/fulltext-resources/package.json → server/kafka/package.json
+8 -9
View File
@@ -1,23 +1,19 @@
{
"name": "@hcengineering/server-fulltext-resources",
"name": "@hcengineering/kafka",
"version": "0.6.0",
"main": "lib/index.js",
"svelte": "src/index.ts",
"types": "types/index.d.ts",
"files": [
"lib/**/*",
"types/**/*",
"tsconfig.json"
],
"author": "Anticrm Platform Contributors",
"template": "@hcengineering/node-package",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"test": "jest --passWithNoTests --silent --forceExit",
"format": "format src",
"test": "jest --passWithNoTests --silent",
"_phase:build": "compile transpile src",
"_phase:test": "jest --passWithNoTests --silent",
"_phase:test": "jest --passWithNoTests --silent --forceExit",
"_phase:format": "format src",
"_phase:validate": "compile validate"
},
@@ -32,6 +28,7 @@
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"@types/node": "~20.11.16",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5"
@@ -39,6 +36,8 @@
"dependencies": {
"@hcengineering/core": "^0.6.32",
"@hcengineering/platform": "^0.6.11",
"@hcengineering/server-core": "^0.6.1"
"@hcengineering/server-core": "^0.6.1",
"@hcengineering/storage": "^0.6.0",
"kafkajs": "^2.2.4"
}
}
server/kafka/src/index.ts
+262
View File
@@ -0,0 +1,262 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type MeasureContext, type WorkspaceUuid } from '@hcengineering/core'
import {
QueueTopic,
type ConsumerControl,
type ConsumerHandle,
type ConsumerMessage,
type PlatformQueue,
type PlatformQueueProducer
} from '@hcengineering/server-core'
import { Kafka, Partitioners, type Consumer, type Producer } from 'kafkajs'
import type * as tls from 'tls'
export interface QueueConfig {
postfix: string // a topic prefix to be used do distinguish between staging and production topics in same broker
brokers: string[]
clientId: string
region: string
ssl?: tls.ConnectionOptions | boolean
}
/**
* Query config parser in the form of 'brokers;clientId'
* brokers are comma separated
*/
export function parseQueueConfig (config: string, serviceId: string, region: string): QueueConfig {
const [brokers, postfix] = config.split(';')
return {
postfix: postfix ?? '', // Empty by default
brokers: brokers.split(','),
clientId: serviceId,
region
}
}
function getKafkaTopicId (topic: QueueTopic, config: QueueConfig): string {
if (config.region !== '') {
return `${config.region}.${topic}${config.postfix ?? ''}`
}
return `${topic}${config.postfix ?? ''}`
}
class PlatformQueueImpl implements PlatformQueue {
constructor (
private readonly kafka: Kafka,
readonly config: QueueConfig
) {}
getClientId (): string {
return this.config.clientId
}
createProducer<T>(ctx: MeasureContext, topic: QueueTopic): PlatformQueueProducer<T> {
return new PlatformQueueProducerImpl(ctx, this.kafka, getKafkaTopicId(topic, this.config))
}
createConsumer<T>(
ctx: MeasureContext,
topic: QueueTopic,
groupId: string,
onMessage: (msg: ConsumerMessage<T>[], queue: ConsumerControl) => Promise<void>,
options?: {
fromBegining?: boolean
}
): ConsumerHandle {
return new PlatformQueueConsumerImpl(ctx, this.kafka, this.config, topic, groupId, onMessage, options)
}
async createTopics (tx: number): Promise<void> {
const topics = new Set(await this.kafka.admin({}).listTopics())
if (!topics.has(QueueTopic.Tx)) {
await this.kafka.admin().createTopics({
topics: [
{
topic: getKafkaTopicId(QueueTopic.Tx, this.config),
numPartitions: tx
}
]
})
}
}
}
class PlatformQueueProducerImpl implements PlatformQueueProducer<any> {
txProducer: Producer
connected: Promise<void> | undefined
constructor (
readonly ctx: MeasureContext,
kafka: Kafka,
private readonly topic: string
) {
this.txProducer = kafka.producer({
allowAutoTopicCreation: true,
createPartitioner: Partitioners.DefaultPartitioner
})
this.connected = this.ctx.with('connect-broker', {}, () => this.txProducer.connect())
}
async send (id: WorkspaceUuid | string, msgs: any[]): Promise<void> {
if (this.connected !== undefined) {
await this.connected
this.connected = undefined
}
await this.ctx.with('send', { topic: this.topic }, () =>
this.txProducer.send({
topic: this.topic,
messages: msgs.map((m) => ({
key: Buffer.from(`${id}`),
value: Buffer.from(JSON.stringify(m))
}))
})
)
}
async close (): Promise<void> {
await this.ctx.with('disconnect', {}, () => this.txProducer.disconnect())
}
}
class PlatformQueueConsumerImpl implements ConsumerHandle {
connected = false
cc: Consumer
constructor (
readonly ctx: MeasureContext,
readonly kafka: Kafka,
readonly config: QueueConfig,
private readonly topic: QueueTopic,
groupId: string,
private readonly onMessage: (msg: ConsumerMessage<any>[], queue: ConsumerControl) => Promise<void>,
private readonly options?: {
fromBegining?: boolean
}
) {
this.cc = this.kafka.consumer({ groupId: `${getKafkaTopicId(this.topic, this.config)}-${groupId}` })
void this.start().catch((err) => {
ctx.error('failed to consume', { err })
})
}
async start (): Promise<void> {
await this.doConnect()
await this.doSubscribe()
await this.cc.run({
eachMessage: async ({ topic, message, pause, heartbeat }) => {
await this.onMessage(
[
{
id: message.key?.toString() ?? '',
value: [JSON.parse(message.value?.toString() ?? '{}')]
}
],
{
heartbeat,
pause
}
)
}
// , // TODO: Finish testinf
// eachBatch: async ({ batch, pause, heartbeat, resolveOffset }) => {
// const queueInfo = {
// pause,
// heartbeat
// }
// const batchMessages = batch.messages
// const currentMsg: ConsumerMessage<any> = {
// id: '',
// value: []
// }
// let lastOffset: string = ' '
// const sendLast = async (): Promise<void> => {
// await this.onMessage([currentMsg], queueInfo)
// // Mark last offset as cusomed
// resolveOffset(lastOffset)
// await heartbeat()
// }
// for (const v of batchMessages) {
// const id = v.key?.toString() ?? ''
// if (currentMsg.id !== id && currentMsg.value.length > 0) {
// await sendLast() // Send last message
// currentMsg.id = id
// currentMsg.value = []
// }
// lastOffset = v.offset
// currentMsg.value.push(JSON.parse(v.value?.toString() ?? '{}'))
// }
// await sendLast()
// }
})
}
async doConnect (): Promise<void> {
this.cc.on('consumer.connect', () => {
this.connected = true
this.ctx.info('consumer connected to queue')
})
this.cc.on('consumer.disconnect', () => {
this.connected = false
this.ctx.warn('consumer disconnected from queue')
})
await this.cc.connect()
}
async doSubscribe (): Promise<void> {
await this.cc.subscribe({
topic: getKafkaTopicId(this.topic, this.config),
fromBeginning: this.options?.fromBegining
})
}
isConnected (): boolean {
return this.connected
}
shutdown (): Promise<void> {
return this.cc.disconnect()
}
}
/**
* Constructs a platform queue.
*/
export function getPlatformQueue (serviceId: string, region?: string): PlatformQueue {
const queueConfig = process.env.QUEUE_CONFIG ?? 'huly.local:9092'
if (queueConfig === undefined) {
throw new Error('Please provide queue config')
}
const config = parseQueueConfig(queueConfig, serviceId, region ?? process.env.REGION ?? '')
return createPlatformQueue(config)
}
export function createPlatformQueue (config: QueueConfig): PlatformQueue {
console.info({ message: 'Using queue', config })
return new PlatformQueueImpl(
new Kafka({
clientId: config.clientId,
brokers: config.brokers,
ssl: config.ssl
}),
config
)
}
models/server-fulltext/tsconfig.json → server/kafka/tsconfig.json
+1 -1
View File
@@ -1,5 +1,5 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/model/tsconfig.json",
"extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
server/middleware/src/fulltext.ts
+2 -61
View File
@@ -16,7 +16,6 @@
import { Analytics } from '@hcengineering/analytics'
import core, {
docKey,
DOMAIN_DOC_INDEX_STATE,
isFullTextAttribute,
isIndexedAttribute,
toFindResult,
@@ -33,17 +32,9 @@ import core, {
type Ref,
type SearchOptions,
type SearchQuery,
type SearchResult,
type SessionData,
type Tx
type SearchResult
} from '@hcengineering/core'
import type {
IndexedDoc,
Middleware,
MiddlewareCreator,
PipelineContext,
TxMiddlewareResult
} from '@hcengineering/server-core'
import type { IndexedDoc, Middleware, MiddlewareCreator, PipelineContext } from '@hcengineering/server-core'
import { BaseMiddleware } from '@hcengineering/server-core'
/**
* @public
@@ -52,8 +43,6 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware {
fulltextEndpoint: string
contexts = new Map<Ref<Class<Doc>>, FullTextSearchContext>()
warmupTriggered: boolean = false
constructor (
context: PipelineContext,
next: Middleware | undefined,
@@ -79,54 +68,10 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware {
}
}
async handleBroadcast (ctx: MeasureContext<SessionData>): Promise<void> {
if (ctx.contextData.fulltextUpdates !== undefined && ctx.contextData.fulltextUpdates.size > 0) {
const toUpdate = Array.from(ctx.contextData.fulltextUpdates.values())
ctx.contextData.fulltextUpdates.clear()
await this.context.lowLevelStorage?.upload(ctx, DOMAIN_DOC_INDEX_STATE, toUpdate)
this.sendWarmup()
}
await super.handleBroadcast(ctx)
}
async init (ctx: MeasureContext): Promise<void> {
this.contexts = new Map(
this.context.modelDb.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it])
)
// Send one warumup so reindex will
this.sendWarmup()
}
sendWarmup (): void {
if (!this.warmupTriggered) {
this.warmupTriggered = true
setTimeout(() => {
this.warmupTriggered = false
void this._sendWarmup().catch(() => {})
}, 25)
}
}
async _sendWarmup (): Promise<void> {
// Just send a warmup to indexer to start do something, or just reindex if some work are pending.
try {
await fetch(this.fulltextEndpoint + '/api/v1/warmup', {
method: 'PUT',
keepalive: true,
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
token: this.token
})
})
} catch (err: any) {
if (err?.cause?.code === 'ECONNRESET' || err?.cause?.code === 'ECONNREFUSED') {
return
}
Analytics.handleError(err)
}
}
async search<T extends Doc>(
@@ -385,10 +330,6 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware {
}
}
tx (ctx: MeasureContext, txes: Tx[]): Promise<TxMiddlewareResult> {
return this.provideTx(ctx, txes)
}
async close (): Promise<void> {
try {
await fetch(this.fulltextEndpoint + '/api/v1/close', {
server/middleware/src/index.ts
+1
View File
@@ -35,3 +35,4 @@ export * from './spacePermissions'
export * from './spaceSecurity'
export * from './triggers'
export * from './txPush'
export * from './queue'
server/middleware/src/model.ts
+11 -7
View File
@@ -19,6 +19,7 @@ import core, {
type DocumentQuery,
type FindOptions,
type FindResult,
type Hierarchy,
type LoadModelResponse,
type MeasureContext,
type Ref,
@@ -59,7 +60,8 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware {
constructor (
context: PipelineContext,
next: Middleware | undefined,
readonly systemTx: Tx[]
readonly systemTx: Tx[],
readonly filter?: (h: Hierarchy, model: Tx[]) => Tx[]
) {
super(context, next)
}
@@ -69,16 +71,17 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware {
ctx: MeasureContext,
context: PipelineContext,
next: Middleware | undefined,
systemTx: Tx[]
systemTx: Tx[],
filter?: (h: Hierarchy, model: Tx[]) => Tx[]
): Promise<Middleware> {
const middleware = new ModelMiddleware(context, next, systemTx)
const middleware = new ModelMiddleware(context, next, systemTx, filter)
await middleware.init(ctx)
return middleware
}
static create (tx: Tx[]): MiddlewareCreator {
static create (tx: Tx[], filter?: (h: Hierarchy, model: Tx[]) => Tx[]): MiddlewareCreator {
return (ctx, context, next) => {
return this.doCreate(ctx, context, next, tx)
return this.doCreate(ctx, context, next, tx, filter)
}
}
@@ -116,9 +119,10 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware {
ctx.warn('failed to apply model transaction, skipping', { tx: JSON.stringify(tx), err })
}
}
this.context.modelDb.addTxes(ctx, model, true)
const fmodel = this.filter !== undefined ? this.filter(this.context.hierarchy, model) : model
this.context.modelDb.addTxes(ctx, fmodel, true)
this.setModel(model)
this.setModel(fmodel)
}
private addModelTx (tx: Tx): void {
server/middleware/src/queue.ts
+56
View File
@@ -0,0 +1,56 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type MeasureContext, type SessionData, type Tx } from '@hcengineering/core'
import {
BaseMiddleware,
type Middleware,
type MiddlewareCreator,
type PipelineContext,
type PlatformQueue,
type PlatformQueueProducer,
QueueTopic
} from '@hcengineering/server-core'
export class QueueMiddleware extends BaseMiddleware {
txProducer: PlatformQueueProducer<Tx>
connected: Promise<void> | undefined
constructor (
ctx: MeasureContext,
readonly context: PipelineContext,
protected readonly next: Middleware | undefined,
readonly queue: PlatformQueue
) {
super(context, next)
this.txProducer = queue.createProducer<Tx>(ctx, QueueTopic.Tx)
}
static create (queue: PlatformQueue): MiddlewareCreator {
return async (ctx, context, next) => {
return new QueueMiddleware(ctx, context, next, queue)
}
}
async handleBroadcast (ctx: MeasureContext<SessionData>): Promise<void> {
if (this.connected !== undefined) {
await this.connected
this.connected = undefined
}
await Promise.all([
this.provideBroadcast(ctx),
this.txProducer.send(this.context.workspace.uuid, ctx.contextData.broadcast.txes)
])
}
}
server/middleware/src/spacePermissions.ts
+1 -1
View File
@@ -174,7 +174,7 @@ export class SpacePermissionsMiddleware extends BaseMiddleware implements Middle
if (!this.context.hierarchy.isDerived(createTx.objectClass, core.class.Space)) return
if (this.isTypedSpaceClass(createTx.objectClass)) {
const res = TxProcessor.buildDoc2Doc<TypedSpace>([createTx])
if (res !== undefined) {
if (res != null) {
this.addRestrictedSpace(res)
}
} else {
server/mongo/src/__tests__/minmodel.ts
-9
View File
@@ -21,7 +21,6 @@ import core, {
ClassifierKind,
type Data,
type Doc,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MODEL,
DOMAIN_RELATION,
DOMAIN_TX,
@@ -136,14 +135,6 @@ export function genMinModel (): TxCUD<Doc>[] {
domain: DOMAIN_MODEL
})
)
txes.push(
createClass(core.class.DocIndexState, {
label: 'DocIndexState' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.CLASS,
domain: DOMAIN_DOC_INDEX_STATE
})
)
txes.push(
createClass(core.class.Tx, {
server/mongo/src/storage.ts
+32 -4
View File
@@ -26,10 +26,10 @@ import core, {
groupByArray,
isOperator,
matchQuery,
platformNow,
toFindResult,
withContext,
type AssociationQuery,
type WorkspaceIds,
type Class,
type Doc,
type DocInfo,
@@ -62,16 +62,17 @@ import core, {
type TxResult,
type TxUpdateDoc,
type WithLookup,
platformNow
type WorkspaceIds
} from '@hcengineering/core'
import {
calcHashHash,
type DbAdapter,
type DbAdapterHandler,
type DomainHelperOperations,
type RawFindIterator,
type ServerFindOptions,
type StorageAdapter,
type TxAdapter,
calcHashHash
type TxAdapter
} from '@hcengineering/server-core'
import {
type AbstractCursor,
@@ -1145,6 +1146,33 @@ abstract class MongoAdapterBase implements DbAdapter {
}
}
rawFind (_ctx: MeasureContext, domain: Domain): RawFindIterator {
const ctx = _ctx.newChild('findRaw', { domain })
const coll = this.db.collection<Doc>(domain)
let iterator: FindCursor<Doc>
return {
find: async () => {
if (iterator === undefined) {
iterator = coll.find({})
}
const d = await ctx.with('next', {}, () => iterator.next())
const result: Doc[] = []
if (d != null) {
result.push(this.stripHash(d) as Doc)
}
if (iterator.bufferedCount() > 0) {
result.push(...(this.stripHash(iterator.readBufferedDocuments()) as Doc[]))
}
return result ?? []
},
close: async () => {
await ctx.with('close', {}, () => iterator.close())
ctx.end()
}
}
}
load (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
return ctx.with('load', { domain }, async () => {
if (docs.length === 0) {
server/postgres/migrations/docIndexStateSchema.sql
-11
View File
@@ -1,11 +0,0 @@
ALTER TABLE doc_index_state
ADD "needIndex" bool;
ALTER TABLE doc_index_state
DROP COLUMN "attachedTo";
UPDATE doc_index_state
SET "needIndex" = (data->>'needIndex')::boolean;
ALTER TABLE doc_index_state
ALTER COLUMN "needIndex" SET NOT NULL;
server/postgres/src/__tests__/minmodel.ts
-9
View File
@@ -22,7 +22,6 @@ import core, {
type Data,
type Doc,
type Domain,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MODEL,
DOMAIN_RELATION,
DOMAIN_TX,
@@ -155,14 +154,6 @@ export function genMinModel (): TxCUD<Doc>[] {
domain: DOMAIN_MODEL
})
)
txes.push(
createClass(core.class.DocIndexState, {
label: 'DocIndexState' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.CLASS,
domain: DOMAIN_DOC_INDEX_STATE
})
)
txes.push(
createClass(core.class.Tx, {
server/postgres/src/schemas.ts
+1 -16
View File
@@ -1,4 +1,4 @@
import { DOMAIN_DOC_INDEX_STATE, DOMAIN_MODEL_TX, DOMAIN_RELATION, DOMAIN_SPACE, DOMAIN_TX } from '@hcengineering/core'
import { DOMAIN_MODEL_TX, DOMAIN_RELATION, DOMAIN_SPACE, DOMAIN_TX } from '@hcengineering/core'
export type DataType = 'bigint' | 'bool' | 'text' | 'text[]'
@@ -170,20 +170,6 @@ const userNotificationSchema: Schema = {
}
}
const docIndexStateSchema: Schema = {
...baseSchema,
needIndex: {
type: 'bool',
notNull: true,
index: true
},
objectClass: {
type: 'text',
notNull: true,
index: false
}
}
const timeSchema: Schema = {
...baseSchema,
workslots: {
@@ -305,7 +291,6 @@ export const domainSchemas: Record<string, Schema> = {
[translateDomain('time')]: timeSchema,
[translateDomain('calendar')]: calendarSchema,
[translateDomain('event')]: eventSchema,
[translateDomain(DOMAIN_DOC_INDEX_STATE)]: docIndexStateSchema,
notification: notificationSchema,
[translateDomain('notification-dnc')]: dncSchema,
[translateDomain('notification-user')]: userNotificationSchema,
server/postgres/src/storage.ts
+49
View File
@@ -66,6 +66,7 @@ import {
type DbAdapter,
type DbAdapterHandler,
type DomainHelperOperations,
type RawFindIterator,
type ServerFindOptions,
type TxAdapter
} from '@hcengineering/server-core'
@@ -1615,6 +1616,54 @@ abstract class PostgresAdapterBase implements DbAdapter {
}
}
rawFind (_ctx: MeasureContext, domain: Domain): RawFindIterator {
const ctx = _ctx.newChild('findRaw', { domain })
let initialized: boolean = false
let client: DBClient
const tdomain = translateDomain(domain)
const schema = getSchema(domain)
const workspaceId = this.workspaceId
function createBulk (projection: string, limit = 1000): AsyncGenerator<Doc[]> {
const sql = `
SELECT ${projection}
FROM ${tdomain}
WHERE "workspaceId" = '${workspaceId}'
`
return createCursorGenerator(client.raw(), sql, undefined, schema, limit)
}
let bulk: AsyncGenerator<Doc[]>
return {
find: async () => {
if (!initialized) {
if (client === undefined) {
client = await this.client.reserve()
}
initialized = true
bulk = createBulk('*')
}
const docs = await ctx.with('next', {}, () => bulk.next())
if (docs.done === true || docs.value.length === 0) {
return []
}
const result: Doc[] = []
result.push(...(this.stripHash(docs.value) as Doc[]))
return result
},
close: async () => {
await bulk.return([]) // We need to close generator, just in case
client?.release()
ctx.end()
}
}
}
load (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
return ctx.with('load', { domain }, async () => {
if (docs.length === 0) {
server/server-pipeline/package.json
+2 -3
View File
@@ -168,11 +168,10 @@
"@hcengineering/server-ai-bot-resources": "^0.6.0",
"@hcengineering/time": "^0.6.0",
"@hcengineering/time-assets": "^0.6.0",
"@hcengineering/server-fulltext": "^0.6.0",
"@hcengineering/server-fulltext-resources": "^0.6.0",
"@hcengineering/survey": "^0.6.0",
"@hcengineering/survey-assets": "^0.6.0",
"@hcengineering/server-process": "^0.6.0",
"@hcengineering/server-process-resources": "^0.6.0"
"@hcengineering/server-process-resources": "^0.6.0",
"@hcengineering/kafka": "^0.6.0"
}
}
server/server-pipeline/src/blobStorage.ts
+8
View File
@@ -39,6 +39,7 @@ import { PlatformError, unknownError } from '@hcengineering/platform'
import {
type DbAdapter,
type DbAdapterHandler,
type RawFindIterator,
type StorageAdapter,
type StorageAdapterEx
} from '@hcengineering/server-core'
@@ -78,6 +79,13 @@ class StorageBlobAdapter implements DbAdapter {
return []
}
rawFind (ctx: MeasureContext, domain: Domain): RawFindIterator {
return {
find: async () => [],
close: async () => {}
}
}
async rawUpdate<T extends Doc>(
domain: Domain,
query: DocumentQuery<T>,
server/server-pipeline/src/pipeline.ts
+10 -2
View File
@@ -32,6 +32,7 @@ import {
NotificationsMiddleware,
PrivateMiddleware,
QueryJoinMiddleware,
QueueMiddleware,
SpacePermissionsMiddleware,
SpaceSecurityMiddleware,
TriggersMiddleware,
@@ -50,10 +51,12 @@ import {
type PipelineContext,
type PipelineFactory,
type StorageAdapter,
type WorkspaceDestroyAdapter
type WorkspaceDestroyAdapter,
type PlatformQueue
} from '@hcengineering/server-core'
import { generateToken } from '@hcengineering/server-token'
import { createStorageDataAdapter } from './blobStorage'
/**
* @public
*/
@@ -99,6 +102,8 @@ export function createServerPipeline (
externalStorage: StorageAdapter
queue?: PlatformQueue
extraLogging?: boolean // If passed, will log every request/etc.
pipelineContextVars?: Record<string, any>
},
@@ -131,6 +136,7 @@ export function createServerPipeline (
LiveQueryMiddleware.create,
DomainFindMiddleware.create,
DomainTxMiddleware.create,
...(opt.queue !== undefined ? [QueueMiddleware.create(opt.queue)] : []),
DBAdapterInitMiddleware.create,
ModelMiddleware.create(model),
DBAdapterMiddleware.create(conf), // Configure DB adapters
@@ -205,6 +211,7 @@ export async function getServerPipeline (
wsUrl: WorkspaceIds,
storageAdapter: StorageAdapter,
opt?: {
queue?: PlatformQueue
disableTriggers?: boolean
}
): Promise<Pipeline> {
@@ -212,7 +219,8 @@ export async function getServerPipeline (
externalStorage: storageAdapter,
usePassedCtx: true,
disableTriggers: opt?.disableTriggers ?? false,
adapterSecurity: isAdapterSecurity(dbUrl)
adapterSecurity: isAdapterSecurity(dbUrl),
queue: opt?.queue
})
return await pipelineFactory(ctx, wsUrl, true, () => {}, null)
server/server-pipeline/src/serverPlugins.ts
-2
View File
@@ -28,7 +28,6 @@ import { serverTrackerId } from '@hcengineering/server-tracker'
import { serverTrainingId } from '@hcengineering/server-training'
import { serverViewId } from '@hcengineering/server-view'
import { serverAiBotId } from '@hcengineering/server-ai-bot'
import { serverFulltextId } from '@hcengineering/server-fulltext'
import { serverProcessId } from '@hcengineering/server-process'
export function registerServerPlugins (): void {
@@ -61,6 +60,5 @@ export function registerServerPlugins (): void {
addLocation(serverTrainingId, () => import('@hcengineering/server-training-resources'))
addLocation(serverGithubId, () => import('@hcengineering/server-github-resources'))
addLocation(serverAiBotId, () => import('@hcengineering/server-ai-bot-resources'))
addLocation(serverFulltextId, () => import('@hcengineering/server-fulltext-resources'))
addLocation(serverProcessId, () => import('@hcengineering/server-process-resources'))
}
server/server/src/sessionManager.ts
+69 -20
View File
@@ -13,38 +13,38 @@
// limitations under the License.
//
import { getClient as getAccountClient, isWorkspaceLoginInfo } from '@hcengineering/account-client'
import { Analytics } from '@hcengineering/analytics'
import core, {
AccountRole,
cutObjectArray,
Data,
generateId,
isArchivingMode,
isMigrationMode,
isRestoringMode,
isWorkspaceCreating,
TxFactory,
WorkspaceEvent,
pickPrimarySocialId,
platformNow,
platformNowDiff,
systemAccountUuid,
TxFactory,
Version,
versionToString,
withContext,
WorkspaceEvent,
type Account,
type AccountUuid,
type Branding,
type BrandingMap,
type MeasureContext,
type PersonId,
type Tx,
type TxWorkspaceEvent,
type WorkspaceUuid,
type WorkspaceInfoWithStatus,
Account,
pickPrimarySocialId,
type PersonId,
type WorkspaceDataId,
Data,
Version,
platformNow,
platformNowDiff,
AccountUuid
type WorkspaceInfoWithStatus,
type WorkspaceUuid
} from '@hcengineering/core'
import { getClient as getAccountClient, isWorkspaceLoginInfo } from '@hcengineering/account-client'
import { unknownError, type Status } from '@hcengineering/platform'
import { type HelloRequest, type HelloResponse, type Request, type Response } from '@hcengineering/rpc'
import {
@@ -52,14 +52,21 @@ import {
pingConst,
Pipeline,
PipelineFactory,
QueueTopic,
QueueUserMessage,
ServerFactory,
SessionManager,
StorageAdapter,
type SessionFactory,
userEvents,
workspaceEvents,
type AddSessionResponse,
type ClientSessionCtx,
type ConnectionSocket,
type PlatformQueue,
type PlatformQueueProducer,
type QueueWorkspaceMessage,
type Session,
type SessionFactory,
type Workspace
} from '@hcengineering/server-core'
import { generateToken, type Token } from '@hcengineering/server-token'
@@ -108,7 +115,8 @@ export class TSessionManager implements SessionManager {
oldClientErrors: number = 0
clientErrors: number = 0
lastClients: string[] = []
workspaceProducer: PlatformQueueProducer<QueueWorkspaceMessage>
usersProducer: PlatformQueueProducer<QueueUserMessage>
constructor (
readonly ctx: MeasureContext,
readonly sessionFactory: SessionFactory,
@@ -122,13 +130,16 @@ export class TSessionManager implements SessionManager {
| undefined,
readonly accountsUrl: string,
readonly enableCompression: boolean,
readonly doHandleTick: boolean = true
readonly doHandleTick: boolean = true,
readonly queue: PlatformQueue
) {
if (this.doHandleTick) {
this.checkInterval = setInterval(() => {
this.handleTick()
}, 1000 / ticksPerSecond)
}
this.workspaceProducer = this.queue.createProducer(ctx.newChild('queue', {}), QueueTopic.Workspace)
this.usersProducer = this.queue.createProducer(ctx.newChild('queue', {}), QueueTopic.Users)
}
scheduleMaintenance (timeMinutes: number): void {
@@ -357,6 +368,12 @@ export class TSessionManager implements SessionManager {
}
}
countUserSessions (workspace: Workspace, accountUuid: AccountUuid): number {
return Array.from(workspace.sessions.values())
.filter((it) => it.session.getUser() === accountUuid)
.reduce<number>((acc) => acc + 1, 0)
}
tickCounter = 0
@withContext('📲 add-session')
@@ -479,6 +496,7 @@ export class TSessionManager implements SessionManager {
workspaceInfo.dataId,
branding
)
await this.workspaceProducer.send(workspaceInfo.uuid, [workspaceEvents.open()])
}
let pipeline: Pipeline
@@ -546,13 +564,26 @@ export class TSessionManager implements SessionManager {
this.tickCounter++
workspace.sessions.set(session.sessionId, { session, socket: ws, tickHash: this.tickCounter % ticksPerSecond })
const accountUuid = account.uuid
await this.usersProducer.send(workspaceInfo.uuid, [
userEvents.login({
user: accountUuid,
sessions: this.countUserSessions(workspace, accountUuid),
socialIds: account.socialIds
})
])
// Mark workspace as init completed and we had at least one client.
if (!workspace.workspaceInitCompleted) {
workspace.workspaceInitCompleted = true
}
if (this.timeMinutes > 0) {
void ws.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression)
void ws
.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression)
.catch((err) => {
ctx.error('failed to send maintenance warning', err)
})
}
return { session, context: workspace.context, workspaceId: workspaceInfo.uuid }
}
@@ -809,8 +840,19 @@ export class TSessionManager implements SessionManager {
totalUsers: this.sessions.size
})
this.sessions.delete(ws.id)
if (workspace !== undefined) {
workspace.sessions.delete(sessionRef.session.sessionId)
const userUuid = sessionRef.session.getUser()
await this.usersProducer.send(workspace.workspaceUuid, [
userEvents.logout({
user: userUuid,
sessions: this.countUserSessions(workspace, userUuid),
socialIds: sessionRef.session.getUserSocialIds()
})
])
const pipeline = workspace.pipeline instanceof Promise ? await workspace.pipeline : workspace.pipeline
if (this.doHandleTick) {
@@ -967,6 +1009,8 @@ export class TSessionManager implements SessionManager {
if (LOGGING_ENABLED) {
this.ctx.warn('Closed workspace', logParams)
}
await this.workspaceProducer.send(workspace.workspaceUuid, [workspaceEvents.down()])
}
} catch (err: any) {
Analytics.handleError(err)
@@ -1266,7 +1310,8 @@ export function createSessionManager (
| undefined,
accountsUrl: string,
enableCompression: boolean,
doHandleTick: boolean = true
doHandleTick: boolean = true,
queue: PlatformQueue
): SessionManager {
return new TSessionManager(
ctx,
@@ -1276,7 +1321,8 @@ export function createSessionManager (
profiling,
accountsUrl,
enableCompression,
doHandleTick
doHandleTick,
queue
)
}
@@ -1298,6 +1344,7 @@ export function startSessionManager (
start: () => void
stop: () => Promise<string | undefined>
}
queue: PlatformQueue
} & Partial<Timeouts>
): { shutdown: () => Promise<void>, sessionManager: SessionManager } {
const sessions = createSessionManager(
@@ -1310,7 +1357,9 @@ export function startSessionManager (
},
opt.profiling,
opt.accountsUrl,
opt.enableCompression ?? false
opt.enableCompression ?? false,
true,
opt.queue
)
return {
shutdown: opt.serverFactory(
server/workspace-service/src/index.ts
+3
View File
@@ -25,6 +25,7 @@ import {
import { type MigrateOperation } from '@hcengineering/model'
import { setMetadata } from '@hcengineering/platform'
import serverClientPlugin from '@hcengineering/server-client'
import { QueueTopic, type PlatformQueue, type QueueWorkspaceMessage } from '@hcengineering/server-core'
import serverNotification from '@hcengineering/server-notification'
import { createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import serverToken from '@hcengineering/server-token'
@@ -37,6 +38,7 @@ export * from './ws-operations'
*/
export function serveWorkspaceAccount (
measureCtx: MeasureContext,
queue: PlatformQueue,
version: Data<Version>,
txes: Tx[],
migrateOperations: [string, MigrateOperation][],
@@ -122,6 +124,7 @@ export function serveWorkspaceAccount (
let canceled = false
const worker = new WorkspaceWorker(
queue.createProducer<QueueWorkspaceMessage>(measureCtx, QueueTopic.Workspace),
version,
txes,
migrateOperations,
server/workspace-service/src/service.ts
+18 -29
View File
@@ -14,13 +14,13 @@
//
import { getClient as getAccountClient } from '@hcengineering/account-client'
import {
DOMAIN_BLOB,
getBranding,
isArchivingMode,
isMigrationMode,
isRestoringMode,
systemAccountUuid,
type BrandingMap,
DOMAIN_BLOB,
type Data,
type MeasureContext,
type Tx,
@@ -53,7 +53,13 @@ import {
shutdownPostgres
} from '@hcengineering/postgres'
import { doBackupWorkspace, doRestoreWorkspace } from '@hcengineering/server-backup'
import type { PipelineFactory, StorageAdapter } from '@hcengineering/server-core'
import {
workspaceEvents,
type PipelineFactory,
type PlatformQueueProducer,
type QueueWorkspaceMessage,
type StorageAdapter
} from '@hcengineering/server-core'
import {
createBackupPipeline,
getConfig,
@@ -101,6 +107,7 @@ export class WorkspaceWorker {
resolveBusy: (() => void) | null = null
constructor (
readonly workspaceQueue: PlatformQueueProducer<QueueWorkspaceMessage>,
readonly version: Data<Version>,
readonly txes: Tx[],
readonly migrationOperation: [string, MigrateOperation][],
@@ -270,6 +277,8 @@ export class WorkspaceWorker {
region: this.region,
time: Date.now() - t
})
await this.workspaceQueue.send(ws.uuid, [workspaceEvents.created()])
} catch (err: any) {
await opt.errorHandler(ws, err)
@@ -285,6 +294,7 @@ export class WorkspaceWorker {
region: this.region,
time: Date.now() - t
})
await this.workspaceQueue.send(ws.uuid, [workspaceEvents.createFailed()])
} finally {
if (!opt.console) {
;(logger as FileModelLogger).close()
@@ -365,6 +375,7 @@ export class WorkspaceWorker {
region: this.region,
time: Date.now() - t
})
await this.workspaceQueue.send(ws.uuid, [workspaceEvents.upgraded()])
} catch (err: any) {
await opt.errorHandler(ws, err)
@@ -381,6 +392,7 @@ export class WorkspaceWorker {
region: this.region,
time: Date.now() - t
})
await this.workspaceQueue.send(ws.uuid, [workspaceEvents.upgradeFailed()])
} finally {
if (!opt.console) {
;(logger as FileModelLogger).close()
@@ -396,32 +408,7 @@ export class WorkspaceWorker {
const adapter = getWorkspaceDestroyAdapter(dbUrl)
await adapter.deleteWorkspace(ctx, sharedPipelineContextVars, workspace.uuid, workspace.dataId)
await this.doReindexFulltext(ctx, workspace, cleanIndexes)
}
private async doReindexFulltext (
ctx: MeasureContext,
workspace: WorkspaceInfoWithStatus,
cleanIndexes: boolean
): Promise<void> {
if (this.fulltextUrl !== undefined) {
const token = generateToken(systemAccountUuid, workspace.uuid, { service: 'workspace' })
try {
const res = await fetch(this.fulltextUrl + '/api/v1/reindex', {
method: 'PUT',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ token, onlyDrop: cleanIndexes })
})
if (!res.ok) {
throw new Error(`HTTP Error ${res.status} ${res.statusText}`)
}
} catch (err: any) {
ctx.error('failed to reset index', { err })
}
}
await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.clearIndex()])
}
async sendTransactorMaitenance (token: string, ws: WorkspaceUuid): Promise<void> {
@@ -490,6 +477,7 @@ export class WorkspaceWorker {
return
}
await sendEvent('archiving-clean-done', 100)
await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.archived()])
break
}
case 'pending-deletion':
@@ -504,6 +492,7 @@ export class WorkspaceWorker {
return
}
await sendEvent('delete-done', 100)
await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.deleted()])
break
}
@@ -537,8 +526,8 @@ export class WorkspaceWorker {
await sendEvent('restore-started', 0)
if (await this.doRestore(ctx, workspace, opt)) {
// We should reindex fulltext
await this.doReindexFulltext(ctx, workspace, false)
await sendEvent('restore-done', 100)
await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.restored()])
}
break
default:
server/ws/src/__tests__/server.test.ts
+5 -3
View File
@@ -41,7 +41,7 @@ import {
type WorkspaceUuid
} from '@hcengineering/core'
import { ClientSession, startSessionManager } from '@hcengineering/server'
import { createDummyStorageAdapter } from '@hcengineering/server-core'
import { createDummyQueue, createDummyStorageAdapter } from '@hcengineering/server-core'
import { startHttpServer } from '../server_http'
import { genMinModel } from './minmodel'
@@ -101,7 +101,8 @@ describe('server', () => {
brandingMap: {},
serverFactory: startHttpServer,
accountsUrl: '',
externalStorage: createDummyStorageAdapter()
externalStorage: createDummyStorageAdapter(),
queue: createDummyQueue()
})
function connect (): WebSocket {
@@ -212,7 +213,8 @@ describe('server', () => {
brandingMap: {},
serverFactory: startHttpServer,
accountsUrl: '',
externalStorage: createDummyStorageAdapter()
externalStorage: createDummyStorageAdapter(),
queue: createDummyQueue()
})
async function findClose (token: string, timeoutPromise: Promise<void>, code: number): Promise<string> {
server/ws/src/server_http.ts
+2 -2
View File
@@ -193,7 +193,7 @@ export function startHttpServer (
app.get('/api/v1/profiling', (req, res) => {
try {
const token = req.query.token as string
const token = (req.query.token as string) ?? (req.headers.authorization ?? '').split(' ')[1]
decodeToken(token)
const jsonData = {
profiling
@@ -407,7 +407,7 @@ export function startHttpServer (
app.put('/api/v1/broadcast', (req, res) => {
try {
const token = req.query.token as string
const token = (req.query.token as string) ?? (req.headers.authorization ?? '').split(' ')[1]
decodeToken(token)
const ws = sessions.workspaces.get(req.query.workspace as WorkspaceUuid)
if (ws !== undefined) {

Some files were not shown because too many files have changed in this diff Show More