use redpanda queue (#8291)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev
2025-03-22 13:55:46 +05:00
committed by GitHub
parent 1f82104933
commit 58eb205688
143 changed files with 2288 additions and 2018 deletions
+6
View File
@@ -228,6 +228,9 @@ jobs:
run: node common/scripts/install-run-rush.js docker run: node common/scripts/install-run-rush.js docker
env: env:
DOCKER_CLI_HINTS: false DOCKER_CLI_HINTS: false
- name: Configure /etc/hosts
run: |
sudo echo "127.0.0.1 huly.local" | sudo tee -a /etc/hosts
- name: Prepare server - name: Prepare server
run: | run: |
cd ./tests cd ./tests
@@ -343,6 +346,9 @@ jobs:
run: node common/scripts/install-run-rush.js docker run: node common/scripts/install-run-rush.js docker
env: env:
DOCKER_CLI_HINTS: false DOCKER_CLI_HINTS: false
- name: Configure /etc/hosts
run: |
sudo echo "127.0.0.1 huly.local" | sudo tee -a /etc/hosts
- name: Prepare server - name: Prepare server
run: | run: |
cd ./tests cd ./tests
+59 -26
View File
@@ -11,9 +11,7 @@
"port": 9229, "port": 9229,
"request": "attach", "request": "attach",
"sourceMaps": true, "sourceMaps": true,
"skipFiles": [ "skipFiles": ["<node_internals>/**"],
"<node_internals>/**"
],
"type": "node" "type": "node"
}, },
{ {
@@ -25,7 +23,7 @@
"console": "integratedTerminal", "console": "integratedTerminal",
"justMyCode": true, "justMyCode": true,
"env": { "env": {
"PYTORCH_ENABLE_MPS_FALLBACK":"1" "PYTORCH_ENABLE_MPS_FALLBACK": "1"
} }
}, },
{ {
@@ -54,7 +52,7 @@
"ACCOUNTS_URL": "http://localhost:3000", "ACCOUNTS_URL": "http://localhost:3000",
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
// "SERVER_PROVIDER":"uweb" // "SERVER_PROVIDER":"uweb"
"SERVER_PROVIDER":"ws", "SERVER_PROVIDER": "ws",
"MODEL_VERSION": "0.7.1", "MODEL_VERSION": "0.7.1",
// "VERSION": "0.6.289", // "VERSION": "0.6.289",
"ELASTIC_INDEX_NAME": "local_storage_index", "ELASTIC_INDEX_NAME": "local_storage_index",
@@ -97,13 +95,14 @@
"ACCOUNTS_URL": "http://localhost:3000", "ACCOUNTS_URL": "http://localhost:3000",
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
// "SERVER_PROVIDER":"uweb" // "SERVER_PROVIDER":"uweb"
"SERVER_PROVIDER":"ws", "SERVER_PROVIDER": "ws",
"MODEL_VERSION": "0.7.1", "MODEL_VERSION": "0.7.24",
// "VERSION": "0.6.289", // "VERSION": "0.6.289",
"ELASTIC_INDEX_NAME": "local_storage_index", "ELASTIC_INDEX_NAME": "local_storage_index",
"UPLOAD_URL": "/files", "UPLOAD_URL": "/files",
"AI_BOT_URL": "http://localhost:4010", "AI_BOT_URL": "http://localhost:4010",
"STATS_URL": "http://huly.local:4900" "STATS_URL": "http://huly.local:4900",
"QUEUE_CONFIG": "localhost:19092"
}, },
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"runtimeVersion": "20", "runtimeVersion": "20",
@@ -163,8 +162,10 @@
"REKONI_URL": "http://localhost:4004", "REKONI_URL": "http://localhost:4004",
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
"ELASTIC_INDEX_NAME": "local_storage_index", "ELASTIC_INDEX_NAME": "local_storage_index",
"STATS_URL":"http://huly.local:4900", "REGION": "cockroach",
"STATS_URL": "http://huly.local:4900",
"ACCOUNTS_URL": "http://localhost:3000", "ACCOUNTS_URL": "http://localhost:3000",
"QUEUE_CONFIG": "localhost:19092"
}, },
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"runtimeVersion": "20", "runtimeVersion": "20",
@@ -184,7 +185,7 @@
"DB_URL": "mongodb://localhost:27017", "DB_URL": "mongodb://localhost:27017",
// "DB_URL": "postgresql://root@huly.local:26257/defaultdb?sslmode=disable", // "DB_URL": "postgresql://root@huly.local:26257/defaultdb?sslmode=disable",
"SERVER_SECRET": "secret", "SERVER_SECRET": "secret",
"REGION_INFO":"|Mongo;cockroach|CockroachDB", "REGION_INFO": "|Mongo;cockroach|CockroachDB",
"TRANSACTOR_URL": "ws://huly.local:3333,ws://huly.local:3332;;cockroach", "TRANSACTOR_URL": "ws://huly.local:3333,ws://huly.local:3332;;cockroach",
"ACCOUNTS_URL": "http://localhost:3000", "ACCOUNTS_URL": "http://localhost:3000",
"ACCOUNT_PORT": "3000", "ACCOUNT_PORT": "3000",
@@ -215,8 +216,8 @@
"DB_URL": "mongodb://localhost:27018", "DB_URL": "mongodb://localhost:27018",
// "DB_URL": "postgresql://postgres:example@localhost:5432", // "DB_URL": "postgresql://postgres:example@localhost:5432",
"SERVER_SECRET": "secret", "SERVER_SECRET": "secret",
"REGION_INFO":"|Mongo;pg|Postgres;cockroach|CockroachDB", "REGION_INFO": "|Mongo;pg|Postgres;cockroach|CockroachDB",
"TRANSACTOR_URL": "ws://huly.local:3334;;,ws://huly.local:3335;;europe", "TRANSACTOR_URL": "ws://huly.local:3334;;,ws://huly.local:3335;;europe",
"ACCOUNTS_URL": "http://localhost:3003", "ACCOUNTS_URL": "http://localhost:3003",
"ACCOUNT_PORT": "3003", "ACCOUNT_PORT": "3003",
"FRONT_URL": "http://localhost:8083", "FRONT_URL": "http://localhost:8083",
@@ -261,7 +262,7 @@
"DB_URL": "mongodb://localhost:27017", "DB_URL": "mongodb://localhost:27017",
// "DB_URL": "postgresql://postgres:example@localhost:5432", // "DB_URL": "postgresql://postgres:example@localhost:5432",
// "DB_URL": "postgresql://root@huly.local:26257/defaultdb?sslmode=disable", // "DB_URL": "postgresql://root@huly.local:26257/defaultdb?sslmode=disable",
"REGION": "", "REGION": "",
"SERVER_SECRET": "secret", "SERVER_SECRET": "secret",
"TRANSACTOR_URL": "ws://localhost:3333", "TRANSACTOR_URL": "ws://localhost:3333",
@@ -274,7 +275,7 @@
"MODEL_VERSION": "v0.7.1", "MODEL_VERSION": "v0.7.1",
"WS_OPERATION": "all+backup", "WS_OPERATION": "all+backup",
"BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin", "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
"BACKUP_BUCKET": "dev-backups", "BACKUP_BUCKET": "dev-backups"
// "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init", // "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init",
// "INIT_WORKSPACE": "staging-dev" // "INIT_WORKSPACE": "staging-dev"
}, },
@@ -307,7 +308,7 @@
"MODEL_VERSION": "0.7.1", "MODEL_VERSION": "0.7.1",
"WS_OPERATION": "all+backup", "WS_OPERATION": "all+backup",
"BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin", "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
"BACKUP_BUCKET": "dev-backups", "BACKUP_BUCKET": "dev-backups"
// "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init", // "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init",
// "INIT_WORKSPACE": "staging-dev" // "INIT_WORKSPACE": "staging-dev"
}, },
@@ -447,7 +448,7 @@
"name": "Debug tool upgrade PG(Cockroach)", "name": "Debug tool upgrade PG(Cockroach)",
"type": "node", "type": "node",
"request": "launch", "request": "launch",
"args": ["src/__start.ts", "upgrade-workspace", "platform" ], "args": ["src/__start.ts", "upgrade-workspace", "platform"],
"env": { "env": {
"SERVER_SECRET": "secret", "SERVER_SECRET": "secret",
"MINIO_ACCESS_KEY": "minioadmin", "MINIO_ACCESS_KEY": "minioadmin",
@@ -505,8 +506,8 @@
"MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json",
"SECRET": "secret", "SECRET": "secret",
"REGION": "cockroach", "REGION": "cockroach",
"BUCKET_NAME":"backups", "BUCKET_NAME": "backups",
"INTERVAL":"0" "INTERVAL": "0"
}, },
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"showAsyncStacks": true, "showAsyncStacks": true,
@@ -528,7 +529,7 @@
"DB_URL": "mongodb://localhost:27017", "DB_URL": "mongodb://localhost:27017",
"SERVER_SECRET": "secret", "SERVER_SECRET": "secret",
"SECRET": "secret", "SECRET": "secret",
"BUCKET_NAME":"backups" "BUCKET_NAME": "backups"
}, },
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"showAsyncStacks": true, "showAsyncStacks": true,
@@ -548,7 +549,7 @@
// "DB_URL": "mongodb://localhost:27017", // "DB_URL": "mongodb://localhost:27017",
"DB_URL": "postgresql://root@host.docker.internal:26258/defaultdb?sslmode=disable", "DB_URL": "postgresql://root@host.docker.internal:26258/defaultdb?sslmode=disable",
"SERVER_SECRET": "secret", "SERVER_SECRET": "secret",
"BUCKET_NAME":"dev-backups" "BUCKET_NAME": "dev-backups"
}, },
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"showAsyncStacks": true, "showAsyncStacks": true,
@@ -714,7 +715,7 @@
"AVATAR_PATH": "./assets/avatar.png", "AVATAR_PATH": "./assets/avatar.png",
"AVATAR_CONTENT_TYPE": ".png", "AVATAR_CONTENT_TYPE": ".png",
"STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin", "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
"LOVE_ENDPOINT": "http://localhost:8096", "LOVE_ENDPOINT": "http://localhost:8096"
}, },
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"sourceMaps": true, "sourceMaps": true,
@@ -771,14 +772,26 @@
"urlFilter": "http://localhost:8080/*", "urlFilter": "http://localhost:8080/*",
"port": 9222, "port": 9222,
"sourceMapPathOverrides": { "sourceMapPathOverrides": {
"webpack://*": "${workspaceFolder}/*", "webpack://*": "${workspaceFolder}/*"
} }
}, },
{ {
"name": "Debug Notion import", "name": "Debug Notion import",
"type": "node", "type": "node",
"request": "launch", "request": "launch",
"args": ["src/__start.ts", "import-notion-to-teamspace", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-u", "user1", "-pw", "1234", "-ws", "ws5", "-ts", "notion", ], "args": [
"src/__start.ts",
"import-notion-to-teamspace",
"/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743",
"-u",
"user1",
"-pw",
"1234",
"-ws",
"ws5",
"-ts",
"notion"
],
// "args": ["src/__start.ts", "import-notion-with-teamspaces", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-u", "user1", "-pw", "1234", "-ws", "ws1"], // "args": ["src/__start.ts", "import-notion-with-teamspaces", "/home/anna/work/notion/natalya/Export-fad9ecb4-a1a5-4623-920d-df32dd423743", "-u", "user1", "-pw", "1234", "-ws", "ws1"],
"env": { "env": {
"FRONT_URL": "http://localhost:8087" "FRONT_URL": "http://localhost:8087"
@@ -793,9 +806,19 @@
"name": "Debug ClickUp import", "name": "Debug ClickUp import",
"type": "node", "type": "node",
"request": "launch", "request": "launch",
"args": ["src/__start.ts", "import-clickup-tasks", "/home/anna/work/clickup/aleksandr/debug/mentions.csv", "-u", "user1", "-pw", "1234", "-ws", "ws10"], "args": [
"src/__start.ts",
"import-clickup-tasks",
"/home/anna/work/clickup/aleksandr/debug/mentions.csv",
"-u",
"user1",
"-pw",
"1234",
"-ws",
"ws10"
],
"env": { "env": {
"FRONT_URL": "http://localhost:8087", "FRONT_URL": "http://localhost:8087"
}, },
"runtimeVersion": "20", "runtimeVersion": "20",
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
@@ -807,7 +830,17 @@
"name": "Debug Huly import", "name": "Debug Huly import",
"type": "node", "type": "node",
"request": "launch", "request": "launch",
"args": ["src/__start.ts", "import", "/home/anna/huly/platform/dev/import-tool/docs/huly/example-workspace", "-u", "user1", "-pw", "1234", "-ws", "ws1"], "args": [
"src/__start.ts",
"import",
"/home/anna/huly/platform/dev/import-tool/docs/huly/example-workspace",
"-u",
"user1",
"-pw",
"1234",
"-ws",
"ws1"
],
"env": { "env": {
"FRONT_URL": "http://localhost:8087" "FRONT_URL": "http://localhost:8087"
}, },
+1
View File
@@ -159,6 +159,7 @@ Add the following line to your /etc/hosts file
```plain ```plain
127.0.0.1 huly.local 127.0.0.1 huly.local
::1 huly.local
``` ```
Accessing the URL <http://huly.local:8087> will lead you to the app in development mode. Accessing the URL <http://huly.local:8087> will lead you to the app in development mode.
+108 -151
View File
@@ -316,6 +316,9 @@ importers:
'@rush-temp/inventory-resources': '@rush-temp/inventory-resources':
specifier: file:./projects/inventory-resources.tgz specifier: file:./projects/inventory-resources.tgz
version: file:projects/inventory-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) version: file:projects/inventory-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/kafka':
specifier: file:./projects/kafka.tgz
version: file:projects/kafka.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/kanban': '@rush-temp/kanban':
specifier: file:./projects/kanban.tgz specifier: file:./projects/kanban.tgz
version: file:projects/kanban.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) version: file:projects/kanban.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
@@ -508,9 +511,6 @@ importers:
'@rush-temp/model-server-drive': '@rush-temp/model-server-drive':
specifier: file:./projects/model-server-drive.tgz specifier: file:./projects/model-server-drive.tgz
version: file:projects/model-server-drive.tgz version: file:projects/model-server-drive.tgz
'@rush-temp/model-server-fulltext':
specifier: file:./projects/model-server-fulltext.tgz
version: file:projects/model-server-fulltext.tgz
'@rush-temp/model-server-gmail': '@rush-temp/model-server-gmail':
specifier: file:./projects/model-server-gmail.tgz specifier: file:./projects/model-server-gmail.tgz
version: file:projects/model-server-gmail.tgz version: file:projects/model-server-gmail.tgz
@@ -922,12 +922,6 @@ importers:
'@rush-temp/server-drive-resources': '@rush-temp/server-drive-resources':
specifier: file:./projects/server-drive-resources.tgz specifier: file:./projects/server-drive-resources.tgz
version: file:projects/server-drive-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) version: file:projects/server-drive-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/server-fulltext':
specifier: file:./projects/server-fulltext.tgz
version: file:projects/server-fulltext.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/server-fulltext-resources':
specifier: file:./projects/server-fulltext-resources.tgz
version: file:projects/server-fulltext-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/server-github': '@rush-temp/server-github':
specifier: file:./projects/server-github.tgz specifier: file:./projects/server-github.tgz
version: file:projects/server-github.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) version: file:projects/server-github.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
@@ -1786,6 +1780,9 @@ importers:
jwt-simple: jwt-simple:
specifier: ^0.5.6 specifier: ^0.5.6
version: 0.5.6 version: 0.5.6
kafkajs:
specifier: ^2.2.4
version: 2.2.4
koa: koa:
specifier: ^2.15.4 specifier: ^2.15.4
version: 2.16.0 version: 2.16.0
@@ -3830,7 +3827,7 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/ai-bot-resources@file:projects/ai-bot-resources.tgz': '@rush-temp/ai-bot-resources@file:projects/ai-bot-resources.tgz':
resolution: {integrity: sha512-6E8sBtpn6OP/Vpgmf02OPnglOUBfDvB0iZFmvu6Dp1NhTir70tm8kV8IBCqU4MadnD4f636sZwKEItk4NXv+Rw==, tarball: file:projects/ai-bot-resources.tgz} resolution: {integrity: sha512-F9ehCKoqDjVOCtNkYkhL5J8f2a6LVuZZlFjnlNllq3cdEv9D5s8q17KREQnVjEwdXK6mPrUbsWaDjNDpyd8Bug==, tarball: file:projects/ai-bot-resources.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/ai-bot@file:projects/ai-bot.tgz': '@rush-temp/ai-bot@file:projects/ai-bot.tgz':
@@ -4153,6 +4150,10 @@ packages:
resolution: {integrity: sha512-FVCeKLDYhO53Xnhz0Q/64nU6cMcGaQWrF8jvheuF14ZhPevDUaQfRMwnLR4RGBzIhiTc9jvUIUM54VI1FSkoAA==, tarball: file:projects/inventory.tgz} resolution: {integrity: sha512-FVCeKLDYhO53Xnhz0Q/64nU6cMcGaQWrF8jvheuF14ZhPevDUaQfRMwnLR4RGBzIhiTc9jvUIUM54VI1FSkoAA==, tarball: file:projects/inventory.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/kafka@file:projects/kafka.tgz':
resolution: {integrity: sha512-Q5+NPNp08oBRTGukVK1HY4Bx/XI3TSnbIY2TIhi4HhU9etcWhHL2EGcsmWhFRfG0RC2gcXC9LI3prx9HDc2kEw==, tarball: file:projects/kafka.tgz}
version: 0.0.0
'@rush-temp/kanban@file:projects/kanban.tgz': '@rush-temp/kanban@file:projects/kanban.tgz':
resolution: {integrity: sha512-a1aAx7zBxeZjaejI86Ho69hHcEm7F4dkdD9rVCv2BVHzjG/1C8mq7pv+QWHFGoxkiQwwm9MatVV4IMCSjDjAZw==, tarball: file:projects/kanban.tgz} resolution: {integrity: sha512-a1aAx7zBxeZjaejI86Ho69hHcEm7F4dkdD9rVCv2BVHzjG/1C8mq7pv+QWHFGoxkiQwwm9MatVV4IMCSjDjAZw==, tarball: file:projects/kanban.tgz}
version: 0.0.0 version: 0.0.0
@@ -4222,7 +4223,7 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/model-all@file:projects/model-all.tgz': '@rush-temp/model-all@file:projects/model-all.tgz':
resolution: {integrity: sha512-G7I5APyAySX8V9/mzUjuXX3LGtRNZ+uYQhHTl4/7ulGaAsuSagzDJIvClO2akY+GhplEfRrKBiI2fb0R0/i30g==, tarball: file:projects/model-all.tgz} resolution: {integrity: sha512-UxCRw3u19dTizt2hYLWPNCJ468ntnLnwEtP/8mAEb1Ta93SbPgbV9RfCK+hBBM/vp843HVwdNrab1fwqFXb+3Q==, tarball: file:projects/model-all.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/model-analytics-collector@file:projects/model-analytics-collector.tgz': '@rush-temp/model-analytics-collector@file:projects/model-analytics-collector.tgz':
@@ -4405,10 +4406,6 @@ packages:
resolution: {integrity: sha512-MQUIWrqoJZkFbbKhT569m2pfvCJsK7l3rVWhO9DlNfxFmItei5ZSsMzC7+4qpI7iyQsb620y23RkhEl17/9YKQ==, tarball: file:projects/model-server-drive.tgz} resolution: {integrity: sha512-MQUIWrqoJZkFbbKhT569m2pfvCJsK7l3rVWhO9DlNfxFmItei5ZSsMzC7+4qpI7iyQsb620y23RkhEl17/9YKQ==, tarball: file:projects/model-server-drive.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/model-server-fulltext@file:projects/model-server-fulltext.tgz':
resolution: {integrity: sha512-qOs1G3E2P64j6xsyD1w2+ZPLclBeAyx1xmGEI19kneyA8IB8NNRcpI5GIB9vIrYJ2lJUi0jyI+sHkLoxmgtBaA==, tarball: file:projects/model-server-fulltext.tgz}
version: 0.0.0
'@rush-temp/model-server-gmail@file:projects/model-server-gmail.tgz': '@rush-temp/model-server-gmail@file:projects/model-server-gmail.tgz':
resolution: {integrity: sha512-vdwW7KKXUWLji9gEb2uEGwbjyJfeQr2g4G/vhZH15sy87rI5f7+M7x5p7zZq5IOF9dk6YArAbILD/yl7u+E3Xg==, tarball: file:projects/model-server-gmail.tgz} resolution: {integrity: sha512-vdwW7KKXUWLji9gEb2uEGwbjyJfeQr2g4G/vhZH15sy87rI5f7+M7x5p7zZq5IOF9dk6YArAbILD/yl7u+E3Xg==, tarball: file:projects/model-server-gmail.tgz}
version: 0.0.0 version: 0.0.0
@@ -4614,7 +4611,7 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-ai-bot@file:projects/pod-ai-bot.tgz': '@rush-temp/pod-ai-bot@file:projects/pod-ai-bot.tgz':
resolution: {integrity: sha512-sv9r+DDafpUyjHUabAnULCjQC2xnZ8t4AfhIhvXBM6vl500IrhBVkeAGcNeIk0WYepfAnRn4mQVM3yX+0e0Ygw==, tarball: file:projects/pod-ai-bot.tgz} resolution: {integrity: sha512-iYWt2dpdc5lKElhJEc89Zar3zq7+qXEKsTAlVqFxMtAxwCgDIQqTvds5b/W5FEvgfJsmXUN/Oib9zBr2urn6Fg==, tarball: file:projects/pod-ai-bot.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-analytics-collector@file:projects/pod-analytics-collector.tgz': '@rush-temp/pod-analytics-collector@file:projects/pod-analytics-collector.tgz':
@@ -4646,7 +4643,7 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-fulltext@file:projects/pod-fulltext.tgz': '@rush-temp/pod-fulltext@file:projects/pod-fulltext.tgz':
resolution: {integrity: sha512-rDotFbiaTJUlFISEWa7+hd74cQWXGijcNrq3qeAItyw4A2PVKLSQusomUp5Mo1oP0wIurREXjiKCN1I7I27eAA==, tarball: file:projects/pod-fulltext.tgz} resolution: {integrity: sha512-za9dXz8LTdV3Mwyr3Lbno6Ys7c0tKu2F9+QH0D75THvniEMYUK1mG4Pt/7hByNzTB1VqzIaozs6Id+DfXYqS2Q==, tarball: file:projects/pod-fulltext.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-github@file:projects/pod-github.tgz': '@rush-temp/pod-github@file:projects/pod-github.tgz':
@@ -4654,7 +4651,7 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-gmail@file:projects/pod-gmail.tgz': '@rush-temp/pod-gmail@file:projects/pod-gmail.tgz':
resolution: {integrity: sha512-aYecRf97Yj23bi2BdWXg4DrH4oy3AB2UfrpbwNo6nvLhHJLsrcRgOe9zCTakZ7Yb+nDzsxYuOwZiDIr9aWtsyw==, tarball: file:projects/pod-gmail.tgz} resolution: {integrity: sha512-HyyHadfmgT1OP7pdln0WJvK6cGPTYVVvxq21JBHKshu40WojCB6dLdAch/YrTO0aSS7UzluuLmYnnCn5JGvfcg==, tarball: file:projects/pod-gmail.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-hook@file:projects/pod-hook.tgz': '@rush-temp/pod-hook@file:projects/pod-hook.tgz':
@@ -4662,7 +4659,7 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-love@file:projects/pod-love.tgz': '@rush-temp/pod-love@file:projects/pod-love.tgz':
resolution: {integrity: sha512-m+9dPMR45FJpQB+M5ka4PiJcTpV8COlQA5F36is2LkMLwnsDjcB94Sx+TFdIyjhVFpfB8l+t886mSPveHql4ag==, tarball: file:projects/pod-love.tgz} resolution: {integrity: sha512-K9Hvwm8QesWBTSx/A+swmRDUOy0RY8y4tNJkrW85Zh5wMUxHzeYrX/tQL+usk+cRrvPCw9C8yag1gxsKqPkWIQ==, tarball: file:projects/pod-love.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-mail@file:projects/pod-mail.tgz': '@rush-temp/pod-mail@file:projects/pod-mail.tgz':
@@ -4670,11 +4667,11 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-print@file:projects/pod-print.tgz': '@rush-temp/pod-print@file:projects/pod-print.tgz':
resolution: {integrity: sha512-i0Y9ShIeieNqCNMCXax1CZTStiDTw9M156m7fCDSrNS6VV4v2e5jy/HyNLtpyXz/ZRlOKndBdyiKsls+5pDL8g==, tarball: file:projects/pod-print.tgz} resolution: {integrity: sha512-KNZrBeofhGFWQHMd1aPU7smmL50h/fGw4tXy7bEV2PcO14vphe2MyMfuWcTw1ksmeXFP6V3C2XxmgKFk+DT0dg==, tarball: file:projects/pod-print.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-server@file:projects/pod-server.tgz': '@rush-temp/pod-server@file:projects/pod-server.tgz':
resolution: {integrity: sha512-MJ5TQobQy4kv67qy7qpQgz056fhaZw8Zs/NaXniFafG/xjstgWeIJCXNHDkKvdBm+wT6EdHgNLIN3BaRpMSDsA==, tarball: file:projects/pod-server.tgz} resolution: {integrity: sha512-toWgesqQWTYgkwA6VPDiaJJc7ZmDkHFzNg/MbDjSACtmy0/endv3ioglyXYdoDo/m4IbFZOghMUJOZbQgQvRUA==, tarball: file:projects/pod-server.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-ses@file:projects/pod-ses.tgz': '@rush-temp/pod-ses@file:projects/pod-ses.tgz':
@@ -4682,7 +4679,7 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-sign@file:projects/pod-sign.tgz': '@rush-temp/pod-sign@file:projects/pod-sign.tgz':
resolution: {integrity: sha512-CmOWryDSvF5qg0NTlkxT29gZP0qj0DimD25X2U6sQGtowtEytNvVmmg3cuSNTpwN+WR6+eli4JWyK6gathi+fQ==, tarball: file:projects/pod-sign.tgz} resolution: {integrity: sha512-7ERrvIDW3r+NlglIYXW8kK50EiYLL9za+/EON9pW/8u0gkIaPIjEfL6yvdAh3euZ+khgPkzH+jrV57tu+jxT4w==, tarball: file:projects/pod-sign.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-stats@file:projects/pod-stats.tgz': '@rush-temp/pod-stats@file:projects/pod-stats.tgz':
@@ -4694,11 +4691,11 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-telegram@file:projects/pod-telegram.tgz': '@rush-temp/pod-telegram@file:projects/pod-telegram.tgz':
resolution: {integrity: sha512-FAYbl5GmRCS3fOR5eUXKNS8KmwnwB+7ZZ68vDi5lfeH4hkJky/VdR5yq4dX5SzOB35e7hz9JWriDGxfjUcx6Wg==, tarball: file:projects/pod-telegram.tgz} resolution: {integrity: sha512-CtJJaVgL0u4w9JUOAXdnAJqUnpSIlIpa2aY7OU3vZPR6hLqXt5pmDFvq3siQzkZ+27mJ6vc+IOBAaqbmumOkXg==, tarball: file:projects/pod-telegram.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/pod-workspace@file:projects/pod-workspace.tgz': '@rush-temp/pod-workspace@file:projects/pod-workspace.tgz':
resolution: {integrity: sha512-6wRqrWl0oZUzQ2CZBpPK09doQpYepraNAwCO3OcPDaRnDfQo0WBwLvfWOdDVW3REunCCh/DHMxGpuiLnaQX4KQ==, tarball: file:projects/pod-workspace.tgz} resolution: {integrity: sha512-QrThk+6HwkGIdmj2tiPv9LTIOe5+QJChsG9DtIzmxaeyOjnhtpX2ROEuaVXL3oXl97EBoXX8fBsVpDMQ4i+pRw==, tarball: file:projects/pod-workspace.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/postgres@file:projects/postgres.tgz': '@rush-temp/postgres@file:projects/postgres.tgz':
@@ -4818,7 +4815,7 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/rekoni-service@file:projects/rekoni-service.tgz': '@rush-temp/rekoni-service@file:projects/rekoni-service.tgz':
resolution: {integrity: sha512-drU6icafREaUYcSZfNn/CEYQLJC1ha6FkF7QrRcQ/TsuFaruapqmET0Ttqcr/vQaYemKF6SnshiKRGmeg+tcDA==, tarball: file:projects/rekoni-service.tgz} resolution: {integrity: sha512-O8bBBE0Bw2m69G/KVLXgHnfa2WzWn9MG1CwklzSf6rkyVT/ltUA7WXwdPEA/ZFjLbkRpgd2RmcVRSERlAXDmRw==, tarball: file:projects/rekoni-service.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/rekoni@file:projects/rekoni.tgz': '@rush-temp/rekoni@file:projects/rekoni.tgz':
@@ -4957,14 +4954,6 @@ packages:
resolution: {integrity: sha512-5zQ2vbqHHFW1gdEWDqV8aDGq5u2I4z9qoD4TH0QbBaE5XITQrvJoZfK9WW+CceePEM2O+1clTu9w1KTBIEd5Hg==, tarball: file:projects/server-drive.tgz} resolution: {integrity: sha512-5zQ2vbqHHFW1gdEWDqV8aDGq5u2I4z9qoD4TH0QbBaE5XITQrvJoZfK9WW+CceePEM2O+1clTu9w1KTBIEd5Hg==, tarball: file:projects/server-drive.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/server-fulltext-resources@file:projects/server-fulltext-resources.tgz':
resolution: {integrity: sha512-NebNMPNgIeG8N0qA5Q4ROcs5Jn/dmE0WX92uxw/qVUO/k8DLMxRsrFvyKrrMGtYgeivbxJPe6PEd3FijUAMfiw==, tarball: file:projects/server-fulltext-resources.tgz}
version: 0.0.0
'@rush-temp/server-fulltext@file:projects/server-fulltext.tgz':
resolution: {integrity: sha512-QYDVer6XKHqKih+CXofNkysDl9TafFIlN5sw4GdJjLT3/AJt7L2FeCKMuPgMtYQhihj2kodtBfNUbnSeLA8qJw==, tarball: file:projects/server-fulltext.tgz}
version: 0.0.0
'@rush-temp/server-github-model@file:projects/server-github-model.tgz': '@rush-temp/server-github-model@file:projects/server-github-model.tgz':
resolution: {integrity: sha512-U7kynRwConx+S1nLfFluVkWJtFKCEMJRIHx/oN1OFy2zmIGSw1o7z1NIgEYAiYogj8zP1SPYoVKPQTSkSosKqw==, tarball: file:projects/server-github-model.tgz} resolution: {integrity: sha512-U7kynRwConx+S1nLfFluVkWJtFKCEMJRIHx/oN1OFy2zmIGSw1o7z1NIgEYAiYogj8zP1SPYoVKPQTSkSosKqw==, tarball: file:projects/server-github-model.tgz}
version: 0.0.0 version: 0.0.0
@@ -5038,7 +5027,7 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/server-pipeline@file:projects/server-pipeline.tgz': '@rush-temp/server-pipeline@file:projects/server-pipeline.tgz':
resolution: {integrity: sha512-egC06Gj9kZodIHe8m7g6K99KtOTXZyZJgPymrkDj+GGZX7+7Pd/5Nck0jT4i/Ufk1c0zOxEBXJE4+KTuvo/aew==, tarball: file:projects/server-pipeline.tgz} resolution: {integrity: sha512-LWUTB3SnKZRoKrdx1ywH5lu/7jjYyD71rffj73ITM9s0Y6G9rSVNF7R4otEwOEmD+ZniJlJTz8h2vqfmEuALEg==, tarball: file:projects/server-pipeline.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/server-preference@file:projects/server-preference.tgz': '@rush-temp/server-preference@file:projects/server-preference.tgz':
@@ -5318,7 +5307,7 @@ packages:
version: 0.0.0 version: 0.0.0
'@rush-temp/tool@file:projects/tool.tgz': '@rush-temp/tool@file:projects/tool.tgz':
resolution: {integrity: sha512-1/Aodec9LySBThOZAeA3XJfP83lSVTVRFdlb8uNTORq1sHJxCADx9BliCh2dTzdhydW46wcAxMPeArhopkcN1Q==, tarball: file:projects/tool.tgz} resolution: {integrity: sha512-zqh0++u5ETaUNlwkipP+8CeICqrFYEDBO4sAjFRyLPd1SHZbanUpQRAg58t4C+Z7q4Owqr149ee/hCGWPult3w==, tarball: file:projects/tool.tgz}
version: 0.0.0 version: 0.0.0
'@rush-temp/tracker-assets@file:projects/tracker-assets.tgz': '@rush-temp/tracker-assets@file:projects/tracker-assets.tgz':
@@ -10075,6 +10064,10 @@ packages:
resolution: {integrity: sha512-40aUybvhH9t2h71ncA1/1SbtTNCVZHgsTsTgqPUxGWDmUDrXyDf2wMNQKEbdBjbf4AI+fQhbECNTV6lWxQKUzg==} resolution: {integrity: sha512-40aUybvhH9t2h71ncA1/1SbtTNCVZHgsTsTgqPUxGWDmUDrXyDf2wMNQKEbdBjbf4AI+fQhbECNTV6lWxQKUzg==}
engines: {node: '>= 0.4.0'} engines: {node: '>= 0.4.0'}
kafkajs@2.2.4:
resolution: {integrity: sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==}
engines: {node: '>=14.0.0'}
katex@0.16.15: katex@0.16.15:
resolution: {integrity: sha512-yE9YJIEAk2aZ+FL/G8r+UGw0CTUzEA8ZFy6E+8tc3spHUKq3qBnzCkI1CQwGoI9atJhVyFPEypQsTY7mJ1Pi9w==} resolution: {integrity: sha512-yE9YJIEAk2aZ+FL/G8r+UGw0CTUzEA8ZFy6E+8tc3spHUKq3qBnzCkI1CQwGoI9atJhVyFPEypQsTY7mJ1Pi9w==}
hasBin: true hasBin: true
@@ -11170,8 +11163,8 @@ packages:
resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==}
engines: {node: '>=8'} engines: {node: '>=8'}
parse5@7.1.2: parse5@7.2.1:
resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==}
parseley@0.12.1: parseley@0.12.1:
resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==} resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==}
@@ -11481,8 +11474,8 @@ packages:
resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==}
engines: {node: '>=4'} engines: {node: '>=4'}
postgres-array@3.0.4: postgres-array@3.0.2:
resolution: {integrity: sha512-nAUSGfSDGOaOAEGwqsRY27GPOea7CNipJPOA7lPbdEpx5Kg3qzdP0AaWC5MlhTWV9s4hFX39nomVZ+C4tnGOJQ==} resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==}
engines: {node: '>=12'} engines: {node: '>=12'}
postgres-bytea@1.0.0: postgres-bytea@1.0.0:
@@ -16059,7 +16052,7 @@ snapshots:
'@puppeteer/browsers@2.3.0': '@puppeteer/browsers@2.3.0':
dependencies: dependencies:
debug: 4.4.0 debug: 4.3.5
extract-zip: 2.0.1 extract-zip: 2.0.1
progress: 2.0.3 progress: 2.0.3
proxy-agent: 6.4.0 proxy-agent: 6.4.0
@@ -18853,6 +18846,32 @@ snapshots:
- supports-color - supports-color
- ts-node - ts-node
'@rush-temp/kafka@file:projects/kafka.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
kafkajs: 2.2.4
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3)
typescript: 5.7.3
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
- babel-jest
- babel-plugin-macros
- esbuild
- node-notifier
- supports-color
- ts-node
'@rush-temp/kanban@file:projects/kanban.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': '@rush-temp/kanban@file:projects/kanban.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies: dependencies:
'@types/jest': 29.5.12 '@types/jest': 29.5.12
@@ -19179,17 +19198,17 @@ snapshots:
dependencies: dependencies:
'@types/jest': 29.5.12 '@types/jest': 29.5.12
'@types/node': 20.11.19 '@types/node': 20.11.19
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0 eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5 prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.7.3 typescript: 5.6.2
transitivePeerDependencies: transitivePeerDependencies:
- '@babel/core' - '@babel/core'
- '@jest/types' - '@jest/types'
@@ -19203,10 +19222,10 @@ snapshots:
'@rush-temp/mail-resources@file:projects/mail-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': '@rush-temp/mail-resources@file:projects/mail-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies: dependencies:
'@types/jest': 29.5.12 '@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0 eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0)
@@ -19219,9 +19238,9 @@ snapshots:
svelte-check: 3.6.9(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19) svelte-check: 3.6.9(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)
svelte-eslint-parser: 0.33.1(svelte@4.2.19) svelte-eslint-parser: 0.33.1(svelte@4.2.19)
svelte-loader: 3.2.0(svelte@4.2.19) svelte-loader: 3.2.0(svelte@4.2.19)
svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.7.3) svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.6.2)
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.7.3 typescript: 5.6.2
transitivePeerDependencies: transitivePeerDependencies:
- '@babel/core' - '@babel/core'
- '@jest/types' - '@jest/types'
@@ -19243,17 +19262,17 @@ snapshots:
'@rush-temp/mail@file:projects/mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': '@rush-temp/mail@file:projects/mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies: dependencies:
'@types/jest': 29.5.12 '@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0 eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5 prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.7.3 typescript: 5.6.2
transitivePeerDependencies: transitivePeerDependencies:
- '@babel/core' - '@babel/core'
- '@jest/types' - '@jest/types'
@@ -19650,29 +19669,29 @@ snapshots:
'@rush-temp/model-mail@file:projects/model-mail.tgz': '@rush-temp/model-mail@file:projects/model-mail.tgz':
dependencies: dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0 eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0)
prettier: 3.2.5 prettier: 3.2.5
typescript: 5.7.3 typescript: 5.6.2
transitivePeerDependencies: transitivePeerDependencies:
- supports-color - supports-color
'@rush-temp/model-my-space@file:projects/model-my-space.tgz': '@rush-temp/model-my-space@file:projects/model-my-space.tgz':
dependencies: dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0 eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0)
prettier: 3.2.5 prettier: 3.2.5
typescript: 5.7.3 typescript: 5.6.2
transitivePeerDependencies: transitivePeerDependencies:
- supports-color - supports-color
@@ -20017,20 +20036,6 @@ snapshots:
transitivePeerDependencies: transitivePeerDependencies:
- supports-color - supports-color
'@rush-temp/model-server-fulltext@file:projects/model-server-fulltext.tgz':
dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
prettier: 3.2.5
typescript: 5.6.2
transitivePeerDependencies:
- supports-color
'@rush-temp/model-server-gmail@file:projects/model-server-gmail.tgz': '@rush-temp/model-server-gmail@file:projects/model-server-gmail.tgz':
dependencies: dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
@@ -20601,17 +20606,17 @@ snapshots:
dependencies: dependencies:
'@types/jest': 29.5.12 '@types/jest': 29.5.12
'@types/node': 20.11.19 '@types/node': 20.11.19
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0 eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5 prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.7.3 typescript: 5.6.2
transitivePeerDependencies: transitivePeerDependencies:
- '@babel/core' - '@babel/core'
- '@jest/types' - '@jest/types'
@@ -20625,10 +20630,10 @@ snapshots:
'@rush-temp/my-space-resources@file:projects/my-space-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': '@rush-temp/my-space-resources@file:projects/my-space-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies: dependencies:
'@types/jest': 29.5.12 '@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0 eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0)
@@ -20642,9 +20647,9 @@ snapshots:
svelte-check: 3.6.9(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19) svelte-check: 3.6.9(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)
svelte-eslint-parser: 0.33.1(svelte@4.2.19) svelte-eslint-parser: 0.33.1(svelte@4.2.19)
svelte-loader: 3.2.0(svelte@4.2.19) svelte-loader: 3.2.0(svelte@4.2.19)
svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.7.3) svelte-preprocess: 5.1.3(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.4.35)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.4.35)(sass@1.71.1)(svelte@4.2.19)(typescript@5.6.2)
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.7.3 typescript: 5.6.2
transitivePeerDependencies: transitivePeerDependencies:
- '@babel/core' - '@babel/core'
- '@jest/types' - '@jest/types'
@@ -20666,17 +20671,17 @@ snapshots:
'@rush-temp/my-space@file:projects/my-space.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': '@rush-temp/my-space@file:projects/my-space.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies: dependencies:
'@types/jest': 29.5.12 '@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0 eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5 prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.7.3 typescript: 5.6.2
transitivePeerDependencies: transitivePeerDependencies:
- '@babel/core' - '@babel/core'
- '@jest/types' - '@jest/types'
@@ -23846,56 +23851,6 @@ snapshots:
- supports-color - supports-color
- ts-node - ts-node
'@rush-temp/server-fulltext-resources@file:projects/server-fulltext-resources.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.6.2
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
- '@types/node'
- babel-jest
- babel-plugin-macros
- esbuild
- node-notifier
- supports-color
- ts-node
'@rush-temp/server-fulltext@file:projects/server-fulltext.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.6.2)
typescript: 5.6.2
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
- babel-jest
- babel-plugin-macros
- esbuild
- node-notifier
- supports-color
- ts-node
'@rush-temp/server-github-model@file:projects/server-github-model.tgz': '@rush-temp/server-github-model@file:projects/server-github-model.tgz':
dependencies: dependencies:
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
@@ -28120,7 +28075,7 @@ snapshots:
dependencies: dependencies:
'@types/node': 20.11.19 '@types/node': 20.11.19
'@types/tough-cookie': 4.0.5 '@types/tough-cookie': 4.0.5
parse5: 7.1.2 parse5: 7.2.1
'@types/json-schema@7.0.15': {} '@types/json-schema@7.0.15': {}
@@ -32966,7 +32921,7 @@ snapshots:
https-proxy-agent: 5.0.1 https-proxy-agent: 5.0.1
is-potential-custom-element-name: 1.0.1 is-potential-custom-element-name: 1.0.1
nwsapi: 2.2.13 nwsapi: 2.2.13
parse5: 7.1.2 parse5: 7.2.1
saxes: 6.0.0 saxes: 6.0.0
symbol-tree: 3.2.4 symbol-tree: 3.2.4
tough-cookie: 4.1.4 tough-cookie: 4.1.4
@@ -33076,6 +33031,8 @@ snapshots:
jwt-simple@0.5.6: {} jwt-simple@0.5.6: {}
kafkajs@2.2.4: {}
katex@0.16.15: katex@0.16.15:
dependencies: dependencies:
commander: 8.3.0 commander: 8.3.0
@@ -34249,7 +34206,7 @@ snapshots:
json-parse-even-better-errors: 2.3.1 json-parse-even-better-errors: 2.3.1
lines-and-columns: 1.2.4 lines-and-columns: 1.2.4
parse5@7.1.2: parse5@7.2.1:
dependencies: dependencies:
entities: 4.5.0 entities: 4.5.0
@@ -34369,7 +34326,7 @@ snapshots:
dependencies: dependencies:
pg-int8: 1.0.1 pg-int8: 1.0.1
pg-numeric: 1.0.2 pg-numeric: 1.0.2
postgres-array: 3.0.4 postgres-array: 3.0.2
postgres-bytea: 3.0.0 postgres-bytea: 3.0.0
postgres-date: 2.1.0 postgres-date: 2.1.0
postgres-interval: 3.0.0 postgres-interval: 3.0.0
@@ -34536,7 +34493,7 @@ snapshots:
postgres-array@2.0.0: {} postgres-array@2.0.0: {}
postgres-array@3.0.4: {} postgres-array@3.0.2: {}
postgres-bytea@1.0.0: {} postgres-bytea@1.0.0: {}
@@ -34784,7 +34741,7 @@ snapshots:
dependencies: dependencies:
'@puppeteer/browsers': 2.3.0 '@puppeteer/browsers': 2.3.0
chromium-bidi: 0.6.2(devtools-protocol@0.0.1312386) chromium-bidi: 0.6.2(devtools-protocol@0.0.1312386)
debug: 4.4.0 debug: 4.3.5
devtools-protocol: 0.0.1312386 devtools-protocol: 0.0.1312386
ws: 8.18.0(bufferutil@4.0.8)(utf-8-validate@6.0.4) ws: 8.18.0(bufferutil@4.0.8)(utf-8-validate@6.0.4)
transitivePeerDependencies: transitivePeerDependencies:
@@ -35671,7 +35628,7 @@ snapshots:
dependencies: dependencies:
component-emitter: 1.3.1 component-emitter: 1.3.1
cookiejar: 2.1.4 cookiejar: 2.1.4
debug: 4.4.0 debug: 4.3.5
fast-safe-stringify: 2.1.1 fast-safe-stringify: 2.1.1
form-data: 4.0.0 form-data: 4.0.0
formidable: 2.1.2 formidable: 2.1.2
+2
View File
@@ -3,3 +3,5 @@ MONGO_URL=mongodb://mongodb:27017?compressors=snappy
DB_URL_PG=postgresql://postgres:example@postgres:5432 DB_URL_PG=postgresql://postgres:example@postgres:5432
BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
BACKUP_BUCKET_NAME=dev-backups BACKUP_BUCKET_NAME=dev-backups
DB_CR_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable
QUEUE_CONFIG=redpanda:9092
+131 -82
View File
@@ -1,5 +1,5 @@
services: services:
stream: stream:
image: 'hardcoreeng/stream' image: 'hardcoreeng/stream'
extra_hosts: extra_hosts:
- 'huly.local:host-gateway' - 'huly.local:host-gateway'
@@ -44,9 +44,66 @@ services:
volumes: volumes:
- cockroach_db:/cockroach/cockroach-data - cockroach_db:/cockroach/cockroach-data
restart: unless-stopped restart: unless-stopped
redpanda:
image: docker.redpanda.com/redpandadata/redpanda:v24.3.6
extra_hosts:
- 'huly.local:host-gateway'
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
- --advertise-kafka-addr internal://redpanda:9092,external://localhost:19092
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
- --advertise-pandaproxy-addr internal://redpanda:8082,external://localhost:18082
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
- --rpc-addr redpanda:33145
- --advertise-rpc-addr redpanda:33145
- --mode dev-container
- --smp 1
- --default-log-level=info
container_name: redpanda
volumes:
- redpanda:/var/lib/redpanda/data
ports:
- 18081:18081
- 18082:18082
- 19092:19092
- 19644:9644
healthcheck:
test: ['CMD', 'rpk', 'cluster', 'info', '-X', 'user=superuser', '-X', 'pass=secretpassword']
interval: 10s
timeout: 5s
retries: 10
redpanda_console:
image: docker.redpanda.com/redpandadata/console:v2.8.3
extra_hosts:
- 'huly.local:host-gateway'
entrypoint: /bin/sh
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
environment:
CONFIG_FILEPATH: /tmp/config.yml
CONSOLE_CONFIG_FILE: |
kafka:
brokers: ["redpanda:9092"]
schemaRegistry:
enabled: true
urls: ["http://redpanda:8081"]
redpanda:
adminApi:
enabled: true
urls: ["http://redpanda:9644"]
ports:
- 8000:8080
depends_on:
- redpanda
minio: minio:
image: 'minio/minio' image: 'minio/minio'
command: server /data --address ":9000" --console-address ":9001" command: server /data --address ":9000" --console-address ":9001"
extra_hosts:
- 'huly.local:host-gateway'
expose:
- 9000
- 9001
ports: ports:
- 9000:9000 - 9000:9000
- 9001:9001 - 9001:9001
@@ -55,9 +112,10 @@ services:
restart: unless-stopped restart: unless-stopped
elastic: elastic:
image: 'elasticsearch:7.14.2' image: 'elasticsearch:7.14.2'
command: | expose:
/bin/sh -c "./bin/elasticsearch-plugin list | grep -q ingest-attachment || yes | ./bin/elasticsearch-plugin install --silent ingest-attachment; - 9200
/usr/local/bin/docker-entrypoint.sh eswrapper" extra_hosts:
- 'huly.local:host-gateway'
volumes: volumes:
- elastic:/usr/share/elasticsearch/data - elastic:/usr/share/elasticsearch/data
ports: ports:
@@ -67,8 +125,6 @@ services:
- BITNAMI_DEBUG=true - BITNAMI_DEBUG=true
- discovery.type=single-node - discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- http.cors.enabled=true
- http.cors.allow-origin=http://localhost:8082
healthcheck: healthcheck:
interval: 20s interval: 20s
retries: 10 retries: 10
@@ -88,21 +144,20 @@ services:
- ./branding.json:/var/cfg/branding.json - ./branding.json:/var/cfg/branding.json
environment: environment:
- ACCOUNT_PORT=3000 - ACCOUNT_PORT=3000
- QUEUE_CONFIG=${QUEUE_CONFIG}
- SERVER_SECRET=secret - SERVER_SECRET=secret
- ADMIN_EMAILS=admin - ADMIN_EMAILS=admin,${PLATFORM_ADMIN_EMAILS}
- STATS_URL=http://huly.local:4900 - STATS_URL=http://huly.local:4900
- WORKSPACE_LIMIT_PER_USER=10000 - WORKSPACE_LIMIT_PER_USER=10000
- DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable - DB_URL=${DB_CR_URL}
# - DB_URL=${MONGO_URL} # - DB_URL=${MONGO_URL}
# - DB_NS=account-2 # - DB_NS=account-2
# Pass only one region to disallow selection for new workspaces.Ø # Pass only one region to disallow selection for new workspaces.
- REGION_INFO=|Mongo;cockroach|CockroachDB - REGION_INFO=|Mongo;cockroach|CockroachDB
# - REGION_INFO=cockroach|CockroachDB
- TRANSACTOR_URL=ws://huly.local:3333,ws://huly.local:3332;;cockroach, - TRANSACTOR_URL=ws://huly.local:3333,ws://huly.local:3332;;cockroach,
- MAIL_URL= - MAIL_URL=
- STORAGE_CONFIG=${STORAGE_CONFIG} - STORAGE_CONFIG=${STORAGE_CONFIG}
- FRONT_URL=http://huly.local:8087 - FRONT_URL=http://huly.local:8087
- RESERVED_DB_NAMES=telegram,gmail,github
- MODEL_ENABLED=* - MODEL_ENABLED=*
- LAST_NAME_FIRST=true - LAST_NAME_FIRST=true
# - WS_LIVENESS_DAYS=1 # - WS_LIVENESS_DAYS=1
@@ -133,15 +188,15 @@ services:
environment: environment:
- WS_OPERATION=all+backup - WS_OPERATION=all+backup
- SERVER_SECRET=secret - SERVER_SECRET=secret
- QUEUE_CONFIG=${QUEUE_CONFIG}
- DB_URL=${MONGO_URL} - DB_URL=${MONGO_URL}
- STATS_URL=http://huly.local:4900 - STATS_URL=http://huly.local:4900
- MAIL_URL= - MAIL_URL=
- STORAGE_CONFIG=${STORAGE_CONFIG} - STORAGE_CONFIG=${STORAGE_CONFIG}
- RESERVED_DB_NAMES=telegram,gmail,github
- MODEL_ENABLED=* - MODEL_ENABLED=*
- REGION=
- ACCOUNTS_URL=http://huly.local:3000 - ACCOUNTS_URL=http://huly.local:3000
- BRANDING_PATH=/var/cfg/branding.json - BRANDING_PATH=/var/cfg/branding.json
# - PARALLEL=2
- BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
- BACKUP_BUCKET=${BACKUP_BUCKET_NAME} - BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
# - INIT_WORKSPACE=staging-dev # - INIT_WORKSPACE=staging-dev
@@ -158,19 +213,18 @@ services:
- ./branding.json:/var/cfg/branding.json - ./branding.json:/var/cfg/branding.json
environment: environment:
- WS_OPERATION=all+backup - WS_OPERATION=all+backup
- SERVER_SECRET=secret
- DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable
- STATS_URL=http://huly.local:4900
- MAIL_URL=
- REGION=cockroach - REGION=cockroach
- SERVER_SECRET=secret
- QUEUE_CONFIG=${QUEUE_CONFIG}
- DB_URL=${DB_CR_URL}
- STATS_URL=http://huly.local:4900
- STORAGE_CONFIG=${STORAGE_CONFIG} - STORAGE_CONFIG=${STORAGE_CONFIG}
- RESERVED_DB_NAMES=telegram,gmail,github
- MODEL_ENABLED=* - MODEL_ENABLED=*
- ACCOUNTS_URL=http://huly.local:3000 - ACCOUNTS_URL=http://huly.local:3000
- BRANDING_PATH=/var/cfg/branding.json - BRANDING_PATH=/var/cfg/branding.json
# - PARALLEL=2
- BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
- BACKUP_BUCKET=${BACKUP_BUCKET_NAME} - BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
- MAIL_URL=
# - INIT_WORKSPACE=staging-dev # - INIT_WORKSPACE=staging-dev
restart: unless-stopped restart: unless-stopped
collaborator: collaborator:
@@ -236,21 +290,19 @@ services:
- minio - minio
- account - account
- stats - stats
# - apm-server
ports: ports:
- 3333:3333 - 3333:3333
volumes: volumes:
- ./branding.json:/var/cfg/branding.json - ./branding.json:/var/cfg/branding.json
environment: environment:
# - SERVER_PROVIDER=uweb - QUEUE_CONFIG=${QUEUE_CONFIG}
# - UWS_HTTP_MAX_HEADERS_SIZE="32768" - REGION=
- UV_THREADPOOL_SIZE=10
- SERVER_PORT=3333 - SERVER_PORT=3333
- SERVER_SECRET=secret - SERVER_SECRET=secret
- ENABLE_COMPRESSION=false - ENABLE_COMPRESSION=true
- STATS_URL=http://huly.local:4900 - STATS_URL=http://huly.local:4900
- FULLTEXT_URL=http://huly.local:4700 - FULLTEXT_URL=http://huly.local:4700
# - DB_URL=postgresql://postgres:example@postgres:5432 # - DB_URL=${DB_CR_URL}
- DB_URL=${MONGO_URL} - DB_URL=${MONGO_URL}
- MONGO_URL=${MONGO_URL} - MONGO_URL=${MONGO_URL}
- 'MONGO_OPTIONS={"appName": "transactor", "maxPoolSize": 10}' - 'MONGO_OPTIONS={"appName": "transactor", "maxPoolSize": 10}'
@@ -258,7 +310,6 @@ services:
- METRICS_FILE=metrics.txt - METRICS_FILE=metrics.txt
- STORAGE_CONFIG=${STORAGE_CONFIG} - STORAGE_CONFIG=${STORAGE_CONFIG}
- FRONT_URL=http://huly.local:8087 - FRONT_URL=http://huly.local:8087
# - APM_SERVER_URL=http://apm-server:8200
- MAIL_URL='' - MAIL_URL=''
- ACCOUNTS_URL=http://huly.local:3000 - ACCOUNTS_URL=http://huly.local:3000
- LAST_NAME_FIRST=true - LAST_NAME_FIRST=true
@@ -274,52 +325,38 @@ services:
- minio - minio
- account - account
- stats - stats
# - apm-server
ports: ports:
- 3332:3332 - 3332:3332
volumes: volumes:
- ./branding.json:/var/cfg/branding.json - ./branding.json:/var/cfg/branding.json
environment: environment:
# - SERVER_PROVIDER=uweb - QUEUE_CONFIG=${QUEUE_CONFIG}
# - UWS_HTTP_MAX_HEADERS_SIZE="32768"
# - UV_THREADPOOL_SIZE=10
- SERVER_PORT=3332 - SERVER_PORT=3332
- REGION=cockroach
- SERVER_SECRET=secret - SERVER_SECRET=secret
- ENABLE_COMPRESSION=false - ENABLE_COMPRESSION=true
- FULLTEXT_URL=http://huly.local:4702 - FULLTEXT_URL=http://huly.local:4702
- STATS_URL=http://huly.local:4900 - STATS_URL=http://huly.local:4900
- DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable - DB_URL=${DB_CR_URL}
- METRICS_CONSOLE=false - METRICS_CONSOLE=false
- METRICS_FILE=metrics.txt - METRICS_FILE=metrics.txt
- STORAGE_CONFIG=${STORAGE_CONFIG} - STORAGE_CONFIG=${STORAGE_CONFIG}
- FRONT_URL=http://huly.local:8087 - FRONT_URL=http://huly.local:8087
# - APM_SERVER_URL=http://apm-server:8200
- MAIL_URL='' - MAIL_URL=''
- ACCOUNTS_URL=http://huly.local:3000 - ACCOUNTS_URL=http://huly.local:3000
- LAST_NAME_FIRST=true - LAST_NAME_FIRST=true
- BRANDING_PATH=/var/cfg/branding.json - BRANDING_PATH=/var/cfg/branding.json
- AI_BOT_URL=http://huly.local:4010 - AI_BOT_URL=http://huly.local:4010
restart: unless-stopped restart: unless-stopped
green:
image: hardcoreeng/green
extra_hosts:
- 'huly.local:host-gateway'
links:
- cockroach
- stats
ports:
- 6767:6767
environment:
- PORT=6767
- AUTH_TOKEN=secret
- STATS_URL=http://huly.local:4900
- DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable
restart: unless-stopped
rekoni: rekoni:
image: hardcoreeng/rekoni-service image: hardcoreeng/rekoni-service
extra_hosts:
- 'huly.local:host-gateway'
restart: unless-stopped restart: unless-stopped
ports: ports:
- 4004:4004 - 4004:4004
environment:
- STATS_URL=http://huly.local:4901
fulltext: fulltext:
image: hardcoreeng/fulltext image: hardcoreeng/fulltext
extra_hosts: extra_hosts:
@@ -328,10 +365,13 @@ services:
links: links:
- elastic - elastic
- mongodb - mongodb
- rekoni
ports: ports:
- 4700:4700 - 4700:4700
environment: environment:
- PORT=4700
- SERVER_SECRET=secret - SERVER_SECRET=secret
- QUEUE_CONFIG=${QUEUE_CONFIG}
- DB_URL=${MONGO_URL} - DB_URL=${MONGO_URL}
- FULLTEXT_DB_URL=http://huly.local:9200 - FULLTEXT_DB_URL=http://huly.local:9200
- ELASTIC_INDEX_NAME=local_storage_index - ELASTIC_INDEX_NAME=local_storage_index
@@ -347,18 +387,22 @@ services:
links: links:
- elastic - elastic
- cockroach - cockroach
- rekoni
ports: ports:
- 4702:4702 - 4702:4702
environment: environment:
- PORT=4702 - PORT=4702
- REGION=cockroach
- SERVER_SECRET=secret - SERVER_SECRET=secret
- DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable - QUEUE_CONFIG=${QUEUE_CONFIG}
- DB_URL=${DB_CR_URL}
- FULLTEXT_DB_URL=http://huly.local:9200 - FULLTEXT_DB_URL=http://huly.local:9200
- ELASTIC_INDEX_NAME=local_storage_index - ELASTIC_INDEX_NAME=local_storage_index # Same index for simplicity
- STORAGE_CONFIG=${STORAGE_CONFIG} - STORAGE_CONFIG=${STORAGE_CONFIG}
- STATS_URL=http://huly.local:4900 - STATS_URL=http://huly.local:4900
- REKONI_URL=http://huly.local:4004 - REKONI_URL=http://huly.local:4004
- ACCOUNTS_URL=http://huly.local:3000 - ACCOUNTS_URL=http://huly.local:3000
- SERVER_SECRET=secret
print: print:
image: hardcoreeng/print image: hardcoreeng/print
extra_hosts: extra_hosts:
@@ -368,6 +412,7 @@ services:
- 4005:4005 - 4005:4005
environment: environment:
- SECRET=secret - SECRET=secret
- QUEUE_CONFIG=${QUEUE_CONFIG}
- STORAGE_CONFIG=${STORAGE_CONFIG} - STORAGE_CONFIG=${STORAGE_CONFIG}
- STATS_URL=http://huly.local:4900 - STATS_URL=http://huly.local:4900
- ACCOUNTS_URL=http://huly.local:3000 - ACCOUNTS_URL=http://huly.local:3000
@@ -384,6 +429,7 @@ services:
environment: environment:
- SECRET=secret - SECRET=secret
- MINIO_ENDPOINT=minio - MINIO_ENDPOINT=minio
- QUEUE_CONFIG=${QUEUE_CONFIG}
- MINIO_ACCESS_KEY=minioadmin - MINIO_ACCESS_KEY=minioadmin
- ACCOUNTS_URL=http://huly.local:3000 - ACCOUNTS_URL=http://huly.local:3000
- MINIO_SECRET_KEY=minioadmin - MINIO_SECRET_KEY=minioadmin
@@ -391,21 +437,21 @@ services:
- SERVICE_ID=sign-service - SERVICE_ID=sign-service
- BRANDING_PATH=/var/cfg/branding.json - BRANDING_PATH=/var/cfg/branding.json
- STATS_URL=http://huly.local:4900 - STATS_URL=http://huly.local:4900
# analytics: # analytics:
# image: hardcoreeng/analytics-collector # image: hardcoreeng/analytics-collector
# extra_hosts: # extra_hosts:
# - 'huly.local:host-gateway' # - 'huly.local:host-gateway'
# restart: unless-stopped # restart: unless-stopped
# ports: # ports:
# - 4017:4017 # - 4017:4017
# environment: # environment:
# - SECRET=secret # - SECRET=secret
# - PORT=4017 # - PORT=4017
# - MONGO_URL=${MONGO_URL} # - MONGO_URL=${MONGO_URL}
# - 'MONGO_OPTIONS={"appName":"analytics","maxPoolSize":1}' # - 'MONGO_OPTIONS={"appName":"analytics","maxPoolSize":1}'
# - SERVICE_ID=analytics-collector-service # - SERVICE_ID=analytics-collector-service
# - ACCOUNTS_URL=http://huly.local:3000 # - ACCOUNTS_URL=http://huly.local:3000
# - STATS_URL=http://huly.local:4900 # - STATS_URL=http://huly.local:4900
aiBot: aiBot:
image: hardcoreeng/ai-bot image: hardcoreeng/ai-bot
ports: ports:
@@ -416,6 +462,7 @@ services:
environment: environment:
- SERVER_SECRET=secret - SERVER_SECRET=secret
- MONGO_URL=${MONGO_URL} - MONGO_URL=${MONGO_URL}
- QUEUE_CONFIG=${QUEUE_CONFIG}
- ACCOUNTS_URL=http://huly.local:3000 - ACCOUNTS_URL=http://huly.local:3000
- STORAGE_CONFIG=${STORAGE_CONFIG} - STORAGE_CONFIG=${STORAGE_CONFIG}
- FIRST_NAME=Jolie - FIRST_NAME=Jolie
@@ -424,23 +471,24 @@ services:
- AVATAR_PATH=./avatar.png - AVATAR_PATH=./avatar.png
- AVATAR_CONTENT_TYPE=.png - AVATAR_CONTENT_TYPE=.png
- STATS_URL=http://huly.local:4900 - STATS_URL=http://huly.local:4900
# - LOVE_ENDPOINT=http://huly.local:8096 # - LOVE_ENDPOINT=http://huly.local:8096
# - OPENAI_API_KEY=token # - OPENAI_API_KEY=token
# telegram-bot: # telegram-bot:
# image: hardcoreeng/telegram-bot # image: hardcoreeng/telegram-bot
# extra_hosts: # extra_hosts:
# - "huly.local:host-gateway" # - "huly.local:host-gateway"
# restart: unless-stopped # restart: unless-stopped
# environment: # environment:
# - PORT=4020 # - PORT=4020
# - BOT_TOKEN=token # - QUEUE_CONFIG=${QUEUE_CONFIG}
# - MONGO_URL=${MONGO_URL} # - BOT_TOKEN=token
# - MONGO_DB=telegram-bot # - MONGO_URL=${MONGO_URL}
# - SECRET=secret # - MONGO_DB=telegram-bot
# - DOMAIN=domain # - SECRET=secret
# - ACCOUNTS_URL=http://huly.local:3000 # - DOMAIN=domain
# - SERVICE_ID=telegram-bot-service # - ACCOUNTS_URL=http://huly.local:3000
# - STATS_URL=http://huly.local:4900 # - SERVICE_ID=telegram-bot-service
# - STATS_URL=http://huly.local:4900
export: export:
image: hardcoreeng/export image: hardcoreeng/export
extra_hosts: extra_hosts:
@@ -465,3 +513,4 @@ volumes:
files: files:
elastic: elastic:
cockroach_db: cockroach_db:
redpanda:
+3 -2
View File
@@ -20,7 +20,7 @@
"docker:abuild": "docker build -t hardcoreeng/tool . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool", "docker:abuild": "docker build -t hardcoreeng/tool . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool",
"docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging",
"docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool",
"run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4700 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js", "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4700 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) QUEUE_CONFIG='localhost:9092' node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js",
"run-local-cr": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 $TOOL_OPT ./bundle/bundle.js", "run-local-cr": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable DB_URL=postgresql://root@huly.local:26257/defaultdb?sslmode=disable TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 $TOOL_OPT ./bundle/bundle.js",
"run-local-brk": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js", "run-local-brk": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js",
"run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js", "run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js",
@@ -171,6 +171,7 @@
"bufferutil": "^4.0.8", "bufferutil": "^4.0.8",
"utf-8-validate": "^6.0.4", "utf-8-validate": "^6.0.4",
"msgpackr": "^1.11.2", "msgpackr": "^1.11.2",
"msgpackr-extract": "^3.0.3" "msgpackr-extract": "^3.0.3",
"@hcengineering/kafka": "^0.6.0"
} }
} }
-12
View File
@@ -26,7 +26,6 @@ import core, {
type CollaborativeDoc, type CollaborativeDoc,
type Client as CoreClient, type Client as CoreClient,
DOMAIN_BENCHMARK, DOMAIN_BENCHMARK,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MIGRATION, DOMAIN_MIGRATION,
DOMAIN_MODEL, DOMAIN_MODEL,
DOMAIN_STATUS, DOMAIN_STATUS,
@@ -1218,17 +1217,6 @@ export async function removeDuplicateIds (
// await updateArrRefs(txop, newId, doc) // await updateArrRefs(txop, newId, doc)
// // update docIndexState
// const docIndexState = await client.findOne(core.class.DocIndexState, { doc: doc._id })
// if (docIndexState !== undefined) {
// const { _id, space, modifiedBy, modifiedOn, createdBy, createdOn, _class, ...data } = docIndexState
// await txop.createDoc(docIndexState._class, docIndexState.space, {
// ...data,
// removed: false
// })
// await txop.update(docIndexState, { removed: true, needIndex: true })
// }
// if (domain !== DOMAIN_MODEL) { // if (domain !== DOMAIN_MODEL) {
// const raw = await db.collection(domain).findOne({ _id: doc._id }) // const raw = await db.collection(domain).findOne({ _id: doc._id })
// await db.collection(domain).insertOne({ // await db.collection(domain).insertOne({
-13
View File
@@ -14,8 +14,6 @@
// //
import { Client as ElasticClient } from '@elastic/elasticsearch' import { Client as ElasticClient } from '@elastic/elasticsearch'
import core, { DOMAIN_DOC_INDEX_STATE } from '@hcengineering/core'
import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo'
import { type StorageAdapter } from '@hcengineering/server-core' import { type StorageAdapter } from '@hcengineering/server-core'
export async function rebuildElastic ( export async function rebuildElastic (
@@ -24,17 +22,6 @@ export async function rebuildElastic (
storageAdapter: StorageAdapter, storageAdapter: StorageAdapter,
elasticUrl: string elasticUrl: string
): Promise<void> { ): Promise<void> {
const client = getMongoClient(mongoUrl)
try {
const _client = await client.getClient()
const db = getWorkspaceMongoDB(_client, dataId)
await db
.collection(DOMAIN_DOC_INDEX_STATE)
.updateMany({ _class: core.class.DocIndexState }, { $set: { elastic: false } })
} finally {
client.close()
}
await dropElastic(elasticUrl, dataId) await dropElastic(elasticUrl, dataId)
} }
-33
View File
@@ -1,33 +0,0 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type MeasureContext } from '@hcengineering/core'
export async function reindexWorkspace (ctx: MeasureContext, fulltextUrl: string, token: string): Promise<void> {
try {
const res = await fetch(fulltextUrl + '/api/v1/reindex', {
method: 'PUT',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ token })
})
if (!res.ok) {
throw new Error(`HTTP Error ${res.status} ${res.statusText}`)
}
} catch (err: any) {
ctx.error('failed to reset index', { err })
}
}
+41 -3
View File
@@ -47,6 +47,7 @@ import {
import serverToken, { generateToken } from '@hcengineering/server-token' import serverToken, { generateToken } from '@hcengineering/server-token'
import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
import { getPlatformQueue } from '@hcengineering/kafka'
import { buildStorageFromConfig, createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { buildStorageFromConfig, createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import { program, type Command } from 'commander' import { program, type Command } from 'commander'
import { updateField } from './workspace' import { updateField } from './workspace'
@@ -81,11 +82,15 @@ import {
createPostgresTxAdapter, createPostgresTxAdapter,
shutdownPostgres shutdownPostgres
} from '@hcengineering/postgres' } from '@hcengineering/postgres'
import type { StorageAdapter } from '@hcengineering/server-core' import {
QueueTopic,
workspaceEvents,
type QueueWorkspaceMessage,
type StorageAdapter
} from '@hcengineering/server-core'
import { getAccountDBUrl, getMongoDBUrl } from './__start' import { getAccountDBUrl, getMongoDBUrl } from './__start'
// import { fillGithubUsers, fixAccountEmails, renameAccount } from './account' // import { fillGithubUsers, fixAccountEmails, renameAccount } from './account'
import { changeConfiguration } from './configuration' import { changeConfiguration } from './configuration'
import { reindexWorkspace } from './fulltext'
import { moveAccountDbFromMongoToPG } from './db' import { moveAccountDbFromMongoToPG } from './db'
import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils' import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
@@ -363,6 +368,13 @@ export function devTool (
progress: 100 progress: 100
}) })
const wsProducer = getPlatformQueue('tool', cmd.region).createProducer<QueueWorkspaceMessage>(
toolCtx,
QueueTopic.Workspace
)
await wsProducer.send(res.workspaceUuid, [workspaceEvents.created()])
await wsProducer.close()
console.log('create-workspace done') console.log('create-workspace done')
}) })
}) })
@@ -442,6 +454,12 @@ export function devTool (
}) })
console.log(metricsToString(measureCtx.metrics, 'upgrade', 60)) console.log(metricsToString(measureCtx.metrics, 'upgrade', 60))
const wsProducer = getPlatformQueue('tool', info.region).createProducer<QueueWorkspaceMessage>(
toolCtx,
QueueTopic.Workspace
)
await wsProducer.send(info.uuid, [workspaceEvents.upgraded()])
await wsProducer.close()
console.log('upgrade-workspace done') console.log('upgrade-workspace done')
}) })
}) })
@@ -1121,6 +1139,12 @@ export function devTool (
storageAdapter: workspaceStorage, storageAdapter: workspaceStorage,
historyFile: cmd.historyFile historyFile: cmd.historyFile
}) })
const wsProducer = getPlatformQueue('tool', ws.region).createProducer<QueueWorkspaceMessage>(
toolCtx,
QueueTopic.Workspace
)
await wsProducer.send(ws.uuid, [workspaceEvents.fullReindex()])
await wsProducer.close()
await workspaceStorage?.close() await workspaceStorage?.close()
}) })
} }
@@ -2063,7 +2087,12 @@ export function devTool (
} }
console.log('reindex workspace', workspace) console.log('reindex workspace', workspace)
await reindexWorkspace(toolCtx, fulltextUrl, getToolToken(ws?.uuid)) const wsProducer = getPlatformQueue('tool', ws.region).createProducer<QueueWorkspaceMessage>(
toolCtx,
QueueTopic.Workspace
)
await wsProducer.send(ws.uuid, [workspaceEvents.fullReindex()])
await wsProducer.close()
console.log('done', workspace) console.log('done', workspace)
}) })
}) })
@@ -2367,6 +2396,15 @@ export function devTool (
// }) // })
// }) // })
program
.command('queue-init-topics')
.description('create required kafka topics')
.option('-tx <tx>', 'Number of TX partitions', '5')
.action(async (cmd: { tx: string }) => {
const queue = getPlatformQueue('tool')
await queue.createTopics(parseInt(cmd.tx ?? '1'))
})
extendProgram?.(program) extendProgram?.(program)
program.parse(process.argv) program.parse(process.argv)
-1
View File
@@ -112,7 +112,6 @@
"@hcengineering/model-server-ai-bot": "^0.6.0", "@hcengineering/model-server-ai-bot": "^0.6.0",
"@hcengineering/model-ai-bot": "^0.6.0", "@hcengineering/model-ai-bot": "^0.6.0",
"@hcengineering/model-recorder": "^0.6.0", "@hcengineering/model-recorder": "^0.6.0",
"@hcengineering/model-server-fulltext": "^0.6.0",
"@hcengineering/model-test-management": "^0.6.0", "@hcengineering/model-test-management": "^0.6.0",
"@hcengineering/model-survey": "^0.6.0", "@hcengineering/model-survey": "^0.6.0",
"@hcengineering/model-presence": "^0.6.0", "@hcengineering/model-presence": "^0.6.0",
-2
View File
@@ -110,7 +110,6 @@ import {
serverDocumentsId, serverDocumentsId,
createModel as serverDocumentsModel createModel as serverDocumentsModel
} from '@hcengineering/model-server-controlled-documents' } from '@hcengineering/model-server-controlled-documents'
import { serverFulltextId, createModel as serverFulltextModel } from '@hcengineering/model-server-fulltext'
import { surveyId, createModel as surveyModel } from '@hcengineering/model-survey' import { surveyId, createModel as surveyModel } from '@hcengineering/model-survey'
import { presenceId, createModel as presenceModel } from '@hcengineering/model-presence' import { presenceId, createModel as presenceModel } from '@hcengineering/model-presence'
import processes, { processId, createModel as processModel } from '@hcengineering/model-process' import processes, { processId, createModel as processModel } from '@hcengineering/model-process'
@@ -493,7 +492,6 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[
[serverTrainingModel, serverTrainingId], [serverTrainingModel, serverTrainingId],
[serverDocumentsModel, serverDocumentsId], [serverDocumentsModel, serverDocumentsId],
[serverAiBotModel, serverAiBotId], [serverAiBotModel, serverAiBotId],
[serverFulltextModel, serverFulltextId],
[serverProcessModel, serverProcessId] [serverProcessModel, serverProcessId]
] ]
@@ -26,8 +26,6 @@ import {
type Class, type Class,
type Data, type Data,
type Doc, type Doc,
type DocIndexState,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_SEQUENCE, DOMAIN_SEQUENCE,
DOMAIN_TX, DOMAIN_TX,
generateId, generateId,
@@ -446,14 +444,6 @@ async function migrateDocumentMetaInternalCode (client: MigrationClient): Promis
} }
await client.bulk(DOMAIN_DOCUMENTS, operations) await client.bulk(DOMAIN_DOCUMENTS, operations)
await client.update<DocIndexState>(
DOMAIN_DOC_INDEX_STATE,
{
_id: { $in: Array.from(updatedIds) as any },
objectClass: documents.class.DocumentMeta
},
{ $set: { needIndex: true } }
)
} }
export const documentsOperation: MigrateOperation = { export const documentsOperation: MigrateOperation = {
-18
View File
@@ -26,11 +26,9 @@ import {
type Configuration, type Configuration,
type ConfigurationElement, type ConfigurationElement,
type Doc, type Doc,
type DocIndexState,
type Domain, type Domain,
DOMAIN_BLOB, DOMAIN_BLOB,
DOMAIN_CONFIGURATION, DOMAIN_CONFIGURATION,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MIGRATION, DOMAIN_MIGRATION,
DOMAIN_MODEL, DOMAIN_MODEL,
DOMAIN_RELATION, DOMAIN_RELATION,
@@ -339,22 +337,6 @@ export class TPluginConfiguration extends TDoc implements PluginConfiguration {
beta!: boolean beta!: boolean
} }
@Model(core.class.DocIndexState, core.class.Doc, DOMAIN_DOC_INDEX_STATE)
export class TDocIndexState extends TDoc implements DocIndexState {
@Prop(TypeRef(core.class.Class), core.string.Class)
@Index(IndexKind.Indexed)
@Hidden()
objectClass!: Ref<Class<Doc>>
@Prop(TypeBoolean(), getEmbeddedLabel('Removed'))
@Hidden()
removed!: boolean
@Prop(TypeBoolean(), getEmbeddedLabel('NeedIndexing'))
@Hidden()
needIndex!: boolean
}
@Model(core.class.FullTextSearchContext, core.class.Doc, DOMAIN_MODEL) @Model(core.class.FullTextSearchContext, core.class.Doc, DOMAIN_MODEL)
export class TFullTextSearchContext extends TDoc implements FullTextSearchContext { export class TFullTextSearchContext extends TDoc implements FullTextSearchContext {
toClass!: Ref<Class<Doc<Space>>> toClass!: Ref<Class<Doc<Space>>>
-27
View File
@@ -17,7 +17,6 @@ import {
DOMAIN_BENCHMARK, DOMAIN_BENCHMARK,
DOMAIN_BLOB, DOMAIN_BLOB,
DOMAIN_CONFIGURATION, DOMAIN_CONFIGURATION,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MIGRATION, DOMAIN_MIGRATION,
DOMAIN_SPACE, DOMAIN_SPACE,
DOMAIN_STATUS, DOMAIN_STATUS,
@@ -38,7 +37,6 @@ import {
TConfiguration, TConfiguration,
TConfigurationElement, TConfigurationElement,
TDoc, TDoc,
TDocIndexState,
TDomainIndexConfiguration, TDomainIndexConfiguration,
TEnum, TEnum,
TEnumOf, TEnumOf,
@@ -149,7 +147,6 @@ export function createModel (builder: Builder): void {
TEnum, TEnum,
TTypeAny, TTypeAny,
TTypeRelatedDocument, TTypeRelatedDocument,
TDocIndexState,
TFullTextSearchContext, TFullTextSearchContext,
TConfiguration, TConfiguration,
TConfigurationElement, TConfigurationElement,
@@ -262,30 +259,6 @@ export function createModel (builder: Builder): void {
] ]
}) })
builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, {
domain: DOMAIN_DOC_INDEX_STATE,
indexes: [
{
keys: { needIndex: 1, objectClass: 1 }
}
],
disabled: [
{ attachedToClass: 1 },
{ stages: 1 },
{ space: 1 },
{ _class: 1 },
{ needIndex: 1 },
{ objectClass: 1 },
{ _class: 1 },
{ attachedTo: 1 },
{ modifiedBy: 1 },
{ modifiedOn: 1 },
{ createdBy: 1 },
{ createdBy: -1 },
{ createdOn: -1 }
]
})
builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
toClass: core.class.Space, toClass: core.class.Space,
childProcessingAllowed: false childProcessingAllowed: false
+1 -1
View File
@@ -306,7 +306,7 @@ export function getAccountsFromTxes (accTxes: TxCUD<Doc>[]): any {
return Object.values(byAccounts) return Object.values(byAccounts)
.map((txes) => TxProcessor.buildDoc2Doc(txes)) .map((txes) => TxProcessor.buildDoc2Doc(txes))
.filter((it) => it !== undefined) .filter((it) => it != null)
} }
export async function getSocialKeyByOldAccount (client: MigrationClient): Promise<Record<string, string>> { export async function getSocialKeyByOldAccount (client: MigrationClient): Promise<Record<string, string>> {
-37
View File
@@ -1,37 +0,0 @@
{
"name": "@hcengineering/model-server-fulltext",
"version": "0.6.0",
"main": "lib/index.js",
"svelte": "src/index.ts",
"types": "types/index.d.ts",
"author": "Anticrm Platform Contributors",
"template": "@hcengineering/model-package",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"format": "format src",
"_phase:build": "compile transpile src",
"_phase:format": "format src",
"_phase:validate": "compile validate"
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3"
},
"dependencies": {
"@hcengineering/core": "^0.6.32",
"@hcengineering/model": "^0.6.11",
"@hcengineering/platform": "^0.6.11",
"@hcengineering/server-fulltext": "^0.6.0",
"@hcengineering/server-core": "^0.6.1"
}
}
-26
View File
@@ -1,26 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type Builder } from '@hcengineering/model'
import core from '@hcengineering/core'
import serverCore from '@hcengineering/server-core'
import serverFulltext from '@hcengineering/server-fulltext'
export { serverFulltextId } from '@hcengineering/server-fulltext'
export function createModel (builder: Builder): void {
builder.createDoc(serverCore.class.Trigger, core.space.Model, {
trigger: serverFulltext.trigger.OnChange
})
}
-13
View File
@@ -421,8 +421,6 @@ export interface TransientConfiguration extends Class<Doc> {
*/ */
export const DOMAIN_BLOB = 'blob' as Domain export const DOMAIN_BLOB = 'blob' as Domain
export const DOMAIN_DOC_INDEX_STATE = 'doc-index-state' as Domain
/** /**
* @public * @public
*/ */
@@ -581,17 +579,6 @@ export function versionToString (version: Version | Data<Version>): string {
return `${version?.major}.${version?.minor}.${version?.patch}` return `${version?.major}.${version?.minor}.${version?.patch}`
} }
/**
* @public
*
* Define status for full text indexing
*/
export interface DocIndexState extends Doc {
objectClass: Ref<Class<Doc>>
needIndex: boolean
removed: boolean
}
/** /**
* @public * @public
*/ */
-2
View File
@@ -28,7 +28,6 @@ import type {
Configuration, Configuration,
ConfigurationElement, ConfigurationElement,
Doc, Doc,
DocIndexState,
DomainIndexConfiguration, DomainIndexConfiguration,
Enum, Enum,
EnumOf, EnumOf,
@@ -149,7 +148,6 @@ export default plugin(coreId, {
PluginConfiguration: '' as Ref<Class<PluginConfiguration>>, PluginConfiguration: '' as Ref<Class<PluginConfiguration>>,
UserStatus: '' as Ref<Class<UserStatus>>, UserStatus: '' as Ref<Class<UserStatus>>,
TypeRelatedDocument: '' as Ref<Class<Type<RelatedDocument>>>, TypeRelatedDocument: '' as Ref<Class<Type<RelatedDocument>>>,
DocIndexState: '' as Ref<Class<DocIndexState>>,
DomainIndexConfiguration: '' as Ref<Class<DomainIndexConfiguration>>, DomainIndexConfiguration: '' as Ref<Class<DomainIndexConfiguration>>,
Configuration: '' as Ref<Class<Configuration>>, Configuration: '' as Ref<Class<Configuration>>,
+10 -7
View File
@@ -198,6 +198,7 @@ function printMetricsChildren (params: Record<string, Metrics>, offset: number,
if (Object.keys(params).length > 0) { if (Object.keys(params).length > 0) {
r += '\n' + toLen('', ' ', offset) r += '\n' + toLen('', ' ', offset)
r += Object.entries(params) r += Object.entries(params)
.filter((it) => it[1].value > 0.1)
.map(([k, vv]) => toString(k, vv, offset, length)) .map(([k, vv]) => toString(k, vv, offset, length))
.join('\n' + toLen('', ' ', offset)) .join('\n' + toLen('', ' ', offset))
} }
@@ -211,11 +212,13 @@ function printMetricsParams (
): string { ): string {
let r = '' let r = ''
const joinP = (key: string, data: Record<string, MetricsData>): string[] => { const joinP = (key: string, data: Record<string, MetricsData>): string[] => {
return Object.entries(data).map(([k, vv]) => return Object.entries(data)
`${toLen('', ' ', offset)}${toLen(key + '=' + k, '-', length - offset)}: avg ${ .filter((it) => it[1].value >= 0.1)
vv.value / (vv.operations > 0 ? vv.operations : 1) .map(([k, vv]) =>
} total: ${vv.value} ops: ${vv.operations}`.trim() `${toLen('', ' ', offset)}${toLen(key + '=' + k, '-', length - offset)}: avg ${
) Math.round((vv.value / (vv.operations > 0 ? vv.operations : 1)) * 100) / 100
} total: ${Math.round(vv.value * 100) / 100} ops: ${vv.operations}`.trim()
)
} }
const joinParams = Object.entries(params).reduce<string[]>((p, c) => [...p, ...joinP(c[0], c[1])], []) const joinParams = Object.entries(params).reduce<string[]>((p, c) => [...p, ...joinP(c[0], c[1])], [])
if (Object.keys(joinParams).length > 0) { if (Object.keys(joinParams).length > 0) {
@@ -227,8 +230,8 @@ function printMetricsParams (
function toString (name: string, m: Metrics, offset: number, length: number): string { function toString (name: string, m: Metrics, offset: number, length: number): string {
let r = `${toLen('', ' ', offset)}${toLen(name, '-', length - offset)}: avg ${ let r = `${toLen('', ' ', offset)}${toLen(name, '-', length - offset)}: avg ${
m.value / (m.operations > 0 ? m.operations : 1) Math.round((m.value / (m.operations > 0 ? m.operations : 1)) * 100) / 100
} total: ${m.value} ops: ${m.operations}`.trim() } total: ${Math.round(m.value * 100) / 100} ops: ${m.operations}`.trim()
r += printMetricsParams(m.params, offset + 4, length) r += printMetricsParams(m.params, offset + 4, length)
r += printMetricsChildren(m.measurements, offset + 4, length) r += printMetricsChildren(m.measurements, offset + 4, length)
return r return r
+1 -2
View File
@@ -13,7 +13,7 @@
// limitations under the License. // limitations under the License.
// //
import type { Account, AccountUuid, Doc, DocIndexState, Domain, PersonId, Ref } from './classes' import type { Account, AccountUuid, Doc, Domain, PersonId, Ref } from './classes'
import { MeasureContext } from './measurements' import { MeasureContext } from './measurements'
import { DocumentQuery, FindOptions } from './storage' import { DocumentQuery, FindOptions } from './storage'
import type { DocumentUpdate, Tx } from './tx' import type { DocumentUpdate, Tx } from './tx'
@@ -52,7 +52,6 @@ export interface SessionData {
workspace: WorkspaceIds workspace: WorkspaceIds
branding: Branding | null branding: Branding | null
socialStringsToUsers: Map<PersonId, AccountUuid> socialStringsToUsers: Map<PersonId, AccountUuid>
fulltextUpdates?: Map<Ref<DocIndexState>, DocIndexState>
asyncRequests?: (() => Promise<void>)[] asyncRequests?: (() => Promise<void>)[]
} }
+9 -3
View File
@@ -15,7 +15,6 @@
import type { KeysByType } from 'simplytyped' import type { KeysByType } from 'simplytyped'
import type { import type {
PersonId,
Arr, Arr,
AttachedDoc, AttachedDoc,
Class, Class,
@@ -23,6 +22,7 @@ import type {
Doc, Doc,
Domain, Domain,
Mixin, Mixin,
PersonId,
PropertyType, PropertyType,
Ref, Ref,
Space, Space,
@@ -399,10 +399,16 @@ export abstract class TxProcessor implements WithTx {
return rawDoc return rawDoc
} }
static buildDoc2Doc<D extends Doc>(txes: Tx[]): D | undefined { static buildDoc2Doc<D extends Doc>(txes: Tx[]): D | undefined | null {
let doc: Doc let doc: Doc
const deleteTx = txes.find((tx) => tx._class === core.class.TxRemoveDoc)
if (deleteTx !== undefined) {
return null
}
const createTx = txes.find((tx) => tx._class === core.class.TxCreateDoc) const createTx = txes.find((tx) => tx._class === core.class.TxCreateDoc)
if (createTx === undefined) return if (createTx === undefined) {
return
}
doc = TxProcessor.createDoc2Doc(createTx as TxCreateDoc<Doc>) doc = TxProcessor.createDoc2Doc(createTx as TxCreateDoc<Doc>)
for (const tx of txes) { for (const tx of txes) {
if (tx._class === core.class.TxUpdateDoc) { if (tx._class === core.class.TxUpdateDoc) {
-2
View File
@@ -27,7 +27,6 @@ import {
Doc, Doc,
DocData, DocData,
DOMAIN_BLOB, DOMAIN_BLOB,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MODEL, DOMAIN_MODEL,
DOMAIN_TRANSIENT, DOMAIN_TRANSIENT,
FullTextSearchContext, FullTextSearchContext,
@@ -673,7 +672,6 @@ export function isClassIndexable (
} }
if ( if (
domain === DOMAIN_DOC_INDEX_STATE ||
domain === DOMAIN_TX || domain === DOMAIN_TX ||
domain === DOMAIN_MODEL || domain === DOMAIN_MODEL ||
domain === DOMAIN_BLOB || domain === DOMAIN_BLOB ||
@@ -42,7 +42,7 @@
const txes = await client.findAll(core.class.Tx, { objectId: value._id }) const txes = await client.findAll(core.class.Tx, { objectId: value._id })
while (txes.length) { while (txes.length) {
const doc = TxProcessor.buildDoc2Doc<Card>(txes) const doc = TxProcessor.buildDoc2Doc<Card>(txes)
if (doc === undefined) { if (doc == null) {
break break
} }
targetState = store.byId.get(doc.status) targetState = store.byId.get(doc.status)
-33
View File
@@ -1,33 +0,0 @@
import core, { type Class, type Client, type Doc, type DocIndexState, type Ref } from '@hcengineering/core'
import { writable } from 'svelte/store'
export interface InferenceFocus {
objectId: Ref<Doc>
objectClass: Ref<Class<Doc>>
state: DocIndexState
}
/**
* @public
*/
export const inferenceFocusStore = writable<InferenceFocus | undefined>(undefined)
/**
* @public
*/
export async function updateFocus (client: Client, doc?: { _id: Ref<Doc>, _class: Ref<Class<Doc>> }): Promise<void> {
if (doc === undefined) {
inferenceFocusStore.set(undefined)
return
}
const state = await client.findOne(core.class.DocIndexState, {
_id: doc._id as Ref<DocIndexState>,
objectClass: doc._class
})
if (state !== undefined) {
inferenceFocusStore.update(() => {
return { objectId: doc._id, objectClass: doc._class, state }
})
}
}
@@ -35,7 +35,14 @@
let profiling = false let profiling = false
async function fetchStats (time: number): Promise<void> { async function fetchStats (time: number): Promise<void> {
await fetch(endpoint + `/api/v1/profiling?token=${token}`, {}) await fetch(endpoint + '/api/v1/profiling', {
method: 'GET',
keepalive: true,
headers: {
'Content-Type': 'application/json',
Authorization: 'Bearer ' + token
}
})
.then(async (json) => { .then(async (json) => {
data = await json.json() data = await json.json()
profiling = data?.profiling ?? false profiling = data?.profiling ?? false
+2 -1
View File
@@ -75,6 +75,7 @@
"@hcengineering/server-client": "^0.6.0", "@hcengineering/server-client": "^0.6.0",
"@hcengineering/server-storage": "^0.6.0", "@hcengineering/server-storage": "^0.6.0",
"@hcengineering/postgres": "^0.6.0", "@hcengineering/postgres": "^0.6.0",
"@hcengineering/mongo": "^0.6.1" "@hcengineering/mongo": "^0.6.1",
"@hcengineering/kafka": "^0.6.0"
} }
} }
+4
View File
@@ -24,6 +24,7 @@ import { createRekoniAdapter, type FulltextDBConfiguration } from '@hcengineerin
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import { readFileSync } from 'fs' import { readFileSync } from 'fs'
import { startIndexer } from './server' import { startIndexer } from './server'
import { getPlatformQueue } from '@hcengineering/kafka'
const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[] const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[]
@@ -103,7 +104,10 @@ if (accountsUrl === undefined) {
const storageConfig: StorageConfiguration = storageConfigFromEnv() const storageConfig: StorageConfiguration = storageConfigFromEnv()
const externalStorage = buildStorageFromConfig(storageConfig) const externalStorage = buildStorageFromConfig(storageConfig)
const queue = getPlatformQueue('fulltext')
const onClose = startIndexer(metricsContext, { const onClose = startIndexer(metricsContext, {
queue,
model, model,
config, config,
externalStorage, externalStorage,
+152 -71
View File
@@ -4,22 +4,24 @@ import type {
Class, Class,
Doc, Doc,
DocumentQuery, DocumentQuery,
Domain,
IndexingUpdateEvent, IndexingUpdateEvent,
MeasureContext, MeasureContext,
Ref, Ref,
SearchOptions, SearchOptions,
SearchQuery, SearchQuery,
Tx, Tx,
TxCUD,
TxWorkspaceEvent, TxWorkspaceEvent,
WorkspaceUuid, WorkspaceIds,
WorkspaceIds WorkspaceUuid
} from '@hcengineering/core' } from '@hcengineering/core'
import core, { import core, {
DOMAIN_DOC_INDEX_STATE,
generateId, generateId,
Hierarchy, Hierarchy,
ModelDb, ModelDb,
systemAccountUuid, systemAccountUuid,
TxProcessor,
WorkspaceEvent WorkspaceEvent
} from '@hcengineering/core' } from '@hcengineering/core'
import { import {
@@ -44,15 +46,22 @@ import {
setDBExtraOptions, setDBExtraOptions,
shutdownPostgres shutdownPostgres
} from '@hcengineering/postgres' } from '@hcengineering/postgres'
import serverClientPlugin, { getTransactorEndpoint, getAccountClient } from '@hcengineering/server-client' import serverClientPlugin, { getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client'
import serverCore, { import serverCore, {
createContentAdapter, createContentAdapter,
createPipeline, createPipeline,
QueueTopic,
QueueWorkspaceEvent,
workspaceEvents,
type ConsumerControl,
type ContentTextAdapter, type ContentTextAdapter,
type FullTextAdapter, type FullTextAdapter,
type MiddlewareCreator, type MiddlewareCreator,
type Pipeline, type Pipeline,
type PipelineContext, type PipelineContext,
type PlatformQueue,
type QueueWorkspaceMessage,
type QueueWorkspaceReindexMessage,
type StorageAdapter type StorageAdapter
} from '@hcengineering/server-core' } from '@hcengineering/server-core'
import { FullTextIndexPipeline, searchFulltext, type FulltextDBConfiguration } from '@hcengineering/server-indexer' import { FullTextIndexPipeline, searchFulltext, type FulltextDBConfiguration } from '@hcengineering/server-indexer'
@@ -66,12 +75,31 @@ import {
setAdapterSecurity, setAdapterSecurity,
sharedPipelineContextVars sharedPipelineContextVars
} from '@hcengineering/server-pipeline' } from '@hcengineering/server-pipeline'
import serverToken, { decodeToken, generateToken, type Token } from '@hcengineering/server-token' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
import cors from '@koa/cors' import cors from '@koa/cors'
import Koa from 'koa' import Koa from 'koa'
import bodyParser from 'koa-bodyparser' import bodyParser from 'koa-bodyparser'
import Router from 'koa-router' import Router from 'koa-router'
function fulltextModelFilter (h: Hierarchy, model: Tx[]): Tx[] {
const allowedClasess: Ref<Class<Doc>>[] = [
core.class.Class,
core.class.Attribute,
core.class.Mixin,
core.class.Type,
core.class.Status,
core.class.Permission,
core.class.Space,
core.class.Tx,
core.class.FullTextSearchContext
]
return model.filter(
(it) =>
TxProcessor.isExtendsCUD(it._class) &&
allowedClasess.some((cl) => h.isDerived((it as TxCUD<Doc>).objectClass, cl))
)
}
class WorkspaceIndexer { class WorkspaceIndexer {
fulltext!: FullTextIndexPipeline fulltext!: FullTextIndexPipeline
pipeline!: Pipeline pipeline!: Pipeline
@@ -100,7 +128,7 @@ class WorkspaceIndexer {
ContextNameMiddleware.create, ContextNameMiddleware.create,
DomainFindMiddleware.create, DomainFindMiddleware.create,
DBAdapterInitMiddleware.create, DBAdapterInitMiddleware.create,
ModelMiddleware.create(model), ModelMiddleware.create(model, fulltextModelFilter), // TODO: Add filtration of only class structure and FullTextSearchContext
DBAdapterMiddleware.create(dbConf) DBAdapterMiddleware.create(dbConf)
] ]
@@ -153,40 +181,37 @@ class WorkspaceIndexer {
} }
// Send tx to pipeline // Send tx to pipeline
// TODO: Fix me // TODO: Fix me
void fetch(transactorEndpoint + `/api/v1/broadcast?token=${token}&workspace=${workspace.uuid}`, { void fetch(transactorEndpoint + `/api/v1/broadcast?workspace=${workspace.uuid}`, {
method: 'PUT', method: 'PUT',
keepalive: true,
headers: {
Authorization: `Bearer ${token}`
},
body: JSON.stringify(tx) body: JSON.stringify(tx)
}) })
},
async () => {
const helper = result.pipeline.context.adapterManager?.domainHelper
if (helper !== undefined) {
const dhelper = result.pipeline.context.adapterManager?.getAdapter(DOMAIN_DOC_INDEX_STATE, true).helper?.()
if (dhelper !== undefined) {
// Force creation of indexes for search domain
await helper.checkDomain(ctx, DOMAIN_DOC_INDEX_STATE, 10000, dhelper)
}
}
} }
) )
await result.fulltext.startIndexing(() => {
result.lastUpdate = Date.now()
})
return result return result
} }
async reindex (onlyDrop: boolean): Promise<void> { async reindex (
await this.fulltext.cancel() ctx: MeasureContext,
await this.fulltext.clearIndex(onlyDrop) domain: Domain,
if (!onlyDrop) { classes: Ref<Class<Doc>>[],
await this.fulltext.startIndexing(() => { control?: ConsumerControl
this.lastUpdate = Date.now() ): Promise<void> {
}) await this.fulltext.reindex(ctx, domain, classes, control)
} }
async dropWorkspace (): Promise<void> {
await this.fulltext.dropWorkspace()
}
async getIndexClassess (): Promise<{ domain: Domain, classes: Ref<Class<Doc>>[] }[]> {
return await this.fulltext.getIndexClassess()
} }
async close (): Promise<void> { async close (): Promise<void> {
await this.fulltext.cancel()
await this.pipeline.close() await this.pipeline.close()
} }
} }
@@ -229,6 +254,7 @@ process.on('exit', () => {
export async function startIndexer ( export async function startIndexer (
ctx: MeasureContext, ctx: MeasureContext,
opt: { opt: {
queue: PlatformQueue
model: Tx[] model: Tx[]
dbURL: string dbURL: string
config: FulltextDBConfiguration config: FulltextDBConfiguration
@@ -292,16 +318,89 @@ export async function startIndexer (
} }
}, closeTimeout) // Every 5 minutes we should close unused indexes. }, closeTimeout) // Every 5 minutes we should close unused indexes.
const workspaceProducer = opt.queue.createProducer<QueueWorkspaceMessage>(ctx, QueueTopic.Workspace)
const workspaceConsumer = opt.queue.createConsumer<QueueWorkspaceMessage>(
ctx,
QueueTopic.Workspace,
opt.queue.getClientId(),
async (msg, control) => {
for (const m of msg) {
ctx.info('workspace message', { message: m })
const ws = m.id as WorkspaceUuid
for (const mm of m.value) {
if (
mm.type === QueueWorkspaceEvent.Created ||
mm.type === QueueWorkspaceEvent.Restored ||
mm.type === QueueWorkspaceEvent.FullReindex
) {
const indexer = await getIndexer(ctx, ws, generateToken(systemAccountUuid, ws), true)
if (indexer !== undefined) {
await indexer.dropWorkspace() // TODO: Add heartbeat
const classes = await indexer.getIndexClassess()
await workspaceProducer.send(
ws,
classes.map((it) => workspaceEvents.reindex(it.domain, it.classes))
)
}
} else if (
mm.type === QueueWorkspaceEvent.Deleted ||
mm.type === QueueWorkspaceEvent.Archived ||
mm.type === QueueWorkspaceEvent.ClearIndex
) {
const token = generateToken(systemAccountUuid, ws)
const accountClient = getAccountClient(token)
const workspaceInfo = await accountClient.getWorkspaceInfo(false)
if (workspaceInfo !== undefined) {
if (workspaceInfo.dataId != null) {
await fulltextAdapter.clean(ctx, workspaceInfo.dataId as unknown as WorkspaceUuid)
}
await fulltextAdapter.clean(ctx, workspaceInfo.uuid)
}
} else if (mm.type === QueueWorkspaceEvent.Reindex) {
const indexer = await getIndexer(ctx, ws, generateToken(systemAccountUuid, ws), true)
const mmd = mm as QueueWorkspaceReindexMessage
await indexer?.reindex(ctx, mmd.domain, mmd.classes, control)
}
}
}
}
)
let txInformer: any
let txMessages: number = 0
const txConsumer = opt.queue.createConsumer<TxCUD<Doc>>(
ctx,
QueueTopic.Tx,
opt.queue.getClientId(),
async (msg, control) => {
clearTimeout(txInformer)
txInformer = setTimeout(() => {
ctx.info('tx message', { count: txMessages })
txMessages = 0
}, 5000)
txMessages += msg.length
for (const m of msg) {
const ws = m.id as WorkspaceUuid
const indexer = await getIndexer(ctx, ws, generateToken(systemAccountUuid, ws), true)
await indexer?.fulltext.processDocuments(ctx, m.value, control)
}
}
)
async function getIndexer ( async function getIndexer (
ctx: MeasureContext, ctx: MeasureContext,
workspace: WorkspaceUuid, workspace: WorkspaceUuid,
token: string, token: string | undefined,
create: boolean = false create: boolean = false
): Promise<WorkspaceIndexer | undefined> { ): Promise<WorkspaceIndexer | undefined> {
const accountClient = getAccountClient(token)
const workspaceInfo = await accountClient.getWorkspaceInfo(false)
let idx = indexers.get(workspace) let idx = indexers.get(workspace)
if (idx === undefined && create) { if (idx === undefined && create) {
const accountClient = getAccountClient(token)
const workspaceInfo = await accountClient.getWorkspaceInfo(false)
if (workspaceInfo === undefined) { if (workspaceInfo === undefined) {
ctx.error('Workspace not available for token') ctx.error('Workspace not available for token')
return return
@@ -339,7 +438,8 @@ export async function startIndexer (
router.put('/api/v1/search', async (req, res) => { router.put('/api/v1/search', async (req, res) => {
try { try {
const request = req.request.body as Search const request = req.request.body as Search
const decoded = decodeToken(request.token) // Just to be safe const token = request.token ?? req.headers.authorization?.split(' ')[1]
const decoded = decodeToken(token) // Just to be safe
ctx.info('search', { classes: request._classes, query: request.query, workspace: decoded.workspace }) ctx.info('search', { classes: request._classes, query: request.query, workspace: decoded.workspace })
await ctx.with('search', {}, async (ctx) => { await ctx.with('search', {}, async (ctx) => {
@@ -348,8 +448,6 @@ export async function startIndexer (
) )
req.body = docs req.body = docs
}) })
void triggerIndexer(decoded, request.token)
} catch (err: any) { } catch (err: any) {
Analytics.handleError(err) Analytics.handleError(err)
console.error(err) console.error(err)
@@ -361,7 +459,8 @@ export async function startIndexer (
router.put('/api/v1/full-text-search', async (req, res) => { router.put('/api/v1/full-text-search', async (req, res) => {
try { try {
const request = req.request.body as FulltextSearch const request = req.request.body as FulltextSearch
const decoded = decodeToken(request.token) // Just to be safe const token = request.token ?? req.headers.authorization?.split(' ')[1]
const decoded = decodeToken(token) // Just to be safe
ctx.info('fulltext-search', { ...request.query, workspace: decoded.workspace }) ctx.info('fulltext-search', { ...request.query, workspace: decoded.workspace })
await ctx.with('full-text-search', {}, async (ctx) => { await ctx.with('full-text-search', {}, async (ctx) => {
const result = await ctx.with('searchFulltext', {}, (ctx) => const result = await ctx.with('searchFulltext', {}, (ctx) =>
@@ -369,24 +468,6 @@ export async function startIndexer (
) )
req.body = result req.body = result
}) })
void triggerIndexer(decoded, request.token)
} catch (err: any) {
Analytics.handleError(err)
console.error(err)
req.res.writeHead(404, {})
req.res.end()
}
})
router.put('/api/v1/warmup', async (req, res) => {
try {
const request = req.request.body as IndexDocuments
const decoded = decodeToken(request.token) // Just to be safe
req.body = {}
ctx.info('warm-up', { workspace: decoded.workspace })
void triggerIndexer(decoded, request.token, true)
} catch (err: any) { } catch (err: any) {
Analytics.handleError(err) Analytics.handleError(err)
console.error(err) console.error(err)
@@ -398,7 +479,8 @@ export async function startIndexer (
router.put('/api/v1/close', async (req, res) => { router.put('/api/v1/close', async (req, res) => {
try { try {
const request = req.request.body as IndexDocuments const request = req.request.body as IndexDocuments
const decoded = decodeToken(request.token) // Just to be safe const token = request.token ?? req.headers.authorization?.split(' ')[1]
const decoded = decodeToken(token) // Just to be safe
req.body = {} req.body = {}
ctx.info('close', { workspace: decoded.workspace }) ctx.info('close', { workspace: decoded.workspace })
@@ -422,12 +504,14 @@ export async function startIndexer (
router.put('/api/v1/index-documents', async (req, res) => { router.put('/api/v1/index-documents', async (req, res) => {
try { try {
const request = req.request.body as IndexDocuments const request = req.request.body as IndexDocuments
const decoded = decodeToken(request.token) // Just to be safe const token = request.token ?? req.headers.authorization?.split(' ')[1]
const decoded = decodeToken(token) // Just to be safe
const indexer = await getIndexer(ctx, decoded.workspace, request.token) const indexer = await getIndexer(ctx, decoded.workspace, token)
if (indexer !== undefined) { if (indexer !== undefined) {
indexer.lastUpdate = Date.now() indexer.lastUpdate = Date.now()
await ctx.with('index-documents', {}, (ctx) => indexer.fulltext.indexDocuments(ctx, request.requests)) // TODO: Fixme
// await ctx.with('index-documents', {}, (ctx) => indexer.fulltext.indexDocuments(ctx, request.requests))
} }
req.body = {} req.body = {}
} catch (err: any) { } catch (err: any) {
@@ -441,14 +525,19 @@ export async function startIndexer (
router.put('/api/v1/reindex', async (req, res) => { router.put('/api/v1/reindex', async (req, res) => {
try { try {
const request = req.request.body as Reindex const request = req.request.body as Reindex
const decoded = decodeToken(request.token) // Just to be safe const token = request.token ?? req.headers.authorization?.split(' ')[1]
const decoded = decodeToken(token) // Just to be safe
req.body = {} req.body = {}
ctx.info('reindex', { workspace: decoded.workspace }) ctx.info('reindex', { workspace: decoded.workspace })
const indexer = await getIndexer(ctx, decoded.workspace, request.token, true) const indexer = await getIndexer(ctx, decoded.workspace, token, true)
if (indexer !== undefined) { if (indexer !== undefined) {
indexer.lastUpdate = Date.now() indexer.lastUpdate = Date.now()
await indexer.reindex(request?.onlyDrop ?? false) if (request?.onlyDrop ?? false) {
await workspaceProducer.send(decoded.workspace, [workspaceEvents.clearIndex()])
} else {
await workspaceProducer.send(decoded.workspace, [workspaceEvents.fullReindex()])
}
} }
} catch (err: any) { } catch (err: any) {
Analytics.handleError(err) Analytics.handleError(err)
@@ -465,19 +554,11 @@ export async function startIndexer (
}) })
const close = (): void => { const close = (): void => {
void txConsumer.shutdown()
void workspaceConsumer.shutdown()
clearInterval(shutdownInterval) clearInterval(shutdownInterval)
server.close() server.close()
} }
return close return close
async function triggerIndexer (decoded: Token, token: string, trigger: boolean = false): Promise<void> {
const indexer = await getIndexer(ctx, decoded.workspace, token, trigger)
if (indexer !== undefined) {
indexer.lastUpdate = Date.now()
if (trigger) {
indexer.fulltext.triggerIndexing()
}
}
}
} }
+2 -1
View File
@@ -81,6 +81,7 @@
"msgpackr-extract": "^3.0.3", "msgpackr-extract": "^3.0.3",
"@hcengineering/postgres": "^0.6.0", "@hcengineering/postgres": "^0.6.0",
"snappy": "^7.2.2", "snappy": "^7.2.2",
"@hcengineering/rpc": "^0.6.5" "@hcengineering/rpc": "^0.6.5",
"@hcengineering/kafka": "^0.6.0"
} }
} }
+20 -6
View File
@@ -4,22 +4,24 @@
// Add this to the VERY top of the first file loaded in your app // Add this to the VERY top of the first file loaded in your app
import { Analytics } from '@hcengineering/analytics' import { Analytics } from '@hcengineering/analytics'
import { SplitLogger, configureAnalytics } from '@hcengineering/analytics-service' import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service'
import contactPlugin from '@hcengineering/contact' import contactPlugin from '@hcengineering/contact'
import { MeasureMetricsContext, newMetrics, setOperationLogProfiling } from '@hcengineering/core' import { MeasureMetricsContext, newMetrics, setOperationLogProfiling } from '@hcengineering/core'
import { getPlatformQueue } from '@hcengineering/kafka'
import { setMetadata } from '@hcengineering/platform' import { setMetadata } from '@hcengineering/platform'
import { setDBExtraOptions } from '@hcengineering/postgres'
import { serverConfigFromEnv } from '@hcengineering/server' import { serverConfigFromEnv } from '@hcengineering/server'
import serverAiBot from '@hcengineering/server-ai-bot' import serverAiBot from '@hcengineering/server-ai-bot'
import serverCalendar from '@hcengineering/server-calendar' import serverCalendar from '@hcengineering/server-calendar'
import serverCore, { import serverCore, {
initStatisticsContext,
loadBrandingMap,
type ConnectionSocket, type ConnectionSocket,
type Session, type Session,
type StorageConfiguration, type StorageConfiguration,
type UserStatistics, type UserStatistics,
type Workspace, type Workspace,
type WorkspaceStatistics, type WorkspaceStatistics
initStatisticsContext,
loadBrandingMap
} from '@hcengineering/server-core' } from '@hcengineering/server-core'
import serverNotification from '@hcengineering/server-notification' import serverNotification from '@hcengineering/server-notification'
import { storageConfigFromEnv } from '@hcengineering/server-storage' import { storageConfigFromEnv } from '@hcengineering/server-storage'
@@ -29,7 +31,6 @@ import { startHttpServer } from '@hcengineering/server-ws'
import { join } from 'path' import { join } from 'path'
import { start } from '.' import { start } from '.'
import { profileStart, profileStop } from './inspector' import { profileStart, profileStop } from './inspector'
import { setDBExtraOptions } from '@hcengineering/postgres'
configureAnalytics(process.env.SENTRY_DSN, {}) configureAnalytics(process.env.SENTRY_DSN, {})
Analytics.setTag('application', 'transactor') Analytics.setTag('application', 'transactor')
@@ -37,6 +38,18 @@ Analytics.setTag('application', 'transactor')
let getUsers: () => WorkspaceStatistics[] = () => { let getUsers: () => WorkspaceStatistics[] = () => {
return [] return []
} }
const queueConfig = process.env.QUEUE_CONFIG
if (queueConfig === undefined) {
throw new Error('Please provide queue config')
}
const queue = getPlatformQueue('transactor')
void queue.createTopics(10).catch((err) => {
console.error('Failed to create required topics', err)
})
// Force create server metrics context with proper logging // Force create server metrics context with proper logging
const metricsContext = initStatisticsContext('transactor', { const metricsContext = initStatisticsContext('transactor', {
getUsers: (): WorkspaceStatistics[] => { getUsers: (): WorkspaceStatistics[] => {
@@ -90,7 +103,8 @@ const { shutdown, sessionManager } = start(metricsContext, config.dbUrl, {
start: profileStart, start: profileStart,
stop: profileStop stop: profileStop
}, },
mongoUrl: config.mongoUrl mongoUrl: config.mongoUrl,
queue
}) })
const entryToUserStats = (session: Session, socket: ConnectionSocket): UserStatistics => { const entryToUserStats = (session: Session, socket: ConnectionSocket): UserStatistics => {
-138
View File
@@ -1,138 +0,0 @@
import core, {
DOMAIN_TX,
generateId,
MeasureMetricsContext,
type WorkspaceDataId,
type WorkspaceUuid,
type Doc,
type LowLevelStorage,
type Ref,
type TxCreateDoc
} from '@hcengineering/core'
import builder from '@hcengineering/model-all'
import { wrapPipeline } from '@hcengineering/server-core'
import { getServerPipeline } from '@hcengineering/server-pipeline'
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
const model = builder().getTxes()
// const dbURL = 'postgresql://root@localhost:26257/defaultdb?sslmode=disable'
const dbURL = 'postgresql://postgres:example@localhost:5432'
const STORAGE_CONFIG = 'minio|localhost:9000?accessKey=minioadmin&secretKey=minioadmin&useSSL=false'
// jest.setTimeout(4500000)
describe.skip('test-backup-find', () => {
it('check create/load/clean', async () => {
const toolCtx = new MeasureMetricsContext('-', {})
// We should setup a DB with docuemnts and try to backup them.
const wsIds = { dataId: 'testdb-backup-test' as WorkspaceDataId, uuid: 'test' as WorkspaceUuid, url: 'test' }
const storageConfig = storageConfigFromEnv(STORAGE_CONFIG)
const storageAdapter = buildStorageFromConfig(storageConfig)
const pipeline = await getServerPipeline(toolCtx, model, dbURL, wsIds, storageAdapter, {
disableTriggers: true
})
try {
const client = wrapPipeline(toolCtx, pipeline, wsIds)
const lowLevel = pipeline.context.lowLevelStorage as LowLevelStorage
// We need to create a backup docs if they are missing.
await prepareTxes(lowLevel, toolCtx)
const docs: Doc[] = []
while (true) {
const chunk = await client.loadChunk(DOMAIN_TX, 0)
const part = await client.loadDocs(
DOMAIN_TX,
chunk.docs.map((doc) => doc.id as Ref<Doc>)
)
docs.push(...part)
if (chunk.finished) {
break
}
}
await client.closeChunk(0)
expect(docs.length).toBeGreaterThan(459)
await client.clean(
DOMAIN_TX,
docs.map((doc) => doc._id)
)
const findDocs = await client.findAll(core.class.Tx, {})
expect(findDocs.length).toBe(0)
//
} finally {
await pipeline.close()
await storageAdapter.close()
}
})
it('check traverse', async () => {
const toolCtx = new MeasureMetricsContext('-', {})
// We should setup a DB with docuemnts and try to backup them.
const wsIds = { dataId: 'testdb-backup-test' as WorkspaceDataId, uuid: 'test' as WorkspaceUuid, url: 'test' }
const storageConfig = storageConfigFromEnv(STORAGE_CONFIG)
const storageAdapter = buildStorageFromConfig(storageConfig)
const pipeline = await getServerPipeline(toolCtx, model, dbURL, wsIds, storageAdapter, {
disableTriggers: true
})
try {
const client = wrapPipeline(toolCtx, pipeline, wsIds)
const lowLevel = pipeline.context.lowLevelStorage as LowLevelStorage
// We need to create a backup docs if they are missing.
await prepareTxes(lowLevel, toolCtx, 1500)
const iter = await lowLevel.traverse(DOMAIN_TX, {})
const allDocs: Doc[] = []
while (true) {
const docs = await iter.next(50)
if (docs == null || docs?.length === 0) {
break
}
await client.clean(
DOMAIN_TX,
docs.map((doc) => doc._id)
)
allDocs.push(...docs)
}
expect(allDocs.length).toBeGreaterThan(1449)
const findDocs = await client.findAll(core.class.Tx, {})
expect(findDocs.length).toBe(0)
//
} finally {
await pipeline.close()
await storageAdapter.close()
}
})
})
async function prepareTxes (
lowLevel: LowLevelStorage,
toolCtx: MeasureMetricsContext,
count: number = 500
): Promise<void> {
const docs = await lowLevel.rawFindAll(DOMAIN_TX, {})
if ((docs?.length ?? 0) < count) {
// We need to fill some documents to be pressent
const docs: TxCreateDoc<Doc>[] = []
for (let i = 0; i < count; i++) {
docs.push({
_class: core.class.TxCreateDoc,
_id: generateId(),
space: core.space.Tx,
modifiedBy: core.account.ConfigUser,
modifiedOn: Date.now(),
attributes: {
qwe: generateId()
},
objectClass: core.class.DocIndexState,
objectId: generateId(),
objectSpace: core.space.Workspace
})
}
await lowLevel.upload(toolCtx, DOMAIN_TX, docs)
}
}
+6 -3
View File
@@ -23,7 +23,8 @@ import {
type Session, type Session,
type SessionManager, type SessionManager,
type StorageConfiguration, type StorageConfiguration,
type Workspace type Workspace,
type PlatformQueue
} from '@hcengineering/server-core' } from '@hcengineering/server-core'
import { type Token } from '@hcengineering/server-token' import { type Token } from '@hcengineering/server-token'
@@ -76,6 +77,7 @@ export function start (
metrics: MeasureContext, metrics: MeasureContext,
dbUrl: string, dbUrl: string,
opt: { opt: {
queue: PlatformQueue
fulltextUrl: string fulltextUrl: string
storageConfig: StorageConfiguration storageConfig: StorageConfiguration
port: number port: number
@@ -120,7 +122,7 @@ export function start (
metrics, metrics,
dbUrl, dbUrl,
model, model,
{ ...opt, externalStorage, adapterSecurity: isAdapterSecurity(dbUrl) }, { ...opt, externalStorage, adapterSecurity: isAdapterSecurity(dbUrl), queue: opt.queue },
{} {}
) )
const sessionFactory = (token: Token, workspace: Workspace, account: Account): Session => { const sessionFactory = (token: Token, workspace: Workspace, account: Account): Session => {
@@ -136,7 +138,8 @@ export function start (
enableCompression: opt.enableCompression, enableCompression: opt.enableCompression,
accountsUrl: opt.accountsUrl, accountsUrl: opt.accountsUrl,
externalStorage, externalStorage,
profiling: opt.profiling profiling: opt.profiling,
queue: opt.queue
}) })
return { return {
shutdown: async () => { shutdown: async () => {
+2 -1
View File
@@ -56,6 +56,7 @@
"@hcengineering/server-core": "^0.6.1", "@hcengineering/server-core": "^0.6.1",
"@hcengineering/model-all": "^0.6.0", "@hcengineering/model-all": "^0.6.0",
"@hcengineering/analytics": "^0.6.0", "@hcengineering/analytics": "^0.6.0",
"@hcengineering/analytics-service": "^0.6.0" "@hcengineering/analytics-service": "^0.6.0",
"@hcengineering/kafka": "^0.6.0"
} }
} }
+4
View File
@@ -4,6 +4,7 @@
import { Analytics } from '@hcengineering/analytics' import { Analytics } from '@hcengineering/analytics'
import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service' import { configureAnalytics, SplitLogger } from '@hcengineering/analytics-service'
import { MeasureMetricsContext, newMetrics, type Tx } from '@hcengineering/core' import { MeasureMetricsContext, newMetrics, type Tx } from '@hcengineering/core'
import { getPlatformQueue } from '@hcengineering/kafka'
import builder, { getModelVersion, migrateOperations } from '@hcengineering/model-all' import builder, { getModelVersion, migrateOperations } from '@hcengineering/model-all'
import { initStatisticsContext, loadBrandingMap } from '@hcengineering/server-core' import { initStatisticsContext, loadBrandingMap } from '@hcengineering/server-core'
import { serveWorkspaceAccount } from '@hcengineering/workspace-service' import { serveWorkspaceAccount } from '@hcengineering/workspace-service'
@@ -34,8 +35,11 @@ const metricsContext = initStatisticsContext('workspace', {
const brandingPath = process.env.BRANDING_PATH const brandingPath = process.env.BRANDING_PATH
const queue = getPlatformQueue('workspace')
serveWorkspaceAccount( serveWorkspaceAccount(
metricsContext, metricsContext,
queue,
getModelVersion(), getModelVersion(),
txes, txes,
migrateOperations, migrateOperations,
+2 -1
View File
@@ -1,2 +1,3 @@
STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
MONGO_URL=mongodb://mongodb:27018?compressors=snappy MONGO_URL=mongodb://mongodb:27018?compressors=snappy
QUEUE_CONFIG=redpanda:9093
+53
View File
@@ -11,6 +11,55 @@ services:
ports: ports:
- 27018:27018 - 27018:27018
restart: unless-stopped restart: unless-stopped
redpanda:
image: docker.redpanda.com/redpandadata/redpanda:v24.3.6
extra_hosts:
- 'huly.local:host-gateway'
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9093,external://0.0.0.0:19093
- --advertise-kafka-addr internal://redpanda:9093,external://localhost:19093
- --pandaproxy-addr internal://0.0.0.0:8084,external://0.0.0.0:18084
- --advertise-pandaproxy-addr internal://redpanda:8084,external://localhost:18084
- --schema-registry-addr internal://0.0.0.0:8083,external://0.0.0.0:18083
- --rpc-addr redpanda:33145
- --advertise-rpc-addr redpanda:33145
- --mode dev-container
- --smp 1
- --default-log-level=info
ports:
- 18083:18083
- 18084:18084
- 19093:19093
- 19645:9644
healthcheck:
test: ['CMD', 'rpk', 'cluster', 'info', '-X', 'user=superuser', '-X', 'pass=secretpassword']
interval: 10s
timeout: 5s
retries: 10
redpanda_console:
image: docker.redpanda.com/redpandadata/console:v2.8.3
extra_hosts:
- 'huly.local:host-gateway'
entrypoint: /bin/sh
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
environment:
CONFIG_FILEPATH: /tmp/config.yml
CONSOLE_CONFIG_FILE: |
kafka:
brokers: ["redpanda:9093"]
schemaRegistry:
enabled: true
urls: ["http://redpanda:8083"]
redpanda:
adminApi:
enabled: true
urls: ["http://redpanda:9644"]
ports:
- 8001:8080
depends_on:
- redpanda
minio: minio:
image: 'minio/minio' image: 'minio/minio'
command: server /data --address ":9000" command: server /data --address ":9000"
@@ -51,6 +100,7 @@ services:
volumes: volumes:
- ./branding-test.json:/var/cfg/branding-test.json - ./branding-test.json:/var/cfg/branding-test.json
environment: environment:
- QUEUE_CONFIG=${QUEUE_CONFIG}
- ACCOUNT_PORT=3003 - ACCOUNT_PORT=3003
- SERVER_SECRET=secret - SERVER_SECRET=secret
- WORKSPACE_LIMIT_PER_USER=100 - WORKSPACE_LIMIT_PER_USER=100
@@ -69,6 +119,7 @@ services:
volumes: volumes:
- ./branding-test.json:/var/cfg/branding.json - ./branding-test.json:/var/cfg/branding.json
environment: environment:
- QUEUE_CONFIG=${QUEUE_CONFIG}
- SERVER_SECRET=secret - SERVER_SECRET=secret
- DB_URL=mongodb://mongodb:27018 - DB_URL=mongodb://mongodb:27018
- MONGO_URL=mongodb://mongodb:27018 - MONGO_URL=mongodb://mongodb:27018
@@ -124,6 +175,7 @@ services:
volumes: volumes:
- ./branding-test.json:/var/cfg/branding-test.json - ./branding-test.json:/var/cfg/branding-test.json
environment: environment:
- QUEUE_CONFIG=${QUEUE_CONFIG}
- SERVER_PROVIDER=${SERVER_PROVIDER} - SERVER_PROVIDER=${SERVER_PROVIDER}
- SERVER_PORT=3334 - SERVER_PORT=3334
- SERVER_SECRET=secret - SERVER_SECRET=secret
@@ -212,6 +264,7 @@ services:
ports: ports:
- 4710:4710 - 4710:4710
environment: environment:
- QUEUE_CONFIG=${QUEUE_CONFIG}
- PORT=4710 - PORT=4710
- SERVER_SECRET=secret - SERVER_SECRET=secret
- DB_URL=mongodb://mongodb:27018 - DB_URL=mongodb://mongodb:27018
+2
View File
@@ -16,6 +16,8 @@ if [ "x$DO_CLEAN" == 'xtrue' ]; then
docker system prune -a -f docker system prune -a -f
fi fi
../tests/wait-elastic.sh 9201
# Create user record in accounts # Create user record in accounts
./tool.sh create-account user1 -f John -l Appleseed -p 1234 ./tool.sh create-account user1 -f John -l Appleseed -p 1234
./tool.sh create-account user2 -f Kainin -l Dirak -p 1234 ./tool.sh create-account user2 -f Kainin -l Dirak -p 1234
+1
View File
@@ -8,6 +8,7 @@ export ACCOUNT_DB_URL=mongodb://localhost:27017
export ACCOUNTS_URL=http://localhost:3000 export ACCOUNTS_URL=http://localhost:3000
export TRANSACTOR_URL=ws://localhost:3333 export TRANSACTOR_URL=ws://localhost:3333
export SERVER_SECRET=secret export SERVER_SECRET=secret
export QUEUE_CONFIG=huly.local:19092
# Restore workspace contents in mongo/elastic # Restore workspace contents in mongo/elastic
node ../dev/tool/bundle/bundle.js $@ node ../dev/tool/bundle/bundle.js $@
+1
View File
@@ -10,5 +10,6 @@ export ACCOUNT_DB_URL=mongodb://localhost:27018
export DB_URL=mongodb://localhost:27018 export DB_URL=mongodb://localhost:27018
export SERVER_SECRET=secret export SERVER_SECRET=secret
export STORAGE_CONFIG="minio|localhost:9002?accessKey=minioadmin&secretKey=minioadmin" export STORAGE_CONFIG="minio|localhost:9002?accessKey=minioadmin&secretKey=minioadmin"
export QUEUE_CONFIG=huly.local:19093
node ../dev/tool/bundle/bundle.js $@ node ../dev/tool/bundle/bundle.js $@
+5 -15
View File
@@ -1518,21 +1518,6 @@
"projectFolder": "pods/fulltext", "projectFolder": "pods/fulltext",
"shouldPublish": false "shouldPublish": false
}, },
{
"packageName": "@hcengineering/server-fulltext",
"projectFolder": "server-plugins/fulltext",
"shouldPublish": false
},
{
"packageName": "@hcengineering/server-fulltext-resources",
"projectFolder": "server-plugins/fulltext-resources",
"shouldPublish": false
},
{
"packageName": "@hcengineering/model-server-fulltext",
"projectFolder": "models/server-fulltext",
"shouldPublish": false
},
{ {
"packageName": "@hcengineering/hr", "packageName": "@hcengineering/hr",
"projectFolder": "plugins/hr", "projectFolder": "plugins/hr",
@@ -2397,6 +2382,11 @@
"packageName": "@hcengineering/server-process-resources", "packageName": "@hcengineering/server-process-resources",
"projectFolder": "server-plugins/process-resources", "projectFolder": "server-plugins/process-resources",
"shouldPublish": false "shouldPublish": false
},
{
"packageName": "@hcengineering/kafka",
"projectFolder": "server/kafka",
"shouldPublish": false
} }
] ]
} }
+1 -1
View File
@@ -38,7 +38,7 @@ export async function createAccountRequest (workspace: WorkspaceUuid, ctx: Measu
body: JSON.stringify({}) body: JSON.stringify({})
}) })
} catch (err) { } catch (err) {
ctx.error('Could not send create ai account request', { err }) ctx.warn('Could not send create ai account request', { err })
} }
} }
@@ -1,7 +0,0 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}
@@ -1,4 +0,0 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/
@@ -1,4 +0,0 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}
@@ -1,103 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import core, {
DocIndexState,
isClassIndexable,
isFullTextAttribute,
Tx,
TxProcessor,
type AnyAttribute,
type Class,
type Doc,
type FullTextSearchContext,
type Ref,
type TxCUD,
type TxUpdateDoc
} from '@hcengineering/core'
import { TriggerControl } from '@hcengineering/server-core'
const indexingCtx = 'indexing_contexts'
export async function OnChange (txes: Tx[], control: TriggerControl): Promise<Tx[]> {
let contexts = control.contextCache.get(indexingCtx) as Map<Ref<Class<Doc>>, FullTextSearchContext>
if (contexts === undefined) {
contexts = new Map()
control.contextCache.set(indexingCtx, contexts)
}
if (control.ctx.contextData.fulltextUpdates === undefined) {
control.ctx.contextData.fulltextUpdates = new Map()
}
const docsToUpsert = control.ctx.contextData.fulltextUpdates
for (const tx of txes) {
const attrs = new Map<string, AnyAttribute>()
if (TxProcessor.isExtendsCUD(tx._class)) {
const cud = tx as TxCUD<Doc>
if (!isClassIndexable(control.hierarchy, cud.objectClass, contexts)) {
// No need, since no indixable fields or attachments.
continue
}
// We need to check if operation has indexable attributes.
if (cud._class === core.class.TxUpdateDoc) {
let hasFulltext = false
// We could skip updates for a lot of transactions if they have non idexable attributes
const upd = cud as TxUpdateDoc<Doc>
for (const [k] of Object.entries(upd.operations)) {
if (k.startsWith('$') || k === 'modifiedOn') {
// Is operator
// Skip operator changes
} else {
const key = upd.objectClass + '.' + k
const attr = attrs.get(key) ?? control.hierarchy.findAttribute(upd.objectClass, k)
if (attr !== undefined) {
attrs.set(key, attr ?? null)
}
if (attr != null) {
if (isFullTextAttribute(attr)) {
hasFulltext = true
}
}
}
}
if (!hasFulltext) {
// No full text changes, no indexing is required
continue
}
}
docsToUpsert.set(cud.objectId as Ref<DocIndexState>, {
_id: cud.objectId as Ref<DocIndexState>,
_class: core.class.DocIndexState,
objectClass: cud.objectClass,
needIndex: true,
modifiedBy: tx.modifiedBy,
modifiedOn: tx.modifiedOn,
space: cud.objectSpace,
removed: cud._class === core.class.TxRemoveDoc
})
}
}
return []
}
// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
export default async () => ({
trigger: {
OnChange
}
})
@@ -1,10 +0,0 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"declarationDir": "./types",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}
-7
View File
@@ -1,7 +0,0 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}
-4
View File
@@ -1,4 +0,0 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/
-4
View File
@@ -1,4 +0,0 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}
-7
View File
@@ -1,7 +0,0 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
roots: ["./src"],
coverageReporters: ["text-summary", "html"]
}
-44
View File
@@ -1,44 +0,0 @@
{
"name": "@hcengineering/server-fulltext",
"version": "0.6.0",
"main": "lib/index.js",
"svelte": "src/index.ts",
"types": "types/index.d.ts",
"files": [
"lib/**/*",
"types/**/*",
"tsconfig.json"
],
"author": "Anticrm Platform Contributors",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"format": "format src",
"test": "jest --passWithNoTests --silent",
"_phase:build": "compile transpile src",
"_phase:test": "jest --passWithNoTests --silent",
"_phase:format": "format src",
"_phase:validate": "compile validate"
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@types/node": "~20.11.16",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5"
},
"dependencies": {
"@hcengineering/platform": "^0.6.11",
"@hcengineering/server-core": "^0.6.1"
}
}
-33
View File
@@ -1,33 +0,0 @@
//
// Copyright © 2020, 2021 Anticrm Platform Contributors.
// Copyright © 2021, 2022 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import type { Plugin, Resource } from '@hcengineering/platform'
import { plugin } from '@hcengineering/platform'
import type { TriggerFunc } from '@hcengineering/server-core'
/**
* @public
*/
export const serverFulltextId = 'server-fulltext' as Plugin
/**
* @public
*/
export default plugin(serverFulltextId, {
trigger: {
OnChange: '' as Resource<TriggerFunc>
}
})
-10
View File
@@ -1,10 +0,0 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"declarationDir": "./types",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}
@@ -125,8 +125,8 @@ async function handleVacancyUpdate (control: TriggerControl, cud: TxCUD<Doc>, re
objectId: updateTx.objectId objectId: updateTx.objectId
}) })
).filter((it) => it._id !== updateTx._id) ).filter((it) => it._id !== updateTx._id)
const vacancy = TxProcessor.buildDoc2Doc(txes) as Vacancy const vacancy = TxProcessor.buildDoc2Doc<Vacancy>(txes)
if (vacancy.company != null) { if (vacancy?.company != null) {
// We have old value // We have old value
res.push( res.push(
control.txFactory.createTxMixin( control.txFactory.createTxMixin(
@@ -331,7 +331,7 @@ async function doTimeReportUpdate (cud: TxCUD<TimeSpendReport>, control: Trigger
objectId: cud.objectId objectId: cud.objectId
}) })
).filter((it) => it._id !== cud._id) ).filter((it) => it._id !== cud._id)
const doc: TimeSpendReport | undefined = TxProcessor.buildDoc2Doc(logTxes) const doc = TxProcessor.buildDoc2Doc<TimeSpendReport>(logTxes)
const res: Tx[] = [] const res: Tx[] = []
const [currentIssue] = await control.findAll( const [currentIssue] = await control.findAll(
@@ -340,7 +340,7 @@ async function doTimeReportUpdate (cud: TxCUD<TimeSpendReport>, control: Trigger
{ _id: attachedTo }, { _id: attachedTo },
{ limit: 1 } { limit: 1 }
) )
if (doc !== undefined) { if (doc != null) {
res.push( res.push(
control.txFactory.createTxUpdateDoc<Issue>( control.txFactory.createTxUpdateDoc<Issue>(
attachedToClass, attachedToClass,
@@ -370,8 +370,8 @@ async function doTimeReportUpdate (cud: TxCUD<TimeSpendReport>, control: Trigger
objectId: cud.objectId objectId: cud.objectId
}) })
).filter((it) => it._id !== cud._id) ).filter((it) => it._id !== cud._id)
const doc: TimeSpendReport | undefined = TxProcessor.buildDoc2Doc(logTxes) const doc = TxProcessor.buildDoc2Doc<TimeSpendReport>(logTxes)
if (doc !== undefined) { if (doc != null) {
const [currentIssue] = await control.findAll( const [currentIssue] = await control.findAll(
control.ctx, control.ctx,
tracker.class.Issue, tracker.class.Issue,
+1 -1
View File
@@ -562,7 +562,7 @@ export async function selectWorkspace (
throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspaceUrl })) throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspaceUrl }))
} }
if (accountUuid === systemAccountUuid) { if (accountUuid === systemAccountUuid || extra?.admin === 'true') {
return { return {
account: accountUuid, account: accountUuid,
token: generateToken(accountUuid, workspace.uuid, extra), token: generateToken(accountUuid, workspace.uuid, extra),
+2 -48
View File
@@ -21,7 +21,6 @@ import core, {
Doc, Doc,
Domain, Domain,
DOMAIN_BLOB, DOMAIN_BLOB,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MODEL, DOMAIN_MODEL,
DOMAIN_MODEL_TX, DOMAIN_MODEL_TX,
DOMAIN_TRANSIENT, DOMAIN_TRANSIENT,
@@ -36,7 +35,6 @@ import core, {
TxProcessor, TxProcessor,
type BackupStatus, type BackupStatus,
type Blob, type Blob,
type DocIndexState,
type Tx, type Tx,
type TxCUD, type TxCUD,
type WorkspaceIds, type WorkspaceIds,
@@ -623,22 +621,6 @@ export async function cloneWorkspace (
// ) // )
} }
// function prepareClonedDocuments (docs: Doc[]): Doc[] {
// docs = docs.map((p) => {
// // if full text is skipped, we need to clean stages for indexes.
// if (p._class === core.class.DocIndexState) {
// ;(p as DocIndexState).needIndex = true
// }
// return {
// ...p,
// modifiedOn: Date.now(),
// createdOn: Date.now()
// }
// })
// return docs
// }
// async function cleanDomain (ctx: MeasureContext, connection: CoreClient & BackupClient, domain: Domain): Promise<void> { // async function cleanDomain (ctx: MeasureContext, connection: CoreClient & BackupClient, domain: Domain): Promise<void> {
// // Load all digest from collection. // // Load all digest from collection.
// let idx: number | undefined // let idx: number | undefined
@@ -1861,6 +1843,7 @@ export async function restore (
// We do not backup elastic anymore // We do not backup elastic anymore
domains.delete('fulltext-blob' as Domain) domains.delete('fulltext-blob' as Domain)
domains.delete('doc-index-state' as Domain)
let uploadedMb = 0 let uploadedMb = 0
let uploaded = 0 let uploaded = 0
@@ -2000,11 +1983,6 @@ export async function restore (
tx.objectSpace = core.space.Workspace tx.objectSpace = core.space.Workspace
} }
} }
if (opt.cleanIndexState === true) {
if (d._class === core.class.DocIndexState) {
;(d as DocIndexState).needIndex = true
}
}
} }
if (opt.recheck === true) { if (opt.recheck === true) {
@@ -2220,34 +2198,10 @@ export async function restore (
} }
} }
} }
async function performCleanDocIndexState (docsToRemove: Ref<Doc>[]): Promise<void> {
ctx.info('cleanup', { toRemove: docsToRemove.length, workspace: workspaceId, domain: c })
while (docsToRemove.length > 0) {
const part = docsToRemove.splice(0, 1000)
try {
const docs = (await connection.loadDocs(DOMAIN_DOC_INDEX_STATE, part)) as DocIndexState[]
docs.forEach((it) => {
it.needIndex = true
it.removed = true
it.modifiedOn = Date.now()
})
await connection.upload(DOMAIN_DOC_INDEX_STATE, docs)
} catch (err: any) {
ctx.error('failed to clean, will retry', { error: err, workspaceId })
docsToRemove.push(...part)
}
}
}
if (c !== DOMAIN_BLOB) { if (c !== DOMAIN_BLOB) {
// Clean domain documents if not blob // Clean domain documents if not blob
if (docsToRemove.length > 0 && opt.merge !== true) { if (docsToRemove.length > 0 && opt.merge !== true) {
if (c === DOMAIN_DOC_INDEX_STATE) { await performCleanOfDomain(docsToRemove, c)
// We need to create a request to clean fulltext for selected docuemnts.
// We need o clean a FULLTEXT domain as well
await performCleanDocIndexState([...docsToRemove])
} else {
await performCleanOfDomain(docsToRemove, c)
}
} }
} }
} }
+8 -1
View File
@@ -14,7 +14,6 @@
// //
import { import {
type WorkspaceIds,
type Class, type Class,
type Doc, type Doc,
type DocumentQuery, type DocumentQuery,
@@ -28,6 +27,7 @@ import {
type Ref, type Ref,
type Tx, type Tx,
type TxResult, type TxResult,
type WorkspaceIds,
type WorkspaceUuid type WorkspaceUuid
} from '@hcengineering/core' } from '@hcengineering/core'
import { type StorageAdapter } from './storage' import { type StorageAdapter } from './storage'
@@ -61,6 +61,11 @@ export type DbAdapterHandler = (
count: number, count: number,
helper: DomainHelperOperations helper: DomainHelperOperations
) => void ) => void
export interface RawFindIterator {
find: (ctx: MeasureContext) => Promise<Doc[]>
close: () => Promise<void>
}
/** /**
* @public * @public
*/ */
@@ -87,6 +92,8 @@ export interface DbAdapter extends LowLevelStorage {
// Allow to register a handler to listen for domain operations // Allow to register a handler to listen for domain operations
on?: (handler: DbAdapterHandler) => void on?: (handler: DbAdapterHandler) => void
rawFind: (ctx: MeasureContext, domain: Domain) => RawFindIterator
} }
/** /**
+4
View File
@@ -104,6 +104,10 @@ export abstract class BaseMiddleware implements Middleware {
return this.next?.handleBroadcast(ctx) ?? emptyBroadcastResult return this.next?.handleBroadcast(ctx) ?? emptyBroadcastResult
} }
provideBroadcast (ctx: MeasureContext<SessionData>): Promise<void> {
return this.next?.handleBroadcast(ctx) ?? emptyBroadcastResult
}
protected provideTx (ctx: MeasureContext<SessionData>, tx: Tx[]): Promise<TxMiddlewareResult> { protected provideTx (ctx: MeasureContext<SessionData>, tx: Tx[]): Promise<TxMiddlewareResult> {
if (this.next !== undefined) { if (this.next !== undefined) {
return this.next.tx(ctx, tx) return this.next.tx(ctx, tx)
+1
View File
@@ -34,3 +34,4 @@ export * from './nullAdapter'
export * from './service' export * from './service'
export * from './stats' export * from './stats'
export * from './triggers' export * from './triggers'
export * from './queue'
+8 -1
View File
@@ -35,7 +35,7 @@ import core, {
type TxResult, type TxResult,
type WorkspaceIds type WorkspaceIds
} from '@hcengineering/core' } from '@hcengineering/core'
import { type DbAdapter, type DbAdapterHandler, type DomainHelperOperations } from './adapter' import { type DbAdapter, type DbAdapterHandler, type DomainHelperOperations, type RawFindIterator } from './adapter'
/** /**
* @public * @public
*/ */
@@ -62,6 +62,13 @@ export class DummyDbAdapter implements DbAdapter {
return toFindResult([]) return toFindResult([])
} }
rawFind (ctx: MeasureContext, domain: Domain): RawFindIterator {
return {
find: async () => [],
close: async () => {}
}
}
async init (): Promise<void> {} async init (): Promise<void> {}
helper (): DomainHelperOperations { helper (): DomainHelperOperations {
+1 -4
View File
@@ -16,7 +16,7 @@
import { type Metadata, type Plugin, plugin } from '@hcengineering/platform' import { type Metadata, type Plugin, plugin } from '@hcengineering/platform'
import type { Class, Ref, Space, Mixin } from '@hcengineering/core' import type { Class, Mixin, Ref } from '@hcengineering/core'
import type { ObjectDDParticipant, SearchPresenter, Trigger } from './types' import type { ObjectDDParticipant, SearchPresenter, Trigger } from './types'
/** /**
@@ -35,9 +35,6 @@ const serverCore = plugin(serverCoreId, {
ObjectDDParticipant: '' as Ref<ObjectDDParticipant>, ObjectDDParticipant: '' as Ref<ObjectDDParticipant>,
SearchPresenter: '' as Ref<Mixin<SearchPresenter>> SearchPresenter: '' as Ref<Mixin<SearchPresenter>>
}, },
space: {
DocIndexState: '' as Ref<Space>
},
metadata: { metadata: {
FrontUrl: '' as Metadata<string>, FrontUrl: '' as Metadata<string>,
FilesUrl: '' as Metadata<string>, FilesUrl: '' as Metadata<string>,
+65
View File
@@ -0,0 +1,65 @@
import type { MeasureContext, WorkspaceUuid } from '@hcengineering/core'
import { type ConsumerHandle, type PlatformQueue, type PlatformQueueProducer, type QueueTopic } from './types'
/**
* A dummy implementation of PlatformQueueProducer for testing and development
*/
class DummyQueueProducer<T> implements PlatformQueueProducer<T> {
async send (id: WorkspaceUuid | string, msgs: T[]): Promise<void> {
await Promise.resolve()
}
async close (): Promise<void> {
await Promise.resolve()
}
}
/**
* A dummy implementation of PlatformQueue for testing and development
*/
export class DummyQueue implements PlatformQueue {
createProducer<T>(ctx: MeasureContext, topic: QueueTopic): PlatformQueueProducer<T> {
return new DummyQueueProducer<T>()
}
getClientId (): string {
return 'dummy'
}
createConsumer<T>(
ctx: MeasureContext,
topic: QueueTopic,
groupId: string,
onMessage: (
msg: { id: WorkspaceUuid | string, value: T }[],
queue: {
pause: () => void
heartbeat: () => Promise<void>
}
) => Promise<void>,
options?: {
bulkSize?: number
fromBegining?: boolean
}
): ConsumerHandle {
return {
shutdown: async (): Promise<void> => {
await Promise.resolve()
},
isConnected: (): boolean => {
return false
}
}
}
async createTopics (tx: number): Promise<void> {
await Promise.resolve()
}
}
/**
* Creates a new instance of the dummy queue implementation
*/
export function createDummyQueue (): PlatformQueue {
return new DummyQueue()
}
+4
View File
@@ -0,0 +1,4 @@
export * from './types'
export * from './workspace'
export * from './dummyQueue'
export * from './users'
+59
View File
@@ -0,0 +1,59 @@
import type { MeasureContext, WorkspaceUuid } from '@hcengineering/core'
export enum QueueTopic {
// Topic with partitions to split workspace transactions into
Tx = 'tx',
// Topic to send workspace information into
Workspace = 'workspace',
// A fulltext topic to hold extra fulltext request, like re-indexing requests etc, private to fulltext service.
Fulltext = 'fulltext',
// A topic about user activity.
Users = 'users'
}
export interface ConsumerHandle {
shutdown: () => Promise<void>
isConnected: () => boolean
}
export interface ConsumerMessage<T> {
id: WorkspaceUuid | string
value: T[]
}
export interface ConsumerControl {
pause: () => void
heartbeat: () => Promise<void>
}
export interface PlatformQueue {
createProducer: <T>(ctx: MeasureContext, topic: QueueTopic) => PlatformQueueProducer<T>
/**
* Create a consumer for a topic.
* return a function to close the reciever.
*/
createConsumer: <T>(
ctx: MeasureContext,
topic: QueueTopic,
groupId: string,
onMessage: (msg: ConsumerMessage<T>[], queue: ConsumerControl) => Promise<void>,
options?: {
fromBegining?: boolean
}
) => ConsumerHandle
createTopics: (tx: number) => Promise<void>
getClientId: () => string
}
/**
* Create a producer for a topic.
*/
export interface PlatformQueueProducer<T> {
send: (id: WorkspaceUuid | string, msgs: T[]) => Promise<void>
close: () => Promise<void>
}
+32
View File
@@ -0,0 +1,32 @@
import type { AccountUuid, PersonId } from '@hcengineering/core'
export enum QueueUserEvent {
login = 'login',
logout = 'logout'
}
export interface QueueUserMessage {
type: QueueUserEvent
user: AccountUuid
}
export interface QueueUserLogin extends QueueUserMessage {
type: QueueUserEvent.login | QueueUserEvent.logout
sessions: number
socialIds: PersonId[]
}
export const userEvents = {
login: function userLogin (data: Omit<QueueUserLogin, 'type'>): QueueUserLogin {
return {
type: QueueUserEvent.login,
...data
}
},
logout: function userLogout (data: Omit<QueueUserLogin, 'type'>): QueueUserLogin {
return {
type: QueueUserEvent.logout,
...data
}
}
}
+46
View File
@@ -0,0 +1,46 @@
import type { Class, Doc, Domain, Ref } from '@hcengineering/core'
export enum QueueWorkspaceEvent {
Up = 'up',
Down = 'down',
Created = 'created',
CreateFailed = 'create-failed',
Upgraded = 'upgraded',
Upgradefailed = 'upgrade-failed',
Deleted = 'deleted',
Archived = 'archived',
Restored = 'restored',
FullReindex = 'full-fulltext-reindex',
Reindex = 'fulltext-reindex',
ClearIndex = 'clear-fulltext-index'
}
export interface QueueWorkspaceMessage {
type: QueueWorkspaceEvent
}
export interface QueueWorkspaceReindexMessage extends QueueWorkspaceMessage {
type: QueueWorkspaceEvent.Reindex
domain: Domain
classes: Ref<Class<Doc>>[]
}
export const workspaceEvents = {
open: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Up }),
down: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Down }),
created: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Created }),
upgraded: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Upgraded }),
upgradeFailed: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Upgradefailed }),
createFailed: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.CreateFailed }),
deleted: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Deleted }),
archived: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Archived }),
restored: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.Restored }),
fullReindex: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.FullReindex }),
clearIndex: (): QueueWorkspaceMessage => ({ type: QueueWorkspaceEvent.ClearIndex }),
reindex: (domain: Domain, classes: Ref<Class<Doc>>[]): QueueWorkspaceReindexMessage => ({
type: QueueWorkspaceEvent.Reindex,
domain,
classes
})
}
File diff suppressed because it is too large Load Diff
-11
View File
@@ -16,8 +16,6 @@
import { import {
type Class, type Class,
type Doc, type Doc,
type DocIndexState,
type DocumentUpdate,
type FullTextSearchContext, type FullTextSearchContext,
type Hierarchy, type Hierarchy,
type ModelDb, type ModelDb,
@@ -34,12 +32,3 @@ export interface FullTextPipeline {
cancelling: boolean cancelling: boolean
} }
/**
* @public
*/
export type DocUpdateHandler = (doc: DocIndexState, update: DocumentUpdate<DocIndexState>) => Promise<void>
export const indexes = 'verify-indexes-v2'
export const fullReindex = 'full-text-indexer-v5'
export const docStructure = 'full-text-structure-v6'
+2 -24
View File
@@ -13,20 +13,17 @@
// limitations under the License. // limitations under the License.
// //
import core, { import {
type AnyAttribute, type AnyAttribute,
type Class, type Class,
type Data,
type Doc, type Doc,
type DocIndexState,
type FullTextSearchContext, type FullTextSearchContext,
getFullTextContext, getFullTextContext,
type Hierarchy, type Hierarchy,
type Obj,
type Ref, type Ref,
type Space type Space
} from '@hcengineering/core' } from '@hcengineering/core'
import plugin, { type IndexedDoc } from '@hcengineering/server-core' import { type IndexedDoc } from '@hcengineering/server-core'
import { type FullTextPipeline } from './types' import { type FullTextPipeline } from './types'
export { docKey, isFullTextAttribute } from '@hcengineering/core' export { docKey, isFullTextAttribute } from '@hcengineering/core'
@@ -55,25 +52,6 @@ export function getContent (
return attrs return attrs
} }
/**
* @public
*/
export function createStateDoc (
id: Ref<Doc>,
objectClass: Ref<Class<Obj>>,
data: Omit<Data<DocIndexState>, 'objectClass'> & { space?: Ref<Space> }
): DocIndexState {
return {
_class: core.class.DocIndexState,
_id: id as Ref<DocIndexState>,
space: data.space ?? plugin.space.DocIndexState,
objectClass,
modifiedBy: core.account.System,
modifiedOn: Date.now(),
...data
}
}
/** /**
* @public * @public
*/ */
@@ -1,5 +1,5 @@
module.exports = { module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/model/eslint.config.json'], extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
parserOptions: { parserOptions: {
tsconfigRootDir: __dirname, tsconfigRootDir: __dirname,
project: './tsconfig.json' project: './tsconfig.json'
@@ -1,5 +1,5 @@
{ {
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig", "rigPackageName": "@hcengineering/platform-rig",
"rigProfile": "model" "rigProfile": "node"
} }
@@ -1,23 +1,19 @@
{ {
"name": "@hcengineering/server-fulltext-resources", "name": "@hcengineering/kafka",
"version": "0.6.0", "version": "0.6.0",
"main": "lib/index.js", "main": "lib/index.js",
"svelte": "src/index.ts", "svelte": "src/index.ts",
"types": "types/index.d.ts", "types": "types/index.d.ts",
"files": [
"lib/**/*",
"types/**/*",
"tsconfig.json"
],
"author": "Anticrm Platform Contributors", "author": "Anticrm Platform Contributors",
"template": "@hcengineering/node-package",
"license": "EPL-2.0", "license": "EPL-2.0",
"scripts": { "scripts": {
"build": "compile", "build": "compile",
"build:watch": "compile", "build:watch": "compile",
"test": "jest --passWithNoTests --silent --forceExit",
"format": "format src", "format": "format src",
"test": "jest --passWithNoTests --silent",
"_phase:build": "compile transpile src", "_phase:build": "compile transpile src",
"_phase:test": "jest --passWithNoTests --silent", "_phase:test": "jest --passWithNoTests --silent --forceExit",
"_phase:format": "format src", "_phase:format": "format src",
"_phase:validate": "compile validate" "_phase:validate": "compile validate"
}, },
@@ -32,6 +28,7 @@
"eslint-config-standard-with-typescript": "^40.0.0", "eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0", "prettier": "^3.1.0",
"typescript": "^5.3.3", "typescript": "^5.3.3",
"@types/node": "~20.11.16",
"jest": "^29.7.0", "jest": "^29.7.0",
"ts-jest": "^29.1.1", "ts-jest": "^29.1.1",
"@types/jest": "^29.5.5" "@types/jest": "^29.5.5"
@@ -39,6 +36,8 @@
"dependencies": { "dependencies": {
"@hcengineering/core": "^0.6.32", "@hcengineering/core": "^0.6.32",
"@hcengineering/platform": "^0.6.11", "@hcengineering/platform": "^0.6.11",
"@hcengineering/server-core": "^0.6.1" "@hcengineering/server-core": "^0.6.1",
"@hcengineering/storage": "^0.6.0",
"kafkajs": "^2.2.4"
} }
} }
+262
View File
@@ -0,0 +1,262 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type MeasureContext, type WorkspaceUuid } from '@hcengineering/core'
import {
QueueTopic,
type ConsumerControl,
type ConsumerHandle,
type ConsumerMessage,
type PlatformQueue,
type PlatformQueueProducer
} from '@hcengineering/server-core'
import { Kafka, Partitioners, type Consumer, type Producer } from 'kafkajs'
import type * as tls from 'tls'
export interface QueueConfig {
postfix: string // a topic prefix to be used do distinguish between staging and production topics in same broker
brokers: string[]
clientId: string
region: string
ssl?: tls.ConnectionOptions | boolean
}
/**
* Query config parser in the form of 'brokers;clientId'
* brokers are comma separated
*/
export function parseQueueConfig (config: string, serviceId: string, region: string): QueueConfig {
const [brokers, postfix] = config.split(';')
return {
postfix: postfix ?? '', // Empty by default
brokers: brokers.split(','),
clientId: serviceId,
region
}
}
function getKafkaTopicId (topic: QueueTopic, config: QueueConfig): string {
if (config.region !== '') {
return `${config.region}.${topic}${config.postfix ?? ''}`
}
return `${topic}${config.postfix ?? ''}`
}
class PlatformQueueImpl implements PlatformQueue {
constructor (
private readonly kafka: Kafka,
readonly config: QueueConfig
) {}
getClientId (): string {
return this.config.clientId
}
createProducer<T>(ctx: MeasureContext, topic: QueueTopic): PlatformQueueProducer<T> {
return new PlatformQueueProducerImpl(ctx, this.kafka, getKafkaTopicId(topic, this.config))
}
createConsumer<T>(
ctx: MeasureContext,
topic: QueueTopic,
groupId: string,
onMessage: (msg: ConsumerMessage<T>[], queue: ConsumerControl) => Promise<void>,
options?: {
fromBegining?: boolean
}
): ConsumerHandle {
return new PlatformQueueConsumerImpl(ctx, this.kafka, this.config, topic, groupId, onMessage, options)
}
async createTopics (tx: number): Promise<void> {
const topics = new Set(await this.kafka.admin({}).listTopics())
if (!topics.has(QueueTopic.Tx)) {
await this.kafka.admin().createTopics({
topics: [
{
topic: getKafkaTopicId(QueueTopic.Tx, this.config),
numPartitions: tx
}
]
})
}
}
}
class PlatformQueueProducerImpl implements PlatformQueueProducer<any> {
txProducer: Producer
connected: Promise<void> | undefined
constructor (
readonly ctx: MeasureContext,
kafka: Kafka,
private readonly topic: string
) {
this.txProducer = kafka.producer({
allowAutoTopicCreation: true,
createPartitioner: Partitioners.DefaultPartitioner
})
this.connected = this.ctx.with('connect-broker', {}, () => this.txProducer.connect())
}
async send (id: WorkspaceUuid | string, msgs: any[]): Promise<void> {
if (this.connected !== undefined) {
await this.connected
this.connected = undefined
}
await this.ctx.with('send', { topic: this.topic }, () =>
this.txProducer.send({
topic: this.topic,
messages: msgs.map((m) => ({
key: Buffer.from(`${id}`),
value: Buffer.from(JSON.stringify(m))
}))
})
)
}
async close (): Promise<void> {
await this.ctx.with('disconnect', {}, () => this.txProducer.disconnect())
}
}
class PlatformQueueConsumerImpl implements ConsumerHandle {
connected = false
cc: Consumer
constructor (
readonly ctx: MeasureContext,
readonly kafka: Kafka,
readonly config: QueueConfig,
private readonly topic: QueueTopic,
groupId: string,
private readonly onMessage: (msg: ConsumerMessage<any>[], queue: ConsumerControl) => Promise<void>,
private readonly options?: {
fromBegining?: boolean
}
) {
this.cc = this.kafka.consumer({ groupId: `${getKafkaTopicId(this.topic, this.config)}-${groupId}` })
void this.start().catch((err) => {
ctx.error('failed to consume', { err })
})
}
async start (): Promise<void> {
await this.doConnect()
await this.doSubscribe()
await this.cc.run({
eachMessage: async ({ topic, message, pause, heartbeat }) => {
await this.onMessage(
[
{
id: message.key?.toString() ?? '',
value: [JSON.parse(message.value?.toString() ?? '{}')]
}
],
{
heartbeat,
pause
}
)
}
// , // TODO: Finish testinf
// eachBatch: async ({ batch, pause, heartbeat, resolveOffset }) => {
// const queueInfo = {
// pause,
// heartbeat
// }
// const batchMessages = batch.messages
// const currentMsg: ConsumerMessage<any> = {
// id: '',
// value: []
// }
// let lastOffset: string = ' '
// const sendLast = async (): Promise<void> => {
// await this.onMessage([currentMsg], queueInfo)
// // Mark last offset as cusomed
// resolveOffset(lastOffset)
// await heartbeat()
// }
// for (const v of batchMessages) {
// const id = v.key?.toString() ?? ''
// if (currentMsg.id !== id && currentMsg.value.length > 0) {
// await sendLast() // Send last message
// currentMsg.id = id
// currentMsg.value = []
// }
// lastOffset = v.offset
// currentMsg.value.push(JSON.parse(v.value?.toString() ?? '{}'))
// }
// await sendLast()
// }
})
}
async doConnect (): Promise<void> {
this.cc.on('consumer.connect', () => {
this.connected = true
this.ctx.info('consumer connected to queue')
})
this.cc.on('consumer.disconnect', () => {
this.connected = false
this.ctx.warn('consumer disconnected from queue')
})
await this.cc.connect()
}
async doSubscribe (): Promise<void> {
await this.cc.subscribe({
topic: getKafkaTopicId(this.topic, this.config),
fromBeginning: this.options?.fromBegining
})
}
isConnected (): boolean {
return this.connected
}
shutdown (): Promise<void> {
return this.cc.disconnect()
}
}
/**
* Constructs a platform queue.
*/
export function getPlatformQueue (serviceId: string, region?: string): PlatformQueue {
const queueConfig = process.env.QUEUE_CONFIG ?? 'huly.local:9092'
if (queueConfig === undefined) {
throw new Error('Please provide queue config')
}
const config = parseQueueConfig(queueConfig, serviceId, region ?? process.env.REGION ?? '')
return createPlatformQueue(config)
}
export function createPlatformQueue (config: QueueConfig): PlatformQueue {
console.info({ message: 'Using queue', config })
return new PlatformQueueImpl(
new Kafka({
clientId: config.clientId,
brokers: config.brokers,
ssl: config.ssl
}),
config
)
}
@@ -1,5 +1,5 @@
{ {
"extends": "./node_modules/@hcengineering/platform-rig/profiles/model/tsconfig.json", "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json",
"compilerOptions": { "compilerOptions": {
"rootDir": "./src", "rootDir": "./src",
+2 -61
View File
@@ -16,7 +16,6 @@
import { Analytics } from '@hcengineering/analytics' import { Analytics } from '@hcengineering/analytics'
import core, { import core, {
docKey, docKey,
DOMAIN_DOC_INDEX_STATE,
isFullTextAttribute, isFullTextAttribute,
isIndexedAttribute, isIndexedAttribute,
toFindResult, toFindResult,
@@ -33,17 +32,9 @@ import core, {
type Ref, type Ref,
type SearchOptions, type SearchOptions,
type SearchQuery, type SearchQuery,
type SearchResult, type SearchResult
type SessionData,
type Tx
} from '@hcengineering/core' } from '@hcengineering/core'
import type { import type { IndexedDoc, Middleware, MiddlewareCreator, PipelineContext } from '@hcengineering/server-core'
IndexedDoc,
Middleware,
MiddlewareCreator,
PipelineContext,
TxMiddlewareResult
} from '@hcengineering/server-core'
import { BaseMiddleware } from '@hcengineering/server-core' import { BaseMiddleware } from '@hcengineering/server-core'
/** /**
* @public * @public
@@ -52,8 +43,6 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware {
fulltextEndpoint: string fulltextEndpoint: string
contexts = new Map<Ref<Class<Doc>>, FullTextSearchContext>() contexts = new Map<Ref<Class<Doc>>, FullTextSearchContext>()
warmupTriggered: boolean = false
constructor ( constructor (
context: PipelineContext, context: PipelineContext,
next: Middleware | undefined, next: Middleware | undefined,
@@ -79,54 +68,10 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware {
} }
} }
async handleBroadcast (ctx: MeasureContext<SessionData>): Promise<void> {
if (ctx.contextData.fulltextUpdates !== undefined && ctx.contextData.fulltextUpdates.size > 0) {
const toUpdate = Array.from(ctx.contextData.fulltextUpdates.values())
ctx.contextData.fulltextUpdates.clear()
await this.context.lowLevelStorage?.upload(ctx, DOMAIN_DOC_INDEX_STATE, toUpdate)
this.sendWarmup()
}
await super.handleBroadcast(ctx)
}
async init (ctx: MeasureContext): Promise<void> { async init (ctx: MeasureContext): Promise<void> {
this.contexts = new Map( this.contexts = new Map(
this.context.modelDb.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it]) this.context.modelDb.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it])
) )
// Send one warumup so reindex will
this.sendWarmup()
}
sendWarmup (): void {
if (!this.warmupTriggered) {
this.warmupTriggered = true
setTimeout(() => {
this.warmupTriggered = false
void this._sendWarmup().catch(() => {})
}, 25)
}
}
async _sendWarmup (): Promise<void> {
// Just send a warmup to indexer to start do something, or just reindex if some work are pending.
try {
await fetch(this.fulltextEndpoint + '/api/v1/warmup', {
method: 'PUT',
keepalive: true,
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
token: this.token
})
})
} catch (err: any) {
if (err?.cause?.code === 'ECONNRESET' || err?.cause?.code === 'ECONNREFUSED') {
return
}
Analytics.handleError(err)
}
} }
async search<T extends Doc>( async search<T extends Doc>(
@@ -385,10 +330,6 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware {
} }
} }
tx (ctx: MeasureContext, txes: Tx[]): Promise<TxMiddlewareResult> {
return this.provideTx(ctx, txes)
}
async close (): Promise<void> { async close (): Promise<void> {
try { try {
await fetch(this.fulltextEndpoint + '/api/v1/close', { await fetch(this.fulltextEndpoint + '/api/v1/close', {
+1
View File
@@ -35,3 +35,4 @@ export * from './spacePermissions'
export * from './spaceSecurity' export * from './spaceSecurity'
export * from './triggers' export * from './triggers'
export * from './txPush' export * from './txPush'
export * from './queue'
+11 -7
View File
@@ -19,6 +19,7 @@ import core, {
type DocumentQuery, type DocumentQuery,
type FindOptions, type FindOptions,
type FindResult, type FindResult,
type Hierarchy,
type LoadModelResponse, type LoadModelResponse,
type MeasureContext, type MeasureContext,
type Ref, type Ref,
@@ -59,7 +60,8 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware {
constructor ( constructor (
context: PipelineContext, context: PipelineContext,
next: Middleware | undefined, next: Middleware | undefined,
readonly systemTx: Tx[] readonly systemTx: Tx[],
readonly filter?: (h: Hierarchy, model: Tx[]) => Tx[]
) { ) {
super(context, next) super(context, next)
} }
@@ -69,16 +71,17 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware {
ctx: MeasureContext, ctx: MeasureContext,
context: PipelineContext, context: PipelineContext,
next: Middleware | undefined, next: Middleware | undefined,
systemTx: Tx[] systemTx: Tx[],
filter?: (h: Hierarchy, model: Tx[]) => Tx[]
): Promise<Middleware> { ): Promise<Middleware> {
const middleware = new ModelMiddleware(context, next, systemTx) const middleware = new ModelMiddleware(context, next, systemTx, filter)
await middleware.init(ctx) await middleware.init(ctx)
return middleware return middleware
} }
static create (tx: Tx[]): MiddlewareCreator { static create (tx: Tx[], filter?: (h: Hierarchy, model: Tx[]) => Tx[]): MiddlewareCreator {
return (ctx, context, next) => { return (ctx, context, next) => {
return this.doCreate(ctx, context, next, tx) return this.doCreate(ctx, context, next, tx, filter)
} }
} }
@@ -116,9 +119,10 @@ export class ModelMiddleware extends BaseMiddleware implements Middleware {
ctx.warn('failed to apply model transaction, skipping', { tx: JSON.stringify(tx), err }) ctx.warn('failed to apply model transaction, skipping', { tx: JSON.stringify(tx), err })
} }
} }
this.context.modelDb.addTxes(ctx, model, true) const fmodel = this.filter !== undefined ? this.filter(this.context.hierarchy, model) : model
this.context.modelDb.addTxes(ctx, fmodel, true)
this.setModel(model) this.setModel(fmodel)
} }
private addModelTx (tx: Tx): void { private addModelTx (tx: Tx): void {
+56
View File
@@ -0,0 +1,56 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type MeasureContext, type SessionData, type Tx } from '@hcengineering/core'
import {
BaseMiddleware,
type Middleware,
type MiddlewareCreator,
type PipelineContext,
type PlatformQueue,
type PlatformQueueProducer,
QueueTopic
} from '@hcengineering/server-core'
export class QueueMiddleware extends BaseMiddleware {
txProducer: PlatformQueueProducer<Tx>
connected: Promise<void> | undefined
constructor (
ctx: MeasureContext,
readonly context: PipelineContext,
protected readonly next: Middleware | undefined,
readonly queue: PlatformQueue
) {
super(context, next)
this.txProducer = queue.createProducer<Tx>(ctx, QueueTopic.Tx)
}
static create (queue: PlatformQueue): MiddlewareCreator {
return async (ctx, context, next) => {
return new QueueMiddleware(ctx, context, next, queue)
}
}
async handleBroadcast (ctx: MeasureContext<SessionData>): Promise<void> {
if (this.connected !== undefined) {
await this.connected
this.connected = undefined
}
await Promise.all([
this.provideBroadcast(ctx),
this.txProducer.send(this.context.workspace.uuid, ctx.contextData.broadcast.txes)
])
}
}
+1 -1
View File
@@ -174,7 +174,7 @@ export class SpacePermissionsMiddleware extends BaseMiddleware implements Middle
if (!this.context.hierarchy.isDerived(createTx.objectClass, core.class.Space)) return if (!this.context.hierarchy.isDerived(createTx.objectClass, core.class.Space)) return
if (this.isTypedSpaceClass(createTx.objectClass)) { if (this.isTypedSpaceClass(createTx.objectClass)) {
const res = TxProcessor.buildDoc2Doc<TypedSpace>([createTx]) const res = TxProcessor.buildDoc2Doc<TypedSpace>([createTx])
if (res !== undefined) { if (res != null) {
this.addRestrictedSpace(res) this.addRestrictedSpace(res)
} }
} else { } else {
-9
View File
@@ -21,7 +21,6 @@ import core, {
ClassifierKind, ClassifierKind,
type Data, type Data,
type Doc, type Doc,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MODEL, DOMAIN_MODEL,
DOMAIN_RELATION, DOMAIN_RELATION,
DOMAIN_TX, DOMAIN_TX,
@@ -136,14 +135,6 @@ export function genMinModel (): TxCUD<Doc>[] {
domain: DOMAIN_MODEL domain: DOMAIN_MODEL
}) })
) )
txes.push(
createClass(core.class.DocIndexState, {
label: 'DocIndexState' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.CLASS,
domain: DOMAIN_DOC_INDEX_STATE
})
)
txes.push( txes.push(
createClass(core.class.Tx, { createClass(core.class.Tx, {
+32 -4
View File
@@ -26,10 +26,10 @@ import core, {
groupByArray, groupByArray,
isOperator, isOperator,
matchQuery, matchQuery,
platformNow,
toFindResult, toFindResult,
withContext, withContext,
type AssociationQuery, type AssociationQuery,
type WorkspaceIds,
type Class, type Class,
type Doc, type Doc,
type DocInfo, type DocInfo,
@@ -62,16 +62,17 @@ import core, {
type TxResult, type TxResult,
type TxUpdateDoc, type TxUpdateDoc,
type WithLookup, type WithLookup,
platformNow type WorkspaceIds
} from '@hcengineering/core' } from '@hcengineering/core'
import { import {
calcHashHash,
type DbAdapter, type DbAdapter,
type DbAdapterHandler, type DbAdapterHandler,
type DomainHelperOperations, type DomainHelperOperations,
type RawFindIterator,
type ServerFindOptions, type ServerFindOptions,
type StorageAdapter, type StorageAdapter,
type TxAdapter, type TxAdapter
calcHashHash
} from '@hcengineering/server-core' } from '@hcengineering/server-core'
import { import {
type AbstractCursor, type AbstractCursor,
@@ -1145,6 +1146,33 @@ abstract class MongoAdapterBase implements DbAdapter {
} }
} }
rawFind (_ctx: MeasureContext, domain: Domain): RawFindIterator {
const ctx = _ctx.newChild('findRaw', { domain })
const coll = this.db.collection<Doc>(domain)
let iterator: FindCursor<Doc>
return {
find: async () => {
if (iterator === undefined) {
iterator = coll.find({})
}
const d = await ctx.with('next', {}, () => iterator.next())
const result: Doc[] = []
if (d != null) {
result.push(this.stripHash(d) as Doc)
}
if (iterator.bufferedCount() > 0) {
result.push(...(this.stripHash(iterator.readBufferedDocuments()) as Doc[]))
}
return result ?? []
},
close: async () => {
await ctx.with('close', {}, () => iterator.close())
ctx.end()
}
}
}
load (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> { load (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
return ctx.with('load', { domain }, async () => { return ctx.with('load', { domain }, async () => {
if (docs.length === 0) { if (docs.length === 0) {
@@ -1,11 +0,0 @@
ALTER TABLE doc_index_state
ADD "needIndex" bool;
ALTER TABLE doc_index_state
DROP COLUMN "attachedTo";
UPDATE doc_index_state
SET "needIndex" = (data->>'needIndex')::boolean;
ALTER TABLE doc_index_state
ALTER COLUMN "needIndex" SET NOT NULL;
@@ -22,7 +22,6 @@ import core, {
type Data, type Data,
type Doc, type Doc,
type Domain, type Domain,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_MODEL, DOMAIN_MODEL,
DOMAIN_RELATION, DOMAIN_RELATION,
DOMAIN_TX, DOMAIN_TX,
@@ -155,14 +154,6 @@ export function genMinModel (): TxCUD<Doc>[] {
domain: DOMAIN_MODEL domain: DOMAIN_MODEL
}) })
) )
txes.push(
createClass(core.class.DocIndexState, {
label: 'DocIndexState' as IntlString,
extends: core.class.Doc,
kind: ClassifierKind.CLASS,
domain: DOMAIN_DOC_INDEX_STATE
})
)
txes.push( txes.push(
createClass(core.class.Tx, { createClass(core.class.Tx, {
+1 -16
View File
@@ -1,4 +1,4 @@
import { DOMAIN_DOC_INDEX_STATE, DOMAIN_MODEL_TX, DOMAIN_RELATION, DOMAIN_SPACE, DOMAIN_TX } from '@hcengineering/core' import { DOMAIN_MODEL_TX, DOMAIN_RELATION, DOMAIN_SPACE, DOMAIN_TX } from '@hcengineering/core'
export type DataType = 'bigint' | 'bool' | 'text' | 'text[]' export type DataType = 'bigint' | 'bool' | 'text' | 'text[]'
@@ -170,20 +170,6 @@ const userNotificationSchema: Schema = {
} }
} }
const docIndexStateSchema: Schema = {
...baseSchema,
needIndex: {
type: 'bool',
notNull: true,
index: true
},
objectClass: {
type: 'text',
notNull: true,
index: false
}
}
const timeSchema: Schema = { const timeSchema: Schema = {
...baseSchema, ...baseSchema,
workslots: { workslots: {
@@ -305,7 +291,6 @@ export const domainSchemas: Record<string, Schema> = {
[translateDomain('time')]: timeSchema, [translateDomain('time')]: timeSchema,
[translateDomain('calendar')]: calendarSchema, [translateDomain('calendar')]: calendarSchema,
[translateDomain('event')]: eventSchema, [translateDomain('event')]: eventSchema,
[translateDomain(DOMAIN_DOC_INDEX_STATE)]: docIndexStateSchema,
notification: notificationSchema, notification: notificationSchema,
[translateDomain('notification-dnc')]: dncSchema, [translateDomain('notification-dnc')]: dncSchema,
[translateDomain('notification-user')]: userNotificationSchema, [translateDomain('notification-user')]: userNotificationSchema,
+49
View File
@@ -66,6 +66,7 @@ import {
type DbAdapter, type DbAdapter,
type DbAdapterHandler, type DbAdapterHandler,
type DomainHelperOperations, type DomainHelperOperations,
type RawFindIterator,
type ServerFindOptions, type ServerFindOptions,
type TxAdapter type TxAdapter
} from '@hcengineering/server-core' } from '@hcengineering/server-core'
@@ -1615,6 +1616,54 @@ abstract class PostgresAdapterBase implements DbAdapter {
} }
} }
rawFind (_ctx: MeasureContext, domain: Domain): RawFindIterator {
const ctx = _ctx.newChild('findRaw', { domain })
let initialized: boolean = false
let client: DBClient
const tdomain = translateDomain(domain)
const schema = getSchema(domain)
const workspaceId = this.workspaceId
function createBulk (projection: string, limit = 1000): AsyncGenerator<Doc[]> {
const sql = `
SELECT ${projection}
FROM ${tdomain}
WHERE "workspaceId" = '${workspaceId}'
`
return createCursorGenerator(client.raw(), sql, undefined, schema, limit)
}
let bulk: AsyncGenerator<Doc[]>
return {
find: async () => {
if (!initialized) {
if (client === undefined) {
client = await this.client.reserve()
}
initialized = true
bulk = createBulk('*')
}
const docs = await ctx.with('next', {}, () => bulk.next())
if (docs.done === true || docs.value.length === 0) {
return []
}
const result: Doc[] = []
result.push(...(this.stripHash(docs.value) as Doc[]))
return result
},
close: async () => {
await bulk.return([]) // We need to close generator, just in case
client?.release()
ctx.end()
}
}
}
load (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> { load (ctx: MeasureContext, domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
return ctx.with('load', { domain }, async () => { return ctx.with('load', { domain }, async () => {
if (docs.length === 0) { if (docs.length === 0) {
+2 -3
View File
@@ -168,11 +168,10 @@
"@hcengineering/server-ai-bot-resources": "^0.6.0", "@hcengineering/server-ai-bot-resources": "^0.6.0",
"@hcengineering/time": "^0.6.0", "@hcengineering/time": "^0.6.0",
"@hcengineering/time-assets": "^0.6.0", "@hcengineering/time-assets": "^0.6.0",
"@hcengineering/server-fulltext": "^0.6.0",
"@hcengineering/server-fulltext-resources": "^0.6.0",
"@hcengineering/survey": "^0.6.0", "@hcengineering/survey": "^0.6.0",
"@hcengineering/survey-assets": "^0.6.0", "@hcengineering/survey-assets": "^0.6.0",
"@hcengineering/server-process": "^0.6.0", "@hcengineering/server-process": "^0.6.0",
"@hcengineering/server-process-resources": "^0.6.0" "@hcengineering/server-process-resources": "^0.6.0",
"@hcengineering/kafka": "^0.6.0"
} }
} }
@@ -39,6 +39,7 @@ import { PlatformError, unknownError } from '@hcengineering/platform'
import { import {
type DbAdapter, type DbAdapter,
type DbAdapterHandler, type DbAdapterHandler,
type RawFindIterator,
type StorageAdapter, type StorageAdapter,
type StorageAdapterEx type StorageAdapterEx
} from '@hcengineering/server-core' } from '@hcengineering/server-core'
@@ -78,6 +79,13 @@ class StorageBlobAdapter implements DbAdapter {
return [] return []
} }
rawFind (ctx: MeasureContext, domain: Domain): RawFindIterator {
return {
find: async () => [],
close: async () => {}
}
}
async rawUpdate<T extends Doc>( async rawUpdate<T extends Doc>(
domain: Domain, domain: Domain,
query: DocumentQuery<T>, query: DocumentQuery<T>,
+10 -2
View File
@@ -32,6 +32,7 @@ import {
NotificationsMiddleware, NotificationsMiddleware,
PrivateMiddleware, PrivateMiddleware,
QueryJoinMiddleware, QueryJoinMiddleware,
QueueMiddleware,
SpacePermissionsMiddleware, SpacePermissionsMiddleware,
SpaceSecurityMiddleware, SpaceSecurityMiddleware,
TriggersMiddleware, TriggersMiddleware,
@@ -50,10 +51,12 @@ import {
type PipelineContext, type PipelineContext,
type PipelineFactory, type PipelineFactory,
type StorageAdapter, type StorageAdapter,
type WorkspaceDestroyAdapter type WorkspaceDestroyAdapter,
type PlatformQueue
} from '@hcengineering/server-core' } from '@hcengineering/server-core'
import { generateToken } from '@hcengineering/server-token' import { generateToken } from '@hcengineering/server-token'
import { createStorageDataAdapter } from './blobStorage' import { createStorageDataAdapter } from './blobStorage'
/** /**
* @public * @public
*/ */
@@ -99,6 +102,8 @@ export function createServerPipeline (
externalStorage: StorageAdapter externalStorage: StorageAdapter
queue?: PlatformQueue
extraLogging?: boolean // If passed, will log every request/etc. extraLogging?: boolean // If passed, will log every request/etc.
pipelineContextVars?: Record<string, any> pipelineContextVars?: Record<string, any>
}, },
@@ -131,6 +136,7 @@ export function createServerPipeline (
LiveQueryMiddleware.create, LiveQueryMiddleware.create,
DomainFindMiddleware.create, DomainFindMiddleware.create,
DomainTxMiddleware.create, DomainTxMiddleware.create,
...(opt.queue !== undefined ? [QueueMiddleware.create(opt.queue)] : []),
DBAdapterInitMiddleware.create, DBAdapterInitMiddleware.create,
ModelMiddleware.create(model), ModelMiddleware.create(model),
DBAdapterMiddleware.create(conf), // Configure DB adapters DBAdapterMiddleware.create(conf), // Configure DB adapters
@@ -205,6 +211,7 @@ export async function getServerPipeline (
wsUrl: WorkspaceIds, wsUrl: WorkspaceIds,
storageAdapter: StorageAdapter, storageAdapter: StorageAdapter,
opt?: { opt?: {
queue?: PlatformQueue
disableTriggers?: boolean disableTriggers?: boolean
} }
): Promise<Pipeline> { ): Promise<Pipeline> {
@@ -212,7 +219,8 @@ export async function getServerPipeline (
externalStorage: storageAdapter, externalStorage: storageAdapter,
usePassedCtx: true, usePassedCtx: true,
disableTriggers: opt?.disableTriggers ?? false, disableTriggers: opt?.disableTriggers ?? false,
adapterSecurity: isAdapterSecurity(dbUrl) adapterSecurity: isAdapterSecurity(dbUrl),
queue: opt?.queue
}) })
return await pipelineFactory(ctx, wsUrl, true, () => {}, null) return await pipelineFactory(ctx, wsUrl, true, () => {}, null)
@@ -28,7 +28,6 @@ import { serverTrackerId } from '@hcengineering/server-tracker'
import { serverTrainingId } from '@hcengineering/server-training' import { serverTrainingId } from '@hcengineering/server-training'
import { serverViewId } from '@hcengineering/server-view' import { serverViewId } from '@hcengineering/server-view'
import { serverAiBotId } from '@hcengineering/server-ai-bot' import { serverAiBotId } from '@hcengineering/server-ai-bot'
import { serverFulltextId } from '@hcengineering/server-fulltext'
import { serverProcessId } from '@hcengineering/server-process' import { serverProcessId } from '@hcengineering/server-process'
export function registerServerPlugins (): void { export function registerServerPlugins (): void {
@@ -61,6 +60,5 @@ export function registerServerPlugins (): void {
addLocation(serverTrainingId, () => import('@hcengineering/server-training-resources')) addLocation(serverTrainingId, () => import('@hcengineering/server-training-resources'))
addLocation(serverGithubId, () => import('@hcengineering/server-github-resources')) addLocation(serverGithubId, () => import('@hcengineering/server-github-resources'))
addLocation(serverAiBotId, () => import('@hcengineering/server-ai-bot-resources')) addLocation(serverAiBotId, () => import('@hcengineering/server-ai-bot-resources'))
addLocation(serverFulltextId, () => import('@hcengineering/server-fulltext-resources'))
addLocation(serverProcessId, () => import('@hcengineering/server-process-resources')) addLocation(serverProcessId, () => import('@hcengineering/server-process-resources'))
} }
+69 -20
View File
@@ -13,38 +13,38 @@
// limitations under the License. // limitations under the License.
// //
import { getClient as getAccountClient, isWorkspaceLoginInfo } from '@hcengineering/account-client'
import { Analytics } from '@hcengineering/analytics' import { Analytics } from '@hcengineering/analytics'
import core, { import core, {
AccountRole, AccountRole,
cutObjectArray, cutObjectArray,
Data,
generateId, generateId,
isArchivingMode, isArchivingMode,
isMigrationMode, isMigrationMode,
isRestoringMode, isRestoringMode,
isWorkspaceCreating, isWorkspaceCreating,
TxFactory, pickPrimarySocialId,
WorkspaceEvent, platformNow,
platformNowDiff,
systemAccountUuid, systemAccountUuid,
TxFactory,
Version,
versionToString, versionToString,
withContext, withContext,
WorkspaceEvent,
type Account,
type AccountUuid,
type Branding, type Branding,
type BrandingMap, type BrandingMap,
type MeasureContext, type MeasureContext,
type PersonId,
type Tx, type Tx,
type TxWorkspaceEvent, type TxWorkspaceEvent,
type WorkspaceUuid,
type WorkspaceInfoWithStatus,
Account,
pickPrimarySocialId,
type PersonId,
type WorkspaceDataId, type WorkspaceDataId,
Data, type WorkspaceInfoWithStatus,
Version, type WorkspaceUuid
platformNow,
platformNowDiff,
AccountUuid
} from '@hcengineering/core' } from '@hcengineering/core'
import { getClient as getAccountClient, isWorkspaceLoginInfo } from '@hcengineering/account-client'
import { unknownError, type Status } from '@hcengineering/platform' import { unknownError, type Status } from '@hcengineering/platform'
import { type HelloRequest, type HelloResponse, type Request, type Response } from '@hcengineering/rpc' import { type HelloRequest, type HelloResponse, type Request, type Response } from '@hcengineering/rpc'
import { import {
@@ -52,14 +52,21 @@ import {
pingConst, pingConst,
Pipeline, Pipeline,
PipelineFactory, PipelineFactory,
QueueTopic,
QueueUserMessage,
ServerFactory, ServerFactory,
SessionManager, SessionManager,
StorageAdapter, StorageAdapter,
type SessionFactory, userEvents,
workspaceEvents,
type AddSessionResponse, type AddSessionResponse,
type ClientSessionCtx, type ClientSessionCtx,
type ConnectionSocket, type ConnectionSocket,
type PlatformQueue,
type PlatformQueueProducer,
type QueueWorkspaceMessage,
type Session, type Session,
type SessionFactory,
type Workspace type Workspace
} from '@hcengineering/server-core' } from '@hcengineering/server-core'
import { generateToken, type Token } from '@hcengineering/server-token' import { generateToken, type Token } from '@hcengineering/server-token'
@@ -108,7 +115,8 @@ export class TSessionManager implements SessionManager {
oldClientErrors: number = 0 oldClientErrors: number = 0
clientErrors: number = 0 clientErrors: number = 0
lastClients: string[] = [] lastClients: string[] = []
workspaceProducer: PlatformQueueProducer<QueueWorkspaceMessage>
usersProducer: PlatformQueueProducer<QueueUserMessage>
constructor ( constructor (
readonly ctx: MeasureContext, readonly ctx: MeasureContext,
readonly sessionFactory: SessionFactory, readonly sessionFactory: SessionFactory,
@@ -122,13 +130,16 @@ export class TSessionManager implements SessionManager {
| undefined, | undefined,
readonly accountsUrl: string, readonly accountsUrl: string,
readonly enableCompression: boolean, readonly enableCompression: boolean,
readonly doHandleTick: boolean = true readonly doHandleTick: boolean = true,
readonly queue: PlatformQueue
) { ) {
if (this.doHandleTick) { if (this.doHandleTick) {
this.checkInterval = setInterval(() => { this.checkInterval = setInterval(() => {
this.handleTick() this.handleTick()
}, 1000 / ticksPerSecond) }, 1000 / ticksPerSecond)
} }
this.workspaceProducer = this.queue.createProducer(ctx.newChild('queue', {}), QueueTopic.Workspace)
this.usersProducer = this.queue.createProducer(ctx.newChild('queue', {}), QueueTopic.Users)
} }
scheduleMaintenance (timeMinutes: number): void { scheduleMaintenance (timeMinutes: number): void {
@@ -357,6 +368,12 @@ export class TSessionManager implements SessionManager {
} }
} }
countUserSessions (workspace: Workspace, accountUuid: AccountUuid): number {
return Array.from(workspace.sessions.values())
.filter((it) => it.session.getUser() === accountUuid)
.reduce<number>((acc) => acc + 1, 0)
}
tickCounter = 0 tickCounter = 0
@withContext('📲 add-session') @withContext('📲 add-session')
@@ -479,6 +496,7 @@ export class TSessionManager implements SessionManager {
workspaceInfo.dataId, workspaceInfo.dataId,
branding branding
) )
await this.workspaceProducer.send(workspaceInfo.uuid, [workspaceEvents.open()])
} }
let pipeline: Pipeline let pipeline: Pipeline
@@ -546,13 +564,26 @@ export class TSessionManager implements SessionManager {
this.tickCounter++ this.tickCounter++
workspace.sessions.set(session.sessionId, { session, socket: ws, tickHash: this.tickCounter % ticksPerSecond }) workspace.sessions.set(session.sessionId, { session, socket: ws, tickHash: this.tickCounter % ticksPerSecond })
const accountUuid = account.uuid
await this.usersProducer.send(workspaceInfo.uuid, [
userEvents.login({
user: accountUuid,
sessions: this.countUserSessions(workspace, accountUuid),
socialIds: account.socialIds
})
])
// Mark workspace as init completed and we had at least one client. // Mark workspace as init completed and we had at least one client.
if (!workspace.workspaceInitCompleted) { if (!workspace.workspaceInitCompleted) {
workspace.workspaceInitCompleted = true workspace.workspaceInitCompleted = true
} }
if (this.timeMinutes > 0) { if (this.timeMinutes > 0) {
void ws.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression) void ws
.send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression)
.catch((err) => {
ctx.error('failed to send maintenance warning', err)
})
} }
return { session, context: workspace.context, workspaceId: workspaceInfo.uuid } return { session, context: workspace.context, workspaceId: workspaceInfo.uuid }
} }
@@ -809,8 +840,19 @@ export class TSessionManager implements SessionManager {
totalUsers: this.sessions.size totalUsers: this.sessions.size
}) })
this.sessions.delete(ws.id) this.sessions.delete(ws.id)
if (workspace !== undefined) { if (workspace !== undefined) {
workspace.sessions.delete(sessionRef.session.sessionId) workspace.sessions.delete(sessionRef.session.sessionId)
const userUuid = sessionRef.session.getUser()
await this.usersProducer.send(workspace.workspaceUuid, [
userEvents.logout({
user: userUuid,
sessions: this.countUserSessions(workspace, userUuid),
socialIds: sessionRef.session.getUserSocialIds()
})
])
const pipeline = workspace.pipeline instanceof Promise ? await workspace.pipeline : workspace.pipeline const pipeline = workspace.pipeline instanceof Promise ? await workspace.pipeline : workspace.pipeline
if (this.doHandleTick) { if (this.doHandleTick) {
@@ -967,6 +1009,8 @@ export class TSessionManager implements SessionManager {
if (LOGGING_ENABLED) { if (LOGGING_ENABLED) {
this.ctx.warn('Closed workspace', logParams) this.ctx.warn('Closed workspace', logParams)
} }
await this.workspaceProducer.send(workspace.workspaceUuid, [workspaceEvents.down()])
} }
} catch (err: any) { } catch (err: any) {
Analytics.handleError(err) Analytics.handleError(err)
@@ -1266,7 +1310,8 @@ export function createSessionManager (
| undefined, | undefined,
accountsUrl: string, accountsUrl: string,
enableCompression: boolean, enableCompression: boolean,
doHandleTick: boolean = true doHandleTick: boolean = true,
queue: PlatformQueue
): SessionManager { ): SessionManager {
return new TSessionManager( return new TSessionManager(
ctx, ctx,
@@ -1276,7 +1321,8 @@ export function createSessionManager (
profiling, profiling,
accountsUrl, accountsUrl,
enableCompression, enableCompression,
doHandleTick doHandleTick,
queue
) )
} }
@@ -1298,6 +1344,7 @@ export function startSessionManager (
start: () => void start: () => void
stop: () => Promise<string | undefined> stop: () => Promise<string | undefined>
} }
queue: PlatformQueue
} & Partial<Timeouts> } & Partial<Timeouts>
): { shutdown: () => Promise<void>, sessionManager: SessionManager } { ): { shutdown: () => Promise<void>, sessionManager: SessionManager } {
const sessions = createSessionManager( const sessions = createSessionManager(
@@ -1310,7 +1357,9 @@ export function startSessionManager (
}, },
opt.profiling, opt.profiling,
opt.accountsUrl, opt.accountsUrl,
opt.enableCompression ?? false opt.enableCompression ?? false,
true,
opt.queue
) )
return { return {
shutdown: opt.serverFactory( shutdown: opt.serverFactory(
+3
View File
@@ -25,6 +25,7 @@ import {
import { type MigrateOperation } from '@hcengineering/model' import { type MigrateOperation } from '@hcengineering/model'
import { setMetadata } from '@hcengineering/platform' import { setMetadata } from '@hcengineering/platform'
import serverClientPlugin from '@hcengineering/server-client' import serverClientPlugin from '@hcengineering/server-client'
import { QueueTopic, type PlatformQueue, type QueueWorkspaceMessage } from '@hcengineering/server-core'
import serverNotification from '@hcengineering/server-notification' import serverNotification from '@hcengineering/server-notification'
import { createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { createStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import serverToken from '@hcengineering/server-token' import serverToken from '@hcengineering/server-token'
@@ -37,6 +38,7 @@ export * from './ws-operations'
*/ */
export function serveWorkspaceAccount ( export function serveWorkspaceAccount (
measureCtx: MeasureContext, measureCtx: MeasureContext,
queue: PlatformQueue,
version: Data<Version>, version: Data<Version>,
txes: Tx[], txes: Tx[],
migrateOperations: [string, MigrateOperation][], migrateOperations: [string, MigrateOperation][],
@@ -122,6 +124,7 @@ export function serveWorkspaceAccount (
let canceled = false let canceled = false
const worker = new WorkspaceWorker( const worker = new WorkspaceWorker(
queue.createProducer<QueueWorkspaceMessage>(measureCtx, QueueTopic.Workspace),
version, version,
txes, txes,
migrateOperations, migrateOperations,
+18 -29
View File
@@ -14,13 +14,13 @@
// //
import { getClient as getAccountClient } from '@hcengineering/account-client' import { getClient as getAccountClient } from '@hcengineering/account-client'
import { import {
DOMAIN_BLOB,
getBranding, getBranding,
isArchivingMode, isArchivingMode,
isMigrationMode, isMigrationMode,
isRestoringMode, isRestoringMode,
systemAccountUuid, systemAccountUuid,
type BrandingMap, type BrandingMap,
DOMAIN_BLOB,
type Data, type Data,
type MeasureContext, type MeasureContext,
type Tx, type Tx,
@@ -53,7 +53,13 @@ import {
shutdownPostgres shutdownPostgres
} from '@hcengineering/postgres' } from '@hcengineering/postgres'
import { doBackupWorkspace, doRestoreWorkspace } from '@hcengineering/server-backup' import { doBackupWorkspace, doRestoreWorkspace } from '@hcengineering/server-backup'
import type { PipelineFactory, StorageAdapter } from '@hcengineering/server-core' import {
workspaceEvents,
type PipelineFactory,
type PlatformQueueProducer,
type QueueWorkspaceMessage,
type StorageAdapter
} from '@hcengineering/server-core'
import { import {
createBackupPipeline, createBackupPipeline,
getConfig, getConfig,
@@ -101,6 +107,7 @@ export class WorkspaceWorker {
resolveBusy: (() => void) | null = null resolveBusy: (() => void) | null = null
constructor ( constructor (
readonly workspaceQueue: PlatformQueueProducer<QueueWorkspaceMessage>,
readonly version: Data<Version>, readonly version: Data<Version>,
readonly txes: Tx[], readonly txes: Tx[],
readonly migrationOperation: [string, MigrateOperation][], readonly migrationOperation: [string, MigrateOperation][],
@@ -270,6 +277,8 @@ export class WorkspaceWorker {
region: this.region, region: this.region,
time: Date.now() - t time: Date.now() - t
}) })
await this.workspaceQueue.send(ws.uuid, [workspaceEvents.created()])
} catch (err: any) { } catch (err: any) {
await opt.errorHandler(ws, err) await opt.errorHandler(ws, err)
@@ -285,6 +294,7 @@ export class WorkspaceWorker {
region: this.region, region: this.region,
time: Date.now() - t time: Date.now() - t
}) })
await this.workspaceQueue.send(ws.uuid, [workspaceEvents.createFailed()])
} finally { } finally {
if (!opt.console) { if (!opt.console) {
;(logger as FileModelLogger).close() ;(logger as FileModelLogger).close()
@@ -365,6 +375,7 @@ export class WorkspaceWorker {
region: this.region, region: this.region,
time: Date.now() - t time: Date.now() - t
}) })
await this.workspaceQueue.send(ws.uuid, [workspaceEvents.upgraded()])
} catch (err: any) { } catch (err: any) {
await opt.errorHandler(ws, err) await opt.errorHandler(ws, err)
@@ -381,6 +392,7 @@ export class WorkspaceWorker {
region: this.region, region: this.region,
time: Date.now() - t time: Date.now() - t
}) })
await this.workspaceQueue.send(ws.uuid, [workspaceEvents.upgradeFailed()])
} finally { } finally {
if (!opt.console) { if (!opt.console) {
;(logger as FileModelLogger).close() ;(logger as FileModelLogger).close()
@@ -396,32 +408,7 @@ export class WorkspaceWorker {
const adapter = getWorkspaceDestroyAdapter(dbUrl) const adapter = getWorkspaceDestroyAdapter(dbUrl)
await adapter.deleteWorkspace(ctx, sharedPipelineContextVars, workspace.uuid, workspace.dataId) await adapter.deleteWorkspace(ctx, sharedPipelineContextVars, workspace.uuid, workspace.dataId)
await this.doReindexFulltext(ctx, workspace, cleanIndexes) await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.clearIndex()])
}
private async doReindexFulltext (
ctx: MeasureContext,
workspace: WorkspaceInfoWithStatus,
cleanIndexes: boolean
): Promise<void> {
if (this.fulltextUrl !== undefined) {
const token = generateToken(systemAccountUuid, workspace.uuid, { service: 'workspace' })
try {
const res = await fetch(this.fulltextUrl + '/api/v1/reindex', {
method: 'PUT',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ token, onlyDrop: cleanIndexes })
})
if (!res.ok) {
throw new Error(`HTTP Error ${res.status} ${res.statusText}`)
}
} catch (err: any) {
ctx.error('failed to reset index', { err })
}
}
} }
async sendTransactorMaitenance (token: string, ws: WorkspaceUuid): Promise<void> { async sendTransactorMaitenance (token: string, ws: WorkspaceUuid): Promise<void> {
@@ -490,6 +477,7 @@ export class WorkspaceWorker {
return return
} }
await sendEvent('archiving-clean-done', 100) await sendEvent('archiving-clean-done', 100)
await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.archived()])
break break
} }
case 'pending-deletion': case 'pending-deletion':
@@ -504,6 +492,7 @@ export class WorkspaceWorker {
return return
} }
await sendEvent('delete-done', 100) await sendEvent('delete-done', 100)
await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.deleted()])
break break
} }
@@ -537,8 +526,8 @@ export class WorkspaceWorker {
await sendEvent('restore-started', 0) await sendEvent('restore-started', 0)
if (await this.doRestore(ctx, workspace, opt)) { if (await this.doRestore(ctx, workspace, opt)) {
// We should reindex fulltext // We should reindex fulltext
await this.doReindexFulltext(ctx, workspace, false)
await sendEvent('restore-done', 100) await sendEvent('restore-done', 100)
await this.workspaceQueue.send(workspace.uuid, [workspaceEvents.restored()])
} }
break break
default: default:
+5 -3
View File
@@ -41,7 +41,7 @@ import {
type WorkspaceUuid type WorkspaceUuid
} from '@hcengineering/core' } from '@hcengineering/core'
import { ClientSession, startSessionManager } from '@hcengineering/server' import { ClientSession, startSessionManager } from '@hcengineering/server'
import { createDummyStorageAdapter } from '@hcengineering/server-core' import { createDummyQueue, createDummyStorageAdapter } from '@hcengineering/server-core'
import { startHttpServer } from '../server_http' import { startHttpServer } from '../server_http'
import { genMinModel } from './minmodel' import { genMinModel } from './minmodel'
@@ -101,7 +101,8 @@ describe('server', () => {
brandingMap: {}, brandingMap: {},
serverFactory: startHttpServer, serverFactory: startHttpServer,
accountsUrl: '', accountsUrl: '',
externalStorage: createDummyStorageAdapter() externalStorage: createDummyStorageAdapter(),
queue: createDummyQueue()
}) })
function connect (): WebSocket { function connect (): WebSocket {
@@ -212,7 +213,8 @@ describe('server', () => {
brandingMap: {}, brandingMap: {},
serverFactory: startHttpServer, serverFactory: startHttpServer,
accountsUrl: '', accountsUrl: '',
externalStorage: createDummyStorageAdapter() externalStorage: createDummyStorageAdapter(),
queue: createDummyQueue()
}) })
async function findClose (token: string, timeoutPromise: Promise<void>, code: number): Promise<string> { async function findClose (token: string, timeoutPromise: Promise<void>, code: number): Promise<string> {
+2 -2
View File
@@ -193,7 +193,7 @@ export function startHttpServer (
app.get('/api/v1/profiling', (req, res) => { app.get('/api/v1/profiling', (req, res) => {
try { try {
const token = req.query.token as string const token = (req.query.token as string) ?? (req.headers.authorization ?? '').split(' ')[1]
decodeToken(token) decodeToken(token)
const jsonData = { const jsonData = {
profiling profiling
@@ -407,7 +407,7 @@ export function startHttpServer (
app.put('/api/v1/broadcast', (req, res) => { app.put('/api/v1/broadcast', (req, res) => {
try { try {
const token = req.query.token as string const token = (req.query.token as string) ?? (req.headers.authorization ?? '').split(' ')[1]
decodeToken(token) decodeToken(token)
const ws = sessions.workspaces.get(req.query.workspace as WorkspaceUuid) const ws = sessions.workspaces.get(req.query.workspace as WorkspaceUuid)
if (ws !== undefined) { if (ws !== undefined) {

Some files were not shown because too many files have changed in this diff Show More