WEBDIENSTE/huly-platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2026-09-22 01:25:00 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into staging

Browse Source 
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev
2025-01-10 23:04:25 +07:00
parent e82f0efaaf 71ce4018eb
commit fbbb3631d6
83 changed files with 1242 additions and 2684 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/config/rush/pnpm-lock.yaml
Generated
+238 -145
View File
@@ -61,10 +61,10 @@ dependencies:
version: 1.49.1
'@rush-temp/account':
specifier: file:./projects/account.tgz
version: file:projects/account.tgz(@types/node@20.11.19)(esbuild@0.24.2)(ts-node@10.9.2)
version: file:projects/account.tgz(@types/node@20.11.19)(esbuild@0.24.2)(snappy@7.2.2)(ts-node@10.9.2)
'@rush-temp/account-service':
specifier: file:./projects/account-service.tgz
version: file:projects/account-service.tgz
version: file:projects/account-service.tgz(snappy@7.2.2)
'@rush-temp/activity':
specifier: file:./projects/activity.tgz
version: file:projects/activity.tgz(@types/node@20.11.19)(esbuild@0.24.2)(ts-node@10.9.2)
@@ -109,7 +109,7 @@ dependencies:
version: file:projects/attachment-resources.tgz(@types/node@20.11.19)(esbuild@0.24.2)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2)
'@rush-temp/auth-providers':
specifier: file:./projects/auth-providers.tgz
version: file:projects/auth-providers.tgz(@types/node@20.11.19)(esbuild@0.24.2)(ts-node@10.9.2)
version: file:projects/auth-providers.tgz(@types/node@20.11.19)(esbuild@0.24.2)(snappy@7.2.2)(ts-node@10.9.2)
'@rush-temp/backup-service':
specifier: file:./projects/backup-service.tgz
version: file:projects/backup-service.tgz(esbuild@0.24.2)(ts-node@10.9.2)
@@ -158,9 +158,6 @@ dependencies:
'@rush-temp/cloud-branding':
specifier: file:./projects/cloud-branding.tgz
version: file:projects/cloud-branding.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2)(utf-8-validate@6.0.4)
'@rush-temp/cloud-datalake':
specifier: file:./projects/cloud-datalake.tgz
version: file:projects/cloud-datalake.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2)(utf-8-validate@6.0.4)
'@rush-temp/cloud-transactor':
specifier: file:./projects/cloud-transactor.tgz
version: file:projects/cloud-transactor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(ts-node@10.9.2)(utf-8-validate@6.0.4)
@@ -169,7 +166,7 @@ dependencies:
version: file:projects/collaboration.tgz(esbuild@0.24.2)(ts-node@10.9.2)
'@rush-temp/collaborator':
specifier: file:./projects/collaborator.tgz
version: file:projects/collaborator.tgz(@tiptap/pm@2.6.6)(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-protocols@1.0.6)
version: file:projects/collaborator.tgz(@tiptap/pm@2.6.6)(bufferutil@4.0.8)(snappy@7.2.2)(utf-8-validate@6.0.4)(y-protocols@1.0.6)
'@rush-temp/collaborator-client':
specifier: file:./projects/collaborator-client.tgz
version: file:projects/collaborator-client.tgz(ts-node@10.9.2)
@@ -583,7 +580,7 @@ dependencies:
version: file:projects/model-workbench.tgz
'@rush-temp/mongo':
specifier: file:./projects/mongo.tgz
version: file:projects/mongo.tgz(esbuild@0.24.2)(ts-node@10.9.2)
version: file:projects/mongo.tgz(esbuild@0.24.2)(snappy@7.2.2)(ts-node@10.9.2)
'@rush-temp/notification':
specifier: file:./projects/notification.tgz
version: file:projects/notification.tgz(@types/node@20.11.19)(esbuild@0.24.2)(ts-node@10.9.2)
@@ -616,19 +613,19 @@ dependencies:
version: file:projects/platform-rig.tgz(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(svelte@4.2.19)
'@rush-temp/pod-account':
specifier: file:./projects/pod-account.tgz
version: file:projects/pod-account.tgz
version: file:projects/pod-account.tgz(snappy@7.2.2)
'@rush-temp/pod-ai-bot':
specifier: file:./projects/pod-ai-bot.tgz
version: file:projects/pod-ai-bot.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(zod@3.23.8)
version: file:projects/pod-ai-bot.tgz(bufferutil@4.0.8)(snappy@7.2.2)(utf-8-validate@6.0.4)(zod@3.23.8)
'@rush-temp/pod-analytics-collector':
specifier: file:./projects/pod-analytics-collector.tgz
version: file:projects/pod-analytics-collector.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)
version: file:projects/pod-analytics-collector.tgz(bufferutil@4.0.8)(snappy@7.2.2)(utf-8-validate@6.0.4)
'@rush-temp/pod-backup':
specifier: file:./projects/pod-backup.tgz
version: file:projects/pod-backup.tgz
'@rush-temp/pod-calendar':
specifier: file:./projects/pod-calendar.tgz
version: file:projects/pod-calendar.tgz(bufferutil@4.0.8)(ts-node@10.9.2)(utf-8-validate@6.0.4)
version: file:projects/pod-calendar.tgz(bufferutil@4.0.8)(snappy@7.2.2)(ts-node@10.9.2)(utf-8-validate@6.0.4)
'@rush-temp/pod-collaborator':
specifier: file:./projects/pod-collaborator.tgz
version: file:projects/pod-collaborator.tgz
@@ -640,10 +637,10 @@ dependencies:
version: file:projects/pod-fulltext.tgz
'@rush-temp/pod-github':
specifier: file:./projects/pod-github.tgz
version: file:projects/pod-github.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-prosemirror@1.2.12)
version: file:projects/pod-github.tgz(bufferutil@4.0.8)(snappy@7.2.2)(utf-8-validate@6.0.4)(y-prosemirror@1.2.12)
'@rush-temp/pod-gmail':
specifier: file:./projects/pod-gmail.tgz
version: file:projects/pod-gmail.tgz(ts-node@10.9.2)
version: file:projects/pod-gmail.tgz(snappy@7.2.2)(ts-node@10.9.2)
'@rush-temp/pod-love':
specifier: file:./projects/pod-love.tgz
version: file:projects/pod-love.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)
@@ -658,19 +655,19 @@ dependencies:
version: file:projects/pod-ses.tgz
'@rush-temp/pod-sign':
specifier: file:./projects/pod-sign.tgz
version: file:projects/pod-sign.tgz
version: file:projects/pod-sign.tgz(snappy@7.2.2)
'@rush-temp/pod-stats':
specifier: file:./projects/pod-stats.tgz
version: file:projects/pod-stats.tgz
'@rush-temp/pod-telegram':
specifier: file:./projects/pod-telegram.tgz
version: file:projects/pod-telegram.tgz(bufferutil@4.0.8)(ts-node@10.9.2)(utf-8-validate@6.0.4)
version: file:projects/pod-telegram.tgz(bufferutil@4.0.8)(snappy@7.2.2)(ts-node@10.9.2)(utf-8-validate@6.0.4)
'@rush-temp/pod-telegram-bot':
specifier: file:./projects/pod-telegram-bot.tgz
version: file:projects/pod-telegram-bot.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)
version: file:projects/pod-telegram-bot.tgz(bufferutil@4.0.8)(snappy@7.2.2)(utf-8-validate@6.0.4)
'@rush-temp/pod-workspace':
specifier: file:./projects/pod-workspace.tgz
version: file:projects/pod-workspace.tgz
version: file:projects/pod-workspace.tgz(snappy@7.2.2)
'@rush-temp/postgres':
specifier: file:./projects/postgres.tgz
version: file:projects/postgres.tgz(esbuild@0.24.2)(ts-node@10.9.2)
@@ -961,7 +958,7 @@ dependencies:
version: file:projects/server-token.tgz(esbuild@0.24.2)(ts-node@10.9.2)
'@rush-temp/server-tool':
specifier: file:./projects/server-tool.tgz
version: file:projects/server-tool.tgz(@types/node@20.11.19)(esbuild@0.24.2)(ts-node@10.9.2)
version: file:projects/server-tool.tgz(@types/node@20.11.19)(esbuild@0.24.2)(snappy@7.2.2)(ts-node@10.9.2)
'@rush-temp/server-tracker':
specifier: file:./projects/server-tracker.tgz
version: file:projects/server-tracker.tgz(esbuild@0.24.2)(ts-node@10.9.2)
@@ -1093,7 +1090,7 @@ dependencies:
version: file:projects/time-resources.tgz(@tiptap/core@2.6.6)(@tiptap/pm@2.6.6)(@types/node@20.11.19)(esbuild@0.24.2)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2)
'@rush-temp/tool':
specifier: file:./projects/tool.tgz
version: file:projects/tool.tgz
version: file:projects/tool.tgz(snappy@7.2.2)
'@rush-temp/tracker':
specifier: file:./projects/tracker.tgz
version: file:projects/tracker.tgz(@types/node@20.11.19)(esbuild@0.24.2)(ts-node@10.9.2)
@@ -1238,6 +1235,9 @@ dependencies:
'@tiptap/extension-text-align':
specifier: ~2.11.0
version: 2.11.0(@tiptap/core@2.6.6)
'@tiptap/extension-text-style':
specifier: ~2.11.0
version: 2.11.0(@tiptap/core@2.6.6)
'@tiptap/extension-typography':
specifier: ^2.6.6
version: 2.6.6(@tiptap/core@2.6.6)
@@ -1376,6 +1376,9 @@ dependencies:
'@types/sharp':
specifier: ~0.32.0
version: 0.32.0
'@types/snappyjs':
specifier: ^0.7.1
version: 0.7.1
'@types/tar-stream':
specifier: ^3.1.3
version: 3.1.3
@@ -1442,9 +1445,6 @@ dependencies:
aws-sdk:
specifier: ^2.1423.0
version: 2.1664.0
aws4fetch:
specifier: ^1.0.20
version: 1.0.20
base64-js:
specifier: ^1.5.1
version: 1.5.1
@@ -1741,7 +1741,7 @@ dependencies:
version: 8.0.0
mongodb:
specifier: ^6.12.0
version: 6.12.0
version: 6.12.0(snappy@7.2.2)
morgan:
specifier: ^1.10.0
version: 1.10.0
@@ -1850,6 +1850,12 @@ dependencies:
smee-client:
specifier: ^1.2.3
version: 1.2.5
snappy:
specifier: ^7.2.2
version: 7.2.2
snappyjs:
specifier: ^0.7.0
version: 0.7.0
style-loader:
specifier: ^3.3.1
version: 3.3.4(webpack@5.97.1)
@@ -4927,6 +4933,123 @@ packages:
dev: false
optional: true
/@napi-rs/snappy-android-arm-eabi@7.2.2:
resolution: {integrity: sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw==}
engines: {node: '>= 10'}
cpu: [arm]
os: [android]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-android-arm64@7.2.2:
resolution: {integrity: sha512-2R/A3qok+nGtpVK8oUMcrIi5OMDckGYNoBLFyli3zp8w6IArPRfg1yOfVUcHvpUDTo9T7LOS1fXgMOoC796eQw==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [android]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-darwin-arm64@7.2.2:
resolution: {integrity: sha512-USgArHbfrmdbuq33bD5ssbkPIoT7YCXCRLmZpDS6dMDrx+iM7eD2BecNbOOo7/v1eu6TRmQ0xOzeQ6I/9FIi5g==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [darwin]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-darwin-x64@7.2.2:
resolution: {integrity: sha512-0APDu8iO5iT0IJKblk2lH0VpWSl9zOZndZKnBYIc+ei1npw2L5QvuErFOTeTdHBtzvUHASB+9bvgaWnQo4PvTQ==}
engines: {node: '>= 10'}
cpu: [x64]
os: [darwin]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-freebsd-x64@7.2.2:
resolution: {integrity: sha512-mRTCJsuzy0o/B0Hnp9CwNB5V6cOJ4wedDTWEthsdKHSsQlO7WU9W1yP7H3Qv3Ccp/ZfMyrmG98Ad7u7lG58WXA==}
engines: {node: '>= 10'}
cpu: [x64]
os: [freebsd]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-linux-arm-gnueabihf@7.2.2:
resolution: {integrity: sha512-v1uzm8+6uYjasBPcFkv90VLZ+WhLzr/tnfkZ/iD9mHYiULqkqpRuC8zvc3FZaJy5wLQE9zTDkTJN1IvUcZ+Vcg==}
engines: {node: '>= 10'}
cpu: [arm]
os: [linux]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-linux-arm64-gnu@7.2.2:
resolution: {integrity: sha512-LrEMa5pBScs4GXWOn6ZYXfQ72IzoolZw5txqUHVGs8eK4g1HR9HTHhb2oY5ySNaKakG5sOgMsb1rwaEnjhChmQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-linux-arm64-musl@7.2.2:
resolution: {integrity: sha512-3orWZo9hUpGQcB+3aTLW7UFDqNCQfbr0+MvV67x8nMNYj5eAeUtMmUE/HxLznHO4eZ1qSqiTwLbVx05/Socdlw==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-linux-x64-gnu@7.2.2:
resolution: {integrity: sha512-jZt8Jit/HHDcavt80zxEkDpH+R1Ic0ssiVCoueASzMXa7vwPJeF4ZxZyqUw4qeSy7n8UUExomu8G8ZbP6VKhgw==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-linux-x64-musl@7.2.2:
resolution: {integrity: sha512-Dh96IXgcZrV39a+Tej/owcd9vr5ihiZ3KRix11rr1v0MWtVb61+H1GXXlz6+Zcx9y8jM1NmOuiIuJwkV4vZ4WA==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-win32-arm64-msvc@7.2.2:
resolution: {integrity: sha512-9No0b3xGbHSWv2wtLEn3MO76Yopn1U2TdemZpCaEgOGccz1V+a/1d16Piz3ofSmnA13HGFz3h9NwZH9EOaIgYA==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [win32]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-win32-ia32-msvc@7.2.2:
resolution: {integrity: sha512-QiGe+0G86J74Qz1JcHtBwM3OYdTni1hX1PFyLRo3HhQUSpmi13Bzc1En7APn+6Pvo7gkrcy81dObGLDSxFAkQQ==}
engines: {node: '>= 10'}
cpu: [ia32]
os: [win32]
requiresBuild: true
dev: false
optional: true
/@napi-rs/snappy-win32-x64-msvc@7.2.2:
resolution: {integrity: sha512-a43cyx1nK0daw6BZxVcvDEXxKMFLSBSDTAhsFD0VqSKcC7MGUBMaqyoWUcMiI7LBSz4bxUmxDWKfCYzpEmeb3w==}
engines: {node: '>= 10'}
cpu: [x64]
os: [win32]
requiresBuild: true
dev: false
optional: true
/@nodelib/fs.scandir@2.1.5:
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
engines: {node: '>= 8'}
@@ -6260,6 +6383,14 @@ packages:
'@tiptap/core': 2.6.6(@tiptap/pm@2.6.6)
dev: false
/@tiptap/extension-text-style@2.11.0(@tiptap/core@2.6.6):
resolution: {integrity: sha512-vuA16wMZ6J3fboL7FObwV2f5uN9Vg0WYmqU7971vxzJyaRj9VE1eeH8Kh5fq4RgwDzc13MZGvZZV4HcE1R8o8A==}
peerDependencies:
'@tiptap/core': ^2.7.0
dependencies:
'@tiptap/core': 2.6.6(@tiptap/pm@2.6.6)
dev: false
/@tiptap/extension-text@2.6.6(@tiptap/core@2.6.6):
resolution: {integrity: sha512-e84uILnRzNzcwK1DVQNpXVmBG1Cq3BJipTOIDl1LHifOok7MBjhI/X+/NR0bd3N2t6gmDTWi63+4GuJ5EeDmsg==}
peerDependencies:
@@ -6918,13 +7049,6 @@ packages:
'@types/koa': 2.15.0
dev: false
/@types/koa-morgan@1.0.8:
resolution: {integrity: sha512-2GredUi+iA3V0XrbzdsOAYgwj4F6+FnN+f5YjoKjessIE2lrMkqnc06YQQnzbMG75hRsXjyD+p6d5vlI70s1vg==}
dependencies:
'@types/koa': 2.14.0
'@types/morgan': 1.9.9
dev: false
/@types/koa-passport@6.0.3:
resolution: {integrity: sha512-9iV8aiSICfBZpqkkzUshqxN2wTuoj4HGRRH0Idv3lHhXV9rZRCOyDdJSg+vZ6RHSQg0ZeBeAdIL4r/l8bR4SPA==}
dependencies:
@@ -6945,19 +7069,6 @@ packages:
'@types/koa': 2.15.0
dev: false
/@types/koa@2.14.0:
resolution: {integrity: sha512-DTDUyznHGNHAl+wd1n0z1jxNajduyTh8R53xoewuerdBzGo6Ogj6F2299BFtrexJw4NtgjsI5SMPCmV9gZwGXA==}
dependencies:
'@types/accepts': 1.3.7
'@types/content-disposition': 0.5.8
'@types/cookies': 0.9.0
'@types/http-assert': 1.5.5
'@types/http-errors': 2.0.4
'@types/keygrip': 1.0.6
'@types/koa-compose': 3.2.8
'@types/node': 20.11.19
dev: false
/@types/koa@2.15.0:
resolution: {integrity: sha512-7QFsywoE5URbuVnG3loe03QXuGajrnotr3gQkXcEBShORai23MePfFYdhz90FEtBBpkyIYQbVD+evKtloCgX3g==}
dependencies:
@@ -7192,6 +7303,10 @@ packages:
sharp: 0.32.6
dev: false
/@types/snappyjs@0.7.1:
resolution: {integrity: sha512-OxjzJ6cQZstysMh6PEwZWmK9qlKZyezHJKOkcUkZDooSFuog2votUEKkxMaTq51UQF3cJkXKQ+XGlj4FSl8JQQ==}
dev: false
/@types/sockjs@0.3.36:
resolution: {integrity: sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==}
dependencies:
@@ -8652,10 +8767,6 @@ packages:
xml2js: 0.6.2
dev: false
/aws4fetch@1.0.20:
resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==}
dev: false
/axobject-query@4.0.0:
resolution: {integrity: sha512-+60uv1hiVFhHZeO+Lz0RYzsVHy5Wr1ayX0mwda9KPDVLNJgZ1T9Ny7VmFbLDzxsH0D87I86vgj3gFrjTJUYznw==}
dependencies:
@@ -15097,12 +15208,6 @@ packages:
koa-compose: 4.1.0
dev: false
/koa-morgan@1.0.1:
resolution: {integrity: sha512-JOUdCNlc21G50afBXfErUrr1RKymbgzlrO5KURY+wmDG1Uvd2jmxUJcHgylb/mYXy2SjiNZyYim/ptUBGsIi3A==}
dependencies:
morgan: 1.10.0
dev: false
/koa-passport@6.0.0:
resolution: {integrity: sha512-bgcrQN7Ylfgi1PVr5l6hHYkr38RHUzx+ty3m7e/xoTte8MR0zbDt6+pvP3/nuF/yXL6Ba7IzX1rSqmCy6OrrIw==}
engines: {node: '>= 4'}
@@ -16209,7 +16314,7 @@ packages:
whatwg-url: 13.0.0
dev: false
/mongodb@6.12.0:
/mongodb@6.12.0(snappy@7.2.2):
resolution: {integrity: sha512-RM7AHlvYfS7jv7+BXund/kR64DryVI+cHbVAy9P61fnb1RcWZqOW1/Wj2YhqMCx+MuYhqTRGv7AwHBzmsCKBfA==}
engines: {node: '>=16.20.1'}
peerDependencies:
@@ -16239,6 +16344,7 @@ packages:
'@mongodb-js/saslprep': 1.1.9
bson: 6.10.1
mongodb-connection-string-url: 3.0.0
snappy: 7.2.2
dev: false
/mongodb@6.3.0:
@@ -19011,6 +19117,29 @@ packages:
- supports-color
dev: false
/snappy@7.2.2:
resolution: {integrity: sha512-iADMq1kY0v3vJmGTuKcFWSXt15qYUz7wFkArOrsSg0IFfI3nJqIJvK2/ZbEIndg7erIJLtAVX2nSOqPz7DcwbA==}
engines: {node: '>= 10'}
optionalDependencies:
'@napi-rs/snappy-android-arm-eabi': 7.2.2
'@napi-rs/snappy-android-arm64': 7.2.2
'@napi-rs/snappy-darwin-arm64': 7.2.2
'@napi-rs/snappy-darwin-x64': 7.2.2
'@napi-rs/snappy-freebsd-x64': 7.2.2
'@napi-rs/snappy-linux-arm-gnueabihf': 7.2.2
'@napi-rs/snappy-linux-arm64-gnu': 7.2.2
'@napi-rs/snappy-linux-arm64-musl': 7.2.2
'@napi-rs/snappy-linux-x64-gnu': 7.2.2
'@napi-rs/snappy-linux-x64-musl': 7.2.2
'@napi-rs/snappy-win32-arm64-msvc': 7.2.2
'@napi-rs/snappy-win32-ia32-msvc': 7.2.2
'@napi-rs/snappy-win32-x64-msvc': 7.2.2
dev: false
/snappyjs@0.7.0:
resolution: {integrity: sha512-u5iEEXkMe2EInQio6Wv9LWHOQYRDbD2O9hzS27GpT/lwfIQhTCnHCTqedqHIHe9ZcvQo+9au6vngQayipz1NYw==}
dev: false
/sockjs@0.3.24:
resolution: {integrity: sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==}
dependencies:
@@ -21795,8 +21924,9 @@ packages:
resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==}
dev: false
file:projects/account-service.tgz:
file:projects/account-service.tgz(snappy@7.2.2):
resolution: {integrity: sha512-oAW0UD+ThGbtuXdyaoWJoe93XY7bLO0K2uZneePiHsI3KZM3z9st9fjxSbJE4wmC9iE2wmzNYTN6AHrcFxvRrQ==, tarball: file:projects/account-service.tgz}
id: file:projects/account-service.tgz
name: '@rush-temp/account-service'
version: 0.0.0
dependencies:
@@ -21804,7 +21934,6 @@ packages:
'@types/jest': 29.5.12
'@types/koa': 2.15.0
'@types/koa-bodyparser': 4.3.12
'@types/koa-morgan': 1.0.8
'@types/koa-router': 7.4.8
'@types/koa__cors': 5.0.0
'@types/node': 20.11.19
@@ -21820,9 +21949,8 @@ packages:
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
koa: 2.15.3
koa-bodyparser: 4.4.1
koa-morgan: 1.0.1
koa-router: 12.0.1
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
@@ -21845,7 +21973,7 @@ packages:
- supports-color
dev: false
file:projects/account.tgz(@types/node@20.11.19)(esbuild@0.24.2)(ts-node@10.9.2):
file:projects/account.tgz(@types/node@20.11.19)(esbuild@0.24.2)(snappy@7.2.2)(ts-node@10.9.2):
resolution: {integrity: sha512-aFUxA1twR0m+7sDVFXyKoeVQtVLMrr973LpLK6LHrRa+PD010X6+AyoOkkyVJKFN0Q6zu2Oqz88PsEt50Z2Mjg==, tarball: file:projects/account.tgz}
id: file:projects/account.tgz
name: '@rush-temp/account'
@@ -21861,7 +21989,7 @@ packages:
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
otp-generator: 4.0.1
postgres: 3.4.5
prettier: 3.2.5
@@ -22385,7 +22513,7 @@ packages:
- ts-node
dev: false
file:projects/auth-providers.tgz(@types/node@20.11.19)(esbuild@0.24.2)(ts-node@10.9.2):
file:projects/auth-providers.tgz(@types/node@20.11.19)(esbuild@0.24.2)(snappy@7.2.2)(ts-node@10.9.2):
resolution: {integrity: sha512-jUJhz8FiPT3x0ybVjdPysK9HBA/E1UzU+QraBxLurUXjwKzqnob4Jw112MUJzyHjAiAmYQGYq7jEePiav6nnMQ==, tarball: file:projects/auth-providers.tgz}
id: file:projects/auth-providers.tgz
name: '@rush-temp/auth-providers'
@@ -22412,7 +22540,7 @@ packages:
koa-passport: 6.0.0
koa-router: 12.0.1
koa-session: 6.4.0
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
openid-client: 5.7.0
passport-custom: 1.1.1
passport-github2: 0.1.12
@@ -22906,12 +23034,13 @@ packages:
dev: false
file:projects/client-resources.tgz(@types/node@20.11.19)(esbuild@0.24.2)(ts-node@10.9.2):
resolution: {integrity: sha512-zqTNrWtf8+kFkWdM9yok0EgXP9QsxCFomNW1HWyBp0arKXDQMjgA6UJ5qviDhYPjFzQzJil5DRax82wZfvVSkw==, tarball: file:projects/client-resources.tgz}
resolution: {integrity: sha512-s2VP7YDaXR6VMITb2/kgWe69RxM1XaTdNHQJfliJfFaV88egZSK6JNmB40FoA/QY0Vvf7SmJH/Ld+bvjZqGpWA==, tarball: file:projects/client-resources.tgz}
id: file:projects/client-resources.tgz
name: '@rush-temp/client-resources'
version: 0.0.0
dependencies:
'@types/jest': 29.5.12
'@types/snappyjs': 0.7.1
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
eslint: 8.56.0
@@ -22921,6 +23050,7 @@ packages:
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
prettier: 3.2.5
snappyjs: 0.7.0
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
typescript: 5.3.3
transitivePeerDependencies:
@@ -23000,51 +23130,15 @@ packages:
- utf-8-validate
dev: false
file:projects/cloud-datalake.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-pOjup2gTfDH4Qq9r4LTeVnf+nDQvlTSUOxLtT+bKnrHM0jQQ705v6wYtmkpWD7CtEwwCvS9CH+iPZwB5ZNpzqQ==, tarball: file:projects/cloud-datalake.tgz}
id: file:projects/cloud-datalake.tgz
name: '@rush-temp/cloud-datalake'
version: 0.0.0
dependencies:
'@cloudflare/workers-types': 4.20241022.0
'@types/jest': 29.5.12
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
aws4fetch: 1.0.20
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
itty-router: 5.0.18
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
postgres: 3.4.5
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.6.2)
typescript: 5.6.2
wrangler: 3.97.0(@cloudflare/workers-types@4.20241022.0)(bufferutil@4.0.8)(utf-8-validate@6.0.4)
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
- '@types/node'
- babel-jest
- babel-plugin-macros
- bufferutil
- esbuild
- node-notifier
- supports-color
- ts-node
- utf-8-validate
dev: false
file:projects/cloud-transactor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(ts-node@10.9.2)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-6kBv/3KfTsOvsDrfVXQvQwtLM72ltx4oPHc3cJ9xtG82XLeQLLpeJmCy6CNDVOtXaqnagwF5TKm/zCw4sc/dFQ==, tarball: file:projects/cloud-transactor.tgz}
resolution: {integrity: sha512-sLn0+hzDxI1PlNcx3XiLzwU9xU+mui2QodGtjpn7oS5NWnGWM42fbnwfVP35akfki8kKOZ3pE5CcMKQ6ffjUzg==, tarball: file:projects/cloud-transactor.tgz}
id: file:projects/cloud-transactor.tgz
name: '@rush-temp/cloud-transactor'
version: 0.0.0
dependencies:
'@cloudflare/workers-types': 4.20241022.0
'@types/jest': 29.5.12
'@types/snappyjs': 0.7.1
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2)
esbuild: 0.24.2
@@ -23056,6 +23150,7 @@ packages:
itty-router: 5.0.18
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
prettier: 3.2.5
snappyjs: 0.7.0
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.6.2)
typescript: 5.6.2
wrangler: 3.97.0(@cloudflare/workers-types@4.20241022.0)(bufferutil@4.0.8)(utf-8-validate@6.0.4)
@@ -23136,7 +23231,7 @@ packages:
- ts-node
dev: false
file:projects/collaborator.tgz(@tiptap/pm@2.6.6)(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-protocols@1.0.6):
file:projects/collaborator.tgz(@tiptap/pm@2.6.6)(bufferutil@4.0.8)(snappy@7.2.2)(utf-8-validate@6.0.4)(y-protocols@1.0.6):
resolution: {integrity: sha512-5wz+fFNfftwF7QP2u/dvfT2DYB7xFgplfhalPFYz8sLmXJ0NPVsXcr7qve637Vw8tjwjf4zzk3MJpfeJRcWLXQ==, tarball: file:projects/collaborator.tgz}
id: file:projects/collaborator.tgz
name: '@rush-temp/collaborator'
@@ -23166,7 +23261,7 @@ packages:
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
express: 4.21.2
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
@@ -26860,7 +26955,7 @@ packages:
- ts-node
dev: false
file:projects/mongo.tgz(esbuild@0.24.2)(ts-node@10.9.2):
file:projects/mongo.tgz(esbuild@0.24.2)(snappy@7.2.2)(ts-node@10.9.2):
resolution: {integrity: sha512-aqU57iG6awiTX9qWSrzXMKyHSbt0V8mKAutQjE3uCx9FdPpnGiZjJflWsMS5PaZ4MdNazv2vUg4XPDHKCc4A/A==, tarball: file:projects/mongo.tgz}
id: file:projects/mongo.tgz
name: '@rush-temp/mongo'
@@ -26877,7 +26972,7 @@ packages:
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
typescript: 5.3.3
@@ -27248,8 +27343,9 @@ packages:
- ts-node
dev: false
file:projects/pod-account.tgz:
file:projects/pod-account.tgz(snappy@7.2.2):
resolution: {integrity: sha512-DCZlxY22Z2TNo08v2hcwz+2yg0HfoH2psDkdlVGlzX5KwZt6knEzAUZeb+MiOb7KpGSQoKp4jJDcydUcC+QsVQ==, tarball: file:projects/pod-account.tgz}
id: file:projects/pod-account.tgz
name: '@rush-temp/pod-account'
version: 0.0.0
dependencies:
@@ -27273,9 +27369,8 @@ packages:
koa: 2.15.3
koa-bodyparser: 4.4.1
koa-router: 12.0.1
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
prettier: 3.2.5
prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11)
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
typescript: 5.3.3
@@ -27297,8 +27392,8 @@ packages:
- supports-color
dev: false
file:projects/pod-ai-bot.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(zod@3.23.8):
resolution: {integrity: sha512-OPo+KhRKsPQhO1eqOIgL30ef9s/TAnEf21bAwzxLLcB70AIwMyjfPRzzJHDOIbVjCzA8t4YFQ5MWWoLUveylFg==, tarball: file:projects/pod-ai-bot.tgz}
file:projects/pod-ai-bot.tgz(bufferutil@4.0.8)(snappy@7.2.2)(utf-8-validate@6.0.4)(zod@3.23.8):
resolution: {integrity: sha512-R2ByQWsnvxxBGk/Ql7z8CK8NRb4XXREC0rndthdpfqw1dhwofuHiHpU2sb7L+OTHiz0do16L5gOLb7R1CbMecw==, tarball: file:projects/pod-ai-bot.tgz}
id: file:projects/pod-ai-bot.tgz
name: '@rush-temp/pod-ai-bot'
version: 0.0.0
@@ -27308,7 +27403,6 @@ packages:
'@types/express': 4.17.21
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/node-fetch': 2.6.11
'@types/uuid': 8.3.4
'@types/ws': 8.5.11
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
@@ -27327,8 +27421,7 @@ packages:
form-data: 4.0.0
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
js-tiktoken: 1.0.14
mongodb: 6.12.0
node-fetch: 2.7.0
mongodb: 6.12.0(snappy@7.2.2)
openai: 4.56.0(zod@3.23.8)
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
@@ -27358,7 +27451,7 @@ packages:
- zod
dev: false
file:projects/pod-analytics-collector.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4):
file:projects/pod-analytics-collector.tgz(bufferutil@4.0.8)(snappy@7.2.2)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-KuE4/ZuLMoEOqY2QyXv7AD0uPt5J7Uwt6jP70iWXcR2oET1+BhD2QTm/9MNsgOsnfiqp0Tdv06pfu7fb2A2Gcw==, tarball: file:projects/pod-analytics-collector.tgz}
id: file:projects/pod-analytics-collector.tgz
name: '@rush-temp/pod-analytics-collector'
@@ -27369,7 +27462,6 @@ packages:
'@types/express': 4.17.21
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/ws': 8.5.11
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
cors: 2.8.5
@@ -27383,13 +27475,12 @@ packages:
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
express: 4.21.2
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
prettier: 3.2.5
puppeteer: 22.14.0(bufferutil@4.0.8)(typescript@5.3.3)(utf-8-validate@6.0.4)
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
typescript: 5.3.3
ws: 8.18.0(bufferutil@4.0.8)(utf-8-validate@6.0.4)
transitivePeerDependencies:
- '@aws-sdk/credential-providers'
- '@babel/core'
@@ -27446,7 +27537,7 @@ packages:
- supports-color
dev: false
file:projects/pod-calendar.tgz(bufferutil@4.0.8)(ts-node@10.9.2)(utf-8-validate@6.0.4):
file:projects/pod-calendar.tgz(bufferutil@4.0.8)(snappy@7.2.2)(ts-node@10.9.2)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-THWVQC4htKDjD5lYArV/wgfiwaIoY+a82QlSzT2zRDoJwCzs0ivWxnl+eeCGMgTwb3YA5sbhu9afg3P1MWjNpQ==, tarball: file:projects/pod-calendar.tgz}
id: file:projects/pod-calendar.tgz
name: '@rush-temp/pod-calendar'
@@ -27476,7 +27567,7 @@ packages:
googleapis: 122.0.0
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
jwt-simple: 0.5.6
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
ts-node-dev: 2.0.0(@types/node@20.11.19)(typescript@5.3.3)
@@ -27625,7 +27716,7 @@ packages:
- supports-color
dev: false
file:projects/pod-github.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-prosemirror@1.2.12):
file:projects/pod-github.tgz(bufferutil@4.0.8)(snappy@7.2.2)(utf-8-validate@6.0.4)(y-prosemirror@1.2.12):
resolution: {integrity: sha512-VO1ipLvo8rWOvJVHMLOW5Xl6JkxWOw7wKEpGWweoJDdrYUIK+od4eVqaGDy2YgyXfQOxmJFTGOZ0l84+PcS3hw==, tarball: file:projects/pod-github.tgz}
id: file:projects/pod-github.tgz
name: '@rush-temp/pod-github'
@@ -27685,7 +27776,7 @@ packages:
graphql-binding: 2.5.2
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
markdown-it: 14.0.0
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
octokit: 3.2.1
prettier: 3.2.5
smee-client: 1.2.5
@@ -27714,7 +27805,7 @@ packages:
- y-prosemirror
dev: false
file:projects/pod-gmail.tgz(ts-node@10.9.2):
file:projects/pod-gmail.tgz(snappy@7.2.2)(ts-node@10.9.2):
resolution: {integrity: sha512-/NcwrUow5fptRQxcxnzc5ZT/wtBBdlWQRT+IglOudqZbVoN/aSwAXaKd4BoPTyGOPW0D0M2SLYFXZlPvhV4BjQ==, tarball: file:projects/pod-gmail.tgz}
id: file:projects/pod-gmail.tgz
name: '@rush-temp/pod-gmail'
@@ -27726,7 +27817,6 @@ packages:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/uuid': 8.3.4
'@types/ws': 8.5.11
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
cors: 2.8.5
@@ -27746,7 +27836,7 @@ packages:
googleapis: 122.0.0
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
jwt-simple: 0.5.6
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
ts-node-dev: 2.0.0(@types/node@20.11.19)(typescript@5.3.3)
@@ -27773,7 +27863,7 @@ packages:
dev: false
file:projects/pod-love.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-Ibi2id+aFeeH3lGo2z5GW3Wg3ZWha/5VTWYxXxC0IQr/ivvk+LFlWShEuCyG8XYvIoy6OVAmafgpfOKeUrQjsg==, tarball: file:projects/pod-love.tgz}
resolution: {integrity: sha512-G70rzmJHk/en1EdQHEs2HEK9/D2IY0i/iF0WZtU2FWLtSzlJb5UC0hrpVfxD/aWcnfmmqcLQu+b2siYrnUFSmQ==, tarball: file:projects/pod-love.tgz}
id: file:projects/pod-love.tgz
name: '@rush-temp/pod-love'
version: 0.0.0
@@ -27865,7 +27955,7 @@ packages:
dev: false
file:projects/pod-server.tgz(utf-8-validate@6.0.4):
resolution: {integrity: sha512-4gQfkwfde44+kJ+YPimPEoDHtgdMRy0+J2xk/Uum9wWjKHCe8tQsQZUJxp6S5BU3DPBMaVsC0h9N0jxHpWK49A==, tarball: file:projects/pod-server.tgz}
resolution: {integrity: sha512-U7wNYw+h8a6hEV6fwXYJFStTk5noW4LAsL1G4JCJjyQ7PVzlmXZqGncFe+bq5EqgYgbOewZ2tR+T8rB8HWi9Cw==, tarball: file:projects/pod-server.tgz}
id: file:projects/pod-server.tgz
name: '@rush-temp/pod-server'
version: 0.0.0
@@ -27888,6 +27978,8 @@ packages:
msgpackr: 1.11.2
msgpackr-extract: 3.0.3
prettier: 3.2.5
snappy: 7.2.2
snappyjs: 0.7.0
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
typescript: 5.3.3
@@ -27946,8 +28038,9 @@ packages:
- supports-color
dev: false
file:projects/pod-sign.tgz:
file:projects/pod-sign.tgz(snappy@7.2.2):
resolution: {integrity: sha512-2CvZ6MihYZ/0DXWllTffUrvNyRjLJvvUkcT36EpVmgydo2F4guxAhrnsyaivxNPNwP8NuSy7JJ4uXXOYNeXCQg==, tarball: file:projects/pod-sign.tgz}
id: file:projects/pod-sign.tgz
name: '@rush-temp/pod-sign'
version: 0.0.0
dependencies:
@@ -27960,7 +28053,6 @@ packages:
'@types/express': 4.17.21
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/ws': 8.5.11
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
cors: 2.8.5
@@ -27974,7 +28066,7 @@ packages:
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
express: 4.21.2
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
node-forge: 1.3.1
pdf-lib: 1.17.1
prettier: 3.2.5
@@ -28039,7 +28131,7 @@ packages:
- supports-color
dev: false
file:projects/pod-telegram-bot.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4):
file:projects/pod-telegram-bot.tgz(bufferutil@4.0.8)(snappy@7.2.2)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-QcNX6QCVKgZ6cV4ZkPBzdPH0Nu1F1sQ7yIMYQ4ikoR/BWWBB2U4E/YWd10I3c34QV3DUbKo3CID8EtkzrXadYQ==, tarball: file:projects/pod-telegram-bot.tgz}
id: file:projects/pod-telegram-bot.tgz
name: '@rush-temp/pod-telegram-bot'
@@ -28051,7 +28143,6 @@ packages:
'@types/express': 4.17.21
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/node-fetch': 2.6.11
'@types/otp-generator': 4.0.2
'@types/ws': 8.5.11
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
@@ -28068,8 +28159,7 @@ packages:
express: 4.21.2
htmlparser2: 9.1.0
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
mongodb: 6.12.0
node-fetch: 2.7.0
mongodb: 6.12.0(snappy@7.2.2)
otp-generator: 4.0.1
prettier: 3.2.5
telegraf: 4.16.3
@@ -28098,7 +28188,7 @@ packages:
- utf-8-validate
dev: false
file:projects/pod-telegram.tgz(bufferutil@4.0.8)(ts-node@10.9.2)(utf-8-validate@6.0.4):
file:projects/pod-telegram.tgz(bufferutil@4.0.8)(snappy@7.2.2)(ts-node@10.9.2)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-FAYbl5GmRCS3fOR5eUXKNS8KmwnwB+7ZZ68vDi5lfeH4hkJky/VdR5yq4dX5SzOB35e7hz9JWriDGxfjUcx6Wg==, tarball: file:projects/pod-telegram.tgz}
id: file:projects/pod-telegram.tgz
name: '@rush-temp/pod-telegram'
@@ -28129,7 +28219,7 @@ packages:
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
jwt-simple: 0.5.6
mime: 3.0.0
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
prettier: 3.2.5
telegram: 2.22.2
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
@@ -28159,8 +28249,9 @@ packages:
- utf-8-validate
dev: false
file:projects/pod-workspace.tgz:
file:projects/pod-workspace.tgz(snappy@7.2.2):
resolution: {integrity: sha512-6wRqrWl0oZUzQ2CZBpPK09doQpYepraNAwCO3OcPDaRnDfQo0WBwLvfWOdDVW3REunCCh/DHMxGpuiLnaQX4KQ==, tarball: file:projects/pod-workspace.tgz}
id: file:projects/pod-workspace.tgz
name: '@rush-temp/pod-workspace'
version: 0.0.0
dependencies:
@@ -28176,7 +28267,7 @@ packages:
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
@@ -31256,7 +31347,7 @@ packages:
- ts-node
dev: false
file:projects/server-tool.tgz(@types/node@20.11.19)(esbuild@0.24.2)(ts-node@10.9.2):
file:projects/server-tool.tgz(@types/node@20.11.19)(esbuild@0.24.2)(snappy@7.2.2)(ts-node@10.9.2):
resolution: {integrity: sha512-G/gF05AF5jsncB4bMGt9HXus2GLzKjAwYPYPlanY7M4tavCE8gOstoBPgW8I/ETxaRI3oztLV7G5UoCHEGON3g==, tarball: file:projects/server-tool.tgz}
id: file:projects/server-tool.tgz
name: '@rush-temp/server-tool'
@@ -31276,7 +31367,7 @@ packages:
fast-equals: 5.0.1
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
js-yaml: 4.1.0
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
prettier: 3.2.5
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
typescript: 5.3.3
@@ -31483,7 +31574,7 @@ packages:
dev: false
file:projects/server-ws.tgz(esbuild@0.24.2)(ts-node@10.9.2):
resolution: {integrity: sha512-Uw0sIkrmlc+pFs1pavpPYPTUS/SvzBIzXygFUw9PGViypnUo54AYq9QLGEXs1dg2HU8TNcPbgtdRx+4S8fTUMA==, tarball: file:projects/server-ws.tgz}
resolution: {integrity: sha512-9PP7kE8LXoZ+7r6OwLp26KT+JFAQtSwg5QP/awwXjcsV2t3yphDYKbdAJuVG5xlLfHW52Xn4NRbr0wFhfwS0sg==, tarball: file:projects/server-ws.tgz}
id: file:projects/server-ws.tgz
name: '@rush-temp/server-ws'
version: 0.0.0
@@ -31508,6 +31599,7 @@ packages:
express: 4.21.2
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
prettier: 3.2.5
snappy: 7.2.2
ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.3.3)
typescript: 5.3.3
utf-8-validate: 6.0.4
@@ -32686,7 +32778,7 @@ packages:
dev: false
file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2)(utf-8-validate@6.0.4):
resolution: {integrity: sha512-zmM6GDzeNuMbMA7iWwhg8hBc43fylNW7cEmk0fSWw9+G9NqD2MLv2dVb5dnAcPAioj9WBdDakWcrLAAHUBJyMw==, tarball: file:projects/text.tgz}
resolution: {integrity: sha512-p2TRL6FgtqVBooDf+E5A4HB+45VS03IvNxudhC9gCotewS726ZLOxl2XdfeuYB2rHYbzdmmiG2zb8GbPwqTkOw==, tarball: file:projects/text.tgz}
id: file:projects/text.tgz
name: '@rush-temp/text'
version: 0.0.0
@@ -32707,6 +32799,7 @@ packages:
'@tiptap/extension-task-item': 2.6.6(@tiptap/core@2.6.6)(@tiptap/pm@2.6.6)
'@tiptap/extension-task-list': 2.6.6(@tiptap/core@2.6.6)
'@tiptap/extension-text-align': 2.11.0(@tiptap/core@2.6.6)
'@tiptap/extension-text-style': 2.11.0(@tiptap/core@2.6.6)
'@tiptap/extension-typography': 2.6.6(@tiptap/core@2.6.6)
'@tiptap/extension-underline': 2.6.6(@tiptap/core@2.6.6)
'@tiptap/html': 2.6.6(@tiptap/core@2.6.6)(@tiptap/pm@2.6.6)
@@ -32904,8 +32997,9 @@ packages:
- ts-node
dev: false
file:projects/tool.tgz:
file:projects/tool.tgz(snappy@7.2.2):
resolution: {integrity: sha512-Cw36G6uROkPkNIJDl7WeHjhHKmbQth6wmjMu4vBQGz+DfuPf9Eg63SFKq29rOssZ4LlJB1d22Y7XnxilCuXK5g==, tarball: file:projects/tool.tgz}
id: file:projects/tool.tgz
name: '@rush-temp/tool'
version: 0.0.0
dependencies:
@@ -32930,11 +33024,10 @@ packages:
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
fast-equals: 5.0.1
got: 11.8.6
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
libphonenumber-js: 1.10.56
mime-types: 2.1.35
mongodb: 6.12.0
mongodb: 6.12.0(snappy@7.2.2)
msgpackr: 1.11.2
msgpackr-extract: 3.0.3
postgres: 3.4.5
common/scripts/esbuild.js
+1 -1
View File
@@ -13,7 +13,7 @@ const defaultConfig = {
keepNames: false,
sourcemap: false,
logLevel: 'error',
external: [],
external: ['snappy'],
define: {},
};
dev/tool/package.json
+1 -1
View File
@@ -14,7 +14,7 @@
"_phase:bundle": "rushx bundle",
"_phase:docker-build": "rushx docker:build",
"_phase:docker-staging": "rushx docker:staging",
"bundle": "node ../../common/scripts/esbuild.js --entry=src/__start.ts --keep-names=true --sourcemap=external --define:MODEL_VERSION --define:GIT_REVISION",
"bundle": "node ../../common/scripts/esbuild.js --entry=src/__start.ts --keep-names=true --external=snappy --sourcemap=external --define:MODEL_VERSION --define:GIT_REVISION",
"docker:build": "../../common/scripts/docker_build.sh hardcoreeng/tool",
"docker:tbuild": "docker build -t hardcoreeng/tool . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool",
"docker:abuild": "docker build -t hardcoreeng/tool . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool",
models/text-editor/src/index.ts
+27 -2
View File
@@ -247,6 +247,18 @@ export function createModel (builder: Builder): void {
index: 20
})
builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, {
action: textEditor.function.SetTextColor,
icon: textEditor.icon.TextStyle,
visibilityTester: textEditor.function.IsTextStylingEnabled,
isActive: {
name: 'textStyle'
},
label: textEditor.string.SetTextColor,
category: 20,
index: 25
})
// Link category
builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, {
action: textEditor.function.FormatLink,
@@ -338,23 +350,36 @@ export function createModel (builder: Builder): void {
index: 10
})
// Table cell category
builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, {
kind: 'table',
action: textEditor.function.SetBackgroundColor,
icon: textEditor.icon.Brush,
visibilityTester: textEditor.function.IsTableToolbarContext,
label: textEditor.string.SetCellHighlightColor,
category: 65,
index: 5
})
// Table category
builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, {
kind: 'table',
action: textEditor.function.SelectTable,
icon: textEditor.icon.SelectTable,
visibilityTester: textEditor.function.IsTableToolbarContext,
label: textEditor.string.SelectTable,
category: 70,
index: 5
index: 15
})
builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, {
kind: 'table',
action: textEditor.function.OpenTableOptions,
icon: textEditor.icon.TableProps,
visibilityTester: textEditor.function.IsTableToolbarContext,
label: textEditor.string.TableOptions,
category: 70,
index: 10
index: 20
})
// Image align category
models/text-editor/src/plugin.ts
+5 -1
View File
@@ -36,9 +36,13 @@ export default mergeIds(textEditorId, textEditor, {
IsTableToolbarContext: '' as Resource<TextActionVisibleFunction>,
IsEditableNote: '' as Resource<TextActionVisibleFunction>,
IsEditable: '' as Resource<TextActionVisibleFunction>,
IsTextStylingEnabled: '' as Resource<TextActionVisibleFunction>,
IsHeadingVisible: '' as Resource<TextActionVisibleFunction>,
CreateInlineComment: '' as Resource<TextActionFunction>,
ShouldShowCreateInlineCommentAction: '' as Resource<TextActionVisibleFunction>
ShouldShowCreateInlineCommentAction: '' as Resource<TextActionVisibleFunction>,
SetBackgroundColor: '' as Resource<TextActionFunction>,
SetTextColor: '' as Resource<TextActionFunction>
}
})
models/tracker/src/plugin.ts
+1 -2
View File
@@ -98,8 +98,7 @@ export default mergeIds(trackerId, tracker, {
EditProject: '' as ViewAction,
DeleteProject: '' as ViewAction,
DeleteIssue: '' as ViewAction,
DeleteMilestone: '' as ViewAction,
ImportIssues: '' as ViewAction
DeleteMilestone: '' as ViewAction
},
action: {
NewRelatedIssue: '' as Ref<Action<Doc, any>>,
packages/core/src/__tests__/limits.test.ts
+118
View File
@@ -0,0 +1,118 @@
//
// Copyright © 2023 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { TimeRateLimiter } from '../utils'
describe('TimeRateLimiter', () => {
it('should limit rate of executions', async () => {
jest.useFakeTimers()
const limiter = new TimeRateLimiter(2, 1000) // 2 executions per second
const mockFn = jest.fn().mockResolvedValue('result')
const operations: Promise<string>[] = []
// Try to execute 4 operations
for (let i = 0; i < 4; i++) {
operations.push(limiter.exec(mockFn))
}
// First 2 should execute immediately
expect(mockFn).toHaveBeenCalledTimes(2)
// Advance time by 1 second
jest.advanceTimersByTime(1001)
await Promise.resolve()
// Next 2 should execute after time advance
expect(mockFn).toHaveBeenCalledTimes(4)
await Promise.all(operations)
})
it('should cleanup old executions', async () => {
jest.useFakeTimers()
const limiter = new TimeRateLimiter(2, 1000)
const mockFn = jest.fn().mockResolvedValue('result')
// Execute first operation
await limiter.exec(mockFn)
expect(mockFn).toHaveBeenCalledTimes(1)
expect(limiter.executions.length).toBe(1)
// Advance time past period
jest.advanceTimersByTime(1001)
// Execute another operation
await limiter.exec(mockFn)
expect(mockFn).toHaveBeenCalledTimes(2)
expect(limiter.executions.length).toBe(1) // Old execution should be cleaned up
})
it('should handle concurrent operations', async () => {
jest.useFakeTimers()
const limiter = new TimeRateLimiter(2, 1000)
const mockFn = jest.fn().mockImplementation(async () => {
console.log('start#')
await new Promise((resolve) => setTimeout(resolve, 450))
console.log('finished#')
return 'result'
})
const operations = Promise.all([limiter.exec(mockFn), limiter.exec(mockFn), limiter.exec(mockFn)])
expect(mockFn).toHaveBeenCalledTimes(2)
expect(limiter.processingQueue.size).toBe(2)
jest.advanceTimersByTime(500)
await Promise.resolve()
await Promise.resolve()
jest.advanceTimersByTime(1000)
await Promise.resolve()
jest.advanceTimersByTime(2001)
await Promise.resolve()
await Promise.resolve()
expect(limiter.processingQueue.size).toBe(0)
expect(mockFn).toHaveBeenCalledTimes(3)
await operations
})
it('should wait for processing to complete', async () => {
jest.useFakeTimers()
const limiter = new TimeRateLimiter(2, 1000)
const mockFn = jest.fn().mockImplementation(async () => {
await new Promise((resolve) => setTimeout(resolve, 500))
return 'result'
})
const operation = limiter.exec(mockFn)
const waitPromise = limiter.waitProcessing().then(() => {
console.log('wait complete')
})
expect(limiter.processingQueue.size).toBe(1)
jest.advanceTimersByTime(1001)
await Promise.resolve()
await Promise.resolve()
await Promise.resolve()
await waitPromise
await operation
expect(limiter.processingQueue.size).toBe(0)
})
})
packages/core/src/utils.ts
+65
View File
@@ -839,3 +839,68 @@ export function pluginFilterTx (
systemTx = systemTx.filter((t) => !totalExcluded.has(t._id))
return systemTx
}
/**
* @public
*/
export class TimeRateLimiter {
idCounter: number = 0
processingQueue = new Map<number, Promise<void>>()
last: number = 0
rate: number
period: number
executions: { time: number, running: boolean }[] = []
queue: (() => Promise<void>)[] = []
notify: (() => void)[] = []
constructor (rate: number, period: number = 1000) {
this.rate = rate
this.period = period
}
private cleanupExecutions (): void {
const now = Date.now()
this.executions = this.executions.filter((time) => time.running || now - time.time < this.period)
}
async exec<T, B extends Record<string, any> = any>(op: (args?: B) => Promise<T>, args?: B): Promise<T> {
const processingId = this.idCounter++
while (this.processingQueue.size >= this.rate || this.executions.length >= this.rate) {
this.cleanupExecutions()
if (this.executions.length < this.rate) {
break
}
await new Promise<void>((resolve) => {
setTimeout(resolve, this.period / this.rate)
})
}
const v = { time: Date.now(), running: true }
try {
this.executions.push(v)
const p = op(args)
this.processingQueue.set(processingId, p as Promise<void>)
return await p
} finally {
v.running = false
this.processingQueue.delete(processingId)
this.cleanupExecutions()
const n = this.notify.shift()
if (n !== undefined) {
n()
}
}
}
async waitProcessing (): Promise<void> {
while (this.processingQueue.size > 0) {
console.log('wait', this.processingQueue.size)
await new Promise<void>((resolve) => {
this.notify.push(resolve)
})
}
}
}
workers/datalake/config/rig.json → packages/importer/config/rig.json
+1 -2
View File
@@ -1,5 +1,4 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig",
"rigProfile": "node"
"rigPackageName": "@hcengineering/platform-rig"
}
packages/text/package.json
+2 -1
View File
@@ -64,7 +64,8 @@
"prosemirror-codemark": "^0.4.2",
"markdown-it": "^14.0.0",
"fast-equals": "^5.0.1",
"@tiptap/extension-text-align": "~2.11.0"
"@tiptap/extension-text-align": "~2.11.0",
"@tiptap/extension-text-style": "~2.11.0"
},
"repository": "https://github.com/hcengineering/platform",
"publishConfig": {
packages/text/src/index.ts
+3
View File
@@ -22,6 +22,7 @@ export * from './markup/utils'
export * from './nodes'
// export * from './ydoc'
export * from './marks/code'
export * from './marks/colors'
export * from './marks/noteBase'
export * from './markdown'
export * from './markdown/serializer'
@@ -31,3 +32,5 @@ export * from './markdown/node'
export * from './kits/default-kit'
export * from './kits/server-kit'
export { TextStyle, type TextStyleOptions } from '@tiptap/extension-text-style'
packages/text/src/kits/server-kit.ts
+6 -1
View File
@@ -34,6 +34,8 @@ import { CodeExtension, codeOptions } from '../marks/code'
import { NoteBaseExtension } from '../marks/noteBase'
import { CommentNode } from '../nodes/comment'
import TextAlign from '@tiptap/extension-text-align'
import TextStyle from '@tiptap/extension-text-style'
import { BackgroundColor, TextColor } from '../marks/colors'
const headingLevels: Level[] = [1, 2, 3, 4, 5, 6]
@@ -97,7 +99,10 @@ export const ServerKit = Extension.create<ServerKitOptions>({
ReferenceNode,
CommentNode,
NodeUuid,
NoteBaseExtension
NoteBaseExtension,
TextStyle.configure({}),
TextColor.configure({}),
BackgroundColor.configure({ types: ['tableCell'] })
]
}
})
packages/text/src/marks/colors.ts
+142
View File
@@ -0,0 +1,142 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Extension } from '@tiptap/core'
import '@tiptap/extension-text-style'
export interface BackgroundColorOptions {
types: string[]
}
declare module '@tiptap/core' {
interface Commands<ReturnType> {
colors: {
setTextColor: (color: string) => ReturnType
unsetTextColor: () => ReturnType
setBackgroundColor: (color: string) => ReturnType
unsetBackgroundColor: () => ReturnType
}
}
}
export const BackgroundColor = Extension.create<BackgroundColorOptions>({
name: 'backgroundColor',
addOptions () {
return {
types: []
}
},
addGlobalAttributes () {
return [
{
types: this.options.types,
attributes: {
backgroundColor: {
parseHTML: (element) => {
return element.getAttribute('data-background-color') ?? undefined
},
renderHTML: (attributes) => {
if (typeof attributes.backgroundColor !== 'string') {
return {}
}
return {
'data-background-color': attributes.backgroundColor,
style: `background-color: ${attributes.backgroundColor}`
}
}
}
}
}
]
},
addCommands () {
return {
setBackgroundColor:
(backgroundColor: string) =>
({ commands }) => {
return this.options.types
.map((type) => commands.updateAttributes(type, { backgroundColor }))
.every((response) => response)
},
unsetBackgroundColor:
() =>
({ commands }) => {
return this.options.types
.map((type) => commands.resetAttributes(type, 'backgroundColor'))
.every((response) => response)
}
}
}
})
export interface TextColorOptions {
types: string[]
}
export const TextColor = Extension.create<TextColorOptions>({
name: 'textColor',
addOptions () {
return {
types: ['textStyle']
}
},
addGlobalAttributes () {
return [
{
types: this.options.types,
attributes: {
color: {
parseHTML: (element) => {
return element.getAttribute('data-color') ?? undefined
},
renderHTML: (attributes) => {
if (typeof attributes.color !== 'string') {
return {}
}
return {
'data-color': attributes.color,
style: `color: ${attributes.color}`
}
}
}
}
}
]
},
addCommands () {
return {
setTextColor:
(color: string) =>
({ chain }) => {
return chain().setMark('textStyle', { color }).run()
},
unsetTextColor:
() =>
({ chain }) => {
return chain().setMark('textStyle', { color: null }).removeEmptyTextStyle().run()
}
}
}
})
packages/theme/styles/_colors.scss
+45
View File
@@ -65,6 +65,7 @@
--highlight-red-hover: #ff967e;
--highlight-red-press: #f96f50bd;
--text-editor-selected-node-background: rgba(43, 81, 144, 0.1);
--text-editor-selected-node-color: #93CAF3;
--text-editor-highlighted-node-warning-active-background-color: #F2D7AE;
@@ -248,6 +249,28 @@
--theme-text-editor-note-anchor-bg-primary: #688797;
--theme-text-editor-note-anchor-bg-primary-light: #747C81;
--text-editor-table-border-color: hsl(220, 6%, 40%);
--theme-text-editor-palette-text-gray: rgba(155, 155, 155, 1);
--theme-text-editor-palette-text-brown: rgba(186, 133, 111, 1);
--theme-text-editor-palette-text-orange: rgba(199, 125, 72, 1);
--theme-text-editor-palette-text-yellow: rgba(202, 152, 73, 1);
--theme-text-editor-palette-text-green: rgba(82, 158, 114, 1);
--theme-text-editor-palette-text-blue: rgba(94, 135, 201, 1);
--theme-text-editor-palette-text-purple: rgba(157, 104, 211, 1);
--theme-text-editor-palette-text-pink: rgba(209, 87, 150, 1);
--theme-text-editor-palette-text-red: rgba(223, 84, 82, 1);
--theme-text-editor-palette-bg-gray: rgba(47, 47, 47, 1);
--theme-text-editor-palette-bg-brown: rgba(74, 50, 40, 1);
--theme-text-editor-palette-bg-orange: rgba(92, 59, 35, 1);
--theme-text-editor-palette-bg-yellow: rgba(86, 67, 40, 1);
--theme-text-editor-palette-bg-green: rgba(36, 61, 48, 1);
--theme-text-editor-palette-bg-blue: rgba(20, 58, 78, 1);
--theme-text-editor-palette-bg-purple: rgba(60, 45, 73, 1);
--theme-text-editor-palette-bg-pink: rgba(78, 44, 60, 1);
--theme-text-editor-palette-bg-red:rgba(82, 46, 42, 1);
--accent-bg-color: #27282b;
--accent-shadow: rgb(0 0 0 / 10%) 0px 2px 4px;
@@ -507,6 +530,28 @@
--theme-text-editor-note-anchor-bg-primary: #AAC5E9;
--theme-text-editor-note-anchor-bg-primary-light: #D5E5F5;
--text-editor-table-border-color: #c9cbcd;
--theme-text-editor-palette-text-gray: rgba(120, 119, 116, 1);
--theme-text-editor-palette-text-brown: rgba(159, 107, 83, 1);
--theme-text-editor-palette-text-orange: rgba(217, 115, 13, 1);
--theme-text-editor-palette-text-yellow: rgba(203, 145, 47, 1);
--theme-text-editor-palette-text-green: rgba(68, 131, 97, 1);
--theme-text-editor-palette-text-blue: rgba(51, 126, 169, 1);
--theme-text-editor-palette-text-purple: rgba(144, 101, 176, 1);
--theme-text-editor-palette-text-pink: rgba(193, 76, 138, 1);
--theme-text-editor-palette-text-red: rgba(212, 76, 71, 1);
--theme-text-editor-palette-bg-gray: rgba(241, 241, 239, 1);
--theme-text-editor-palette-bg-brown: rgba(244, 238, 238, 1);
--theme-text-editor-palette-bg-orange: rgba(251, 236, 221, 1);
--theme-text-editor-palette-bg-yellow: rgba(251, 243, 219, 1);
--theme-text-editor-palette-bg-green: rgba(237, 243, 236, 1);
--theme-text-editor-palette-bg-blue: rgba(231, 243, 248, 1);
--theme-text-editor-palette-bg-purple: rgba(244, 240, 247, 0.8);
--theme-text-editor-palette-bg-pink: rgba(249, 238, 243, 0.8);
--theme-text-editor-palette-bg-red: rgba(253, 235, 236, 1);
--accent-bg-color: #eff0f2; // HZ
--accent-shadow: rgb(0 0 0 / 10%) 0px 2px 4px; // Dark
packages/theme/styles/button.scss
+1 -2
View File
@@ -196,8 +196,7 @@
text-overflow: ellipsis;
overflow: hidden;
min-width: 0;
font-weight: 700;
color: var(--theme-caption-color);
font-weight: 500;
}
&-icons { gap: var(--spacing-0_25); }
&:hover {
packages/theme/styles/prose.scss
+7 -1
View File
@@ -38,7 +38,7 @@ table.proseTable {
th {
min-width: 1rem;
height: 2rem;
border: 1px solid var(--button-border-hover);
border: 1px solid var(--text-editor-table-border-color);
padding: .25rem .5rem;
vertical-align: top;
box-sizing: border-box;
@@ -390,6 +390,12 @@ table.proseTable {
}
}
.table-node-selected {
.proseTable {
background-color: var(--text-editor-selected-node-background);
}
}
.proseBlockQuote {
margin-inline: 1px 0;
padding-left: 1.5em;
plugins/client-resources/package.json
+4 -2
View File
@@ -34,14 +34,16 @@
"typescript": "^5.3.3",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5"
"@types/jest": "^29.5.5",
"@types/snappyjs": "^0.7.1"
},
"dependencies": {
"@hcengineering/analytics": "^0.6.0",
"@hcengineering/platform": "^0.6.11",
"@hcengineering/core": "^0.6.32",
"@hcengineering/client": "^0.6.18",
"@hcengineering/rpc": "^0.6.5"
"@hcengineering/rpc": "^0.6.5",
"snappyjs": "^0.7.0"
},
"repository": "https://github.com/hcengineering/platform",
"publishConfig": {
plugins/client-resources/src/connection.ts
+28 -10
View File
@@ -57,6 +57,7 @@ import platform, {
broadcastEvent,
getMetadata
} from '@hcengineering/platform'
import { uncompress } from 'snappyjs'
import { HelloRequest, HelloResponse, RPCHandler, ReqId, type Response } from '@hcengineering/rpc'
@@ -89,6 +90,7 @@ class RequestPromise {
class Connection implements ClientConnection {
private websocket: ClientSocket | null = null
binaryMode = false
compressionMode = false
private readonly requests = new Map<ReqId, RequestPromise>()
private lastId = 0
private interval: number | undefined
@@ -107,7 +109,7 @@ class Connection implements ClientConnection {
private pingResponse: number = Date.now()
private helloRecieved: boolean = false
private helloReceived: boolean = false
private account: Account | undefined
@@ -205,7 +207,7 @@ class Connection implements ClientConnection {
}
isConnected (): boolean {
return this.websocket != null && this.websocket.readyState === ClientSocketReadyState.OPEN && this.helloRecieved
return this.websocket != null && this.websocket.readyState === ClientSocketReadyState.OPEN && this.helloReceived
}
delay = 0
@@ -289,9 +291,8 @@ class Connection implements ClientConnection {
}
if (resp.result === 'hello') {
const helloResp = resp as HelloResponse
if (helloResp.binary) {
this.binaryMode = true
}
this.binaryMode = helloResp.binary
this.compressionMode = helloResp.useCompression ?? false
// We need to clear dial timer, since we recieve hello response.
clearTimeout(this.dialTimer)
@@ -307,7 +308,7 @@ class Connection implements ClientConnection {
return
}
this.account = helloResp.account
this.helloRecieved = true
this.helloReceived = true
if (this.upgrading) {
// We need to call upgrade since connection is upgraded
this.opt?.onUpgrade?.()
@@ -439,6 +440,7 @@ class Connection implements ClientConnection {
private openConnection (ctx: MeasureContext, socketId: number): void {
this.binaryMode = false
this.helloReceived = false
// Use defined factory or browser default one.
const clientSocketFactory =
this.opt?.socketFactory ??
@@ -503,11 +505,28 @@ class Connection implements ClientConnection {
}
if (event.data instanceof Blob) {
void event.data.arrayBuffer().then((data) => {
if (this.compressionMode && this.helloReceived) {
try {
data = uncompress(data)
} catch (err: any) {
// Ignore
console.error(err)
}
}
const resp = this.rpcHandler.readResponse<any>(data, this.binaryMode)
this.handleMsg(socketId, resp)
})
} else {
const resp = this.rpcHandler.readResponse<any>(event.data, this.binaryMode)
let data = event.data
if (this.compressionMode && this.helloReceived) {
try {
data = uncompress(data)
} catch (err: any) {
// Ignore
console.error(err)
}
}
const resp = this.rpcHandler.readResponse<any>(data, this.binaryMode)
this.handleMsg(socketId, resp)
}
}
@@ -525,15 +544,14 @@ class Connection implements ClientConnection {
return
}
const useBinary = this.opt?.useBinaryProtocol ?? getMetadata(client.metadata.UseBinaryProtocol) ?? true
const useCompression =
this.compressionMode =
this.opt?.useProtocolCompression ?? getMetadata(client.metadata.UseProtocolCompression) ?? false
this.helloRecieved = false
const helloRequest: HelloRequest = {
method: 'hello',
params: [],
id: -1,
binary: useBinary,
compression: useCompression
compression: this.compressionMode
}
ctx.withSync('send-hello', {}, () => this.websocket?.send(this.rpcHandler.serialize(helloRequest, false)))
}
plugins/text-editor-assets/assets/icons.svg
+7
View File
@@ -214,4 +214,11 @@
<path d="M3 6C3 4.34315 4.34315 3 6 3H17C18.6569 3 20 4.34315 20 6V10.5556V17C20 18.6569 18.6569 20 17 20H10.5556H6C4.34314 20 3 18.6569 3 17V6Z" stroke="currentColor" stroke-width="2"/>
<path d="M12 15C12 13.3431 13.3431 12 15 12H26C27.6569 12 29 13.3431 29 15V26C29 27.6569 27.6569 29 26 29H15C13.3431 29 12 27.6569 12 26V15Z" stroke="currentColor" stroke-width="2"/>
</symbol>
<symbol id="brush" viewBox="0 0 14 14" fill="none">
<path fill-rule="evenodd" clip-rule="evenodd" d="M11.9142 1.08579L13.4142 2.58579C14.1952 3.36683 14.1952 4.63316 13.4142 5.41421L6.91416 11.9142C6.52365 12.3047 6.01183 12.5 5.5 12.5H4L2.5 13.5L0.5 12.5L2 10.5V9C2 8.48816 2.19526 7.97631 2.58579 7.58579L9.08579 1.08579C9.86684 0.304738 11.1332 0.304738 11.9142 1.08579ZM6.20705 11.2071C6.01161 11.4025 5.75745 11.5 5.5 11.5H4H3V9C3 8.74253 3.09744 8.48835 3.29289 8.29289L4.43936 7.14643L7.35354 10.0606L6.20705 11.2071ZM8.06065 9.35351L5.14646 6.43932L9.79289 1.79289C10.1834 1.40237 10.8166 1.40238 11.2071 1.79289L12.707 3.29289C13.0976 3.68342 13.0976 4.31658 12.707 4.70711L8.06065 9.35351Z" fill="currentColor"/>
</symbol>
<symbol id="textStyle" viewBox="0 0 16 16" fill="none">
<path d="M13 12H3V14H13V12Z" fill="currentColor"/>
<path d="M11 10.5H12L8.69037 2.46234C8.57518 2.18258 8.30254 2 8 2C7.69746 2 7.42482 2.18258 7.30963 2.46234L4 10.5H5L5.805 8.5H10.175L11 10.5ZM6.215 7.5L7.935 3.315H8.065L9.77 7.5H6.215Z" fill="currentColor"/>
</symbol>
</svg>
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 23 KiB

After

Width:  |  Height:  |  Size: 24 KiB

plugins/text-editor-assets/lang/en.json
+3 -1
View File
@@ -66,6 +66,8 @@
"MermaidDiargram": "Diagram",
"Comment": "Comment",
"AddComment": "Add a comment",
"AddCommentPlaceholder": "Add a comment..."
"AddCommentPlaceholder": "Add a comment...",
"SetCellHighlightColor": "Set cell color",
"SetTextColor": "Set text color"
}
}
plugins/text-editor-assets/lang/ru.json
+3 -1
View File
@@ -66,6 +66,8 @@
"MermaidDiargram": "Диаграмма",
"Comment": "Комментарий",
"AddComment": "Добавить комментарий",
"AddCommentPlaceholder": "Добавьте комментарий..."
"AddCommentPlaceholder": "Добавьте комментарий...",
"SetCellHighlightColor": "Изменить цвет ячеек",
"SetTextColor": "Изменить цвет текста"
}
}
plugins/text-editor-assets/src/index.ts
+3 -1
View File
@@ -43,5 +43,7 @@ loadMetadata(textEditor.icon, {
Comment: `${icons}#comment`,
SelectTable: `${icons}#move`,
MergeCells: `${icons}#union`,
SplitCells: `${icons}#divide`
SplitCells: `${icons}#divide`,
Brush: `${icons}#brush`,
TextStyle: `${icons}#textStyle`
})
plugins/text-editor-resources/src/components/extension/colors.ts
+100
View File
@@ -0,0 +1,100 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { getEventPositionElement, showPopup } from '@hcengineering/ui'
import { type Editor } from '@tiptap/core'
import ColorPicker from './popups/ColorPicker.svelte'
import { type ActionContext } from '@hcengineering/text-editor'
export interface BackgroundColorOptions {
types: string[]
}
interface ColorSpec {
color: string
preview?: string
}
function colorVar (tag: string, prefix = 'text'): string {
return `var(--theme-text-editor-palette-${prefix}-${tag})`
}
function colorSpec (tag: string, prefix = 'text'): ColorSpec {
const color = colorVar(tag, prefix)
return { color, preview: colorVar(tag) }
}
const palette = {
background: [
{ color: 'transparent' },
colorSpec('gray', 'bg'),
colorSpec('brown', 'bg'),
colorSpec('orange', 'bg'),
colorSpec('yellow', 'bg'),
colorSpec('green', 'bg'),
colorSpec('blue', 'bg'),
colorSpec('purple', 'bg'),
colorSpec('pink', 'bg'),
colorSpec('red', 'bg')
],
text: [
{ color: 'var(--theme-text-color-primary)' },
colorSpec('gray'),
colorSpec('brown'),
colorSpec('orange'),
colorSpec('yellow'),
colorSpec('green'),
colorSpec('blue'),
colorSpec('purple'),
colorSpec('pink'),
colorSpec('red')
]
}
export async function openBackgroundColorOptions (editor: Editor, event: MouseEvent): Promise<void> {
await new Promise<void>((resolve) => {
showPopup(ColorPicker, { palette: palette.background }, getEventPositionElement(event), (val) => {
const color: string | undefined = val?.color
if (color === undefined) return
if (color === 'transparent') {
editor.commands.unsetBackgroundColor()
} else {
editor.commands.setBackgroundColor(color)
}
resolve()
})
})
}
export async function openTextColorOptions (editor: Editor, event: MouseEvent): Promise<void> {
await new Promise<void>((resolve) => {
showPopup(ColorPicker, { palette: palette.text }, getEventPositionElement(event), (val) => {
const color: string | undefined = val?.color
if (color === undefined) return
if (color === 'var(--theme-text-color-primary)') {
editor.commands.unsetTextColor()
} else {
editor.commands.setTextColor(color)
}
resolve()
})
})
}
export async function isTextStylingEnabled (editor: Editor, context: ActionContext): Promise<boolean> {
return editor.isEditable && editor.commands.setTextColor !== undefined
}
plugins/text-editor-resources/src/components/extension/popups/ColorPicker.svelte
+65
View File
@@ -0,0 +1,65 @@
<!--
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
-->
<script lang="ts">
import { createEventDispatcher } from 'svelte'
import { Card } from '@hcengineering/presentation'
export let palette: Array<{ color: string, preview?: string }> = [{ color: 'transparent' }]
const dispatch = createEventDispatcher()
function handleSubmit (color: { color: string }): void {
dispatch('close', color)
}
</script>
<div class="picker">
<div class="palette">
{#each palette as k}
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<div
class="colorBox"
style:background-color={k.preview ?? k.color}
on:click={() => {
handleSubmit(k)
}}
/>
{/each}
</div>
</div>
<style lang="scss">
.picker {
background: var(--theme-popup-color);
border-radius: 0.5rem;
box-shadow: var(--theme-popup-shadow);
padding: 0.5rem;
margin-left: -4rem;
}
.palette {
display: flex;
gap: 0.25rem;
}
.colorBox {
position: relative;
width: 1.5rem;
height: 1.5rem;
border: 1px solid var(--theme-button-border);
cursor: pointer;
}
</style>
plugins/text-editor-resources/src/components/extension/table/TableNodeView.svelte
+1 -1
View File
@@ -200,7 +200,7 @@
.table-toolbar-container {
position: absolute;
top: -1.5rem;
right: 0;
right: var(--table-offscreen-spacing);
z-index: 200;
}
</style>
plugins/text-editor-resources/src/components/extension/table/TableToolbar.svelte
+23 -4
View File
@@ -50,15 +50,34 @@
actions = out
}
$: actionsQuery.query(textEditor.class.TextEditorAction, { category: 70 }, (result) => {
$: actionsQuery.query(textEditor.class.TextEditorAction, { kind: 'table' }, (result) => {
void updateActions([...result], actionCtx)
})
$: categories = actions.reduce<[number, TextEditorAction][][]>((acc, action) => {
const { category, index } = action
if (acc[category] === undefined) acc[category] = []
acc[category].push([index, action])
return acc
}, [])
$: categories.forEach((category) => {
category.sort((a, b) => a[0] - b[0])
})
</script>
<div class="table-toolbar flex" contenteditable="false">
{#each actions as action}
<TextActionButton {action} {editor} size="small" {actionCtx} blockMouseEvents={false} />
{/each}
<div class="text-editor-toolbar buttons-group xsmall-gap">
{#each Object.values(categories) as category, index}
{#if index > 0}
<div class="buttons-divider" />
{/if}
{#each category as [_, action]}
<TextActionButton {action} {editor} size="small" {actionCtx} blockMouseEvents={false} />
{/each}
{/each}
</div>
</div>
<style lang="scss">
plugins/text-editor-resources/src/components/extension/table/table.ts
+46 -3
View File
@@ -29,23 +29,66 @@ import AddRowBefore from '../../icons/table/AddRowBefore.svelte'
import DeleteCol from '../../icons/table/DeleteCol.svelte'
import DeleteRow from '../../icons/table/DeleteRow.svelte'
import DeleteTable from '../../icons/table/DeleteTable.svelte'
import { Plugin } from '@tiptap/pm/state'
import { TableSelection } from './types'
import { Decoration, DecorationSet } from '@tiptap/pm/view'
export const Table = TiptapTable.extend({
draggable: true,
addKeyboardShortcuts () {
return {
'Mod-Backspace': () => handleDelete(this.editor),
'Mod-Delete': () => handleDelete(this.editor)
Backspace: () => handleDelete(this.editor),
Delete: () => handleDelete(this.editor),
'Mod-Backspace': () => handleModDelete(this.editor),
'Mod-Delete': () => handleModDelete(this.editor)
}
},
addNodeView () {
return SvelteNodeViewRenderer(TableNodeView, {})
},
addProseMirrorPlugins () {
return [...(this.parent?.() ?? []), tableSelectionHighlight()]
}
})
function handleDelete (editor: Editor): boolean {
const { selection } = editor.state
const { selection } = editor.state.tr
if (selection instanceof TableSelection && isTableSelected(selection)) {
return editor.commands.deleteTable()
}
return false
}
export const tableSelectionHighlight = (): Plugin<DecorationSet> => {
return new Plugin<DecorationSet>({
props: {
decorations (state) {
return this.getState(state)
}
},
state: {
init: () => {
return DecorationSet.empty
},
apply (tr, value, oldState, newState) {
const selection = newState.selection
if (!(selection instanceof TableSelection)) return DecorationSet.empty
const table = findTable(newState.selection)
if (table === undefined) return DecorationSet.empty
const decorations: Decoration[] = [
Decoration.node(table.pos, table.pos + table.node.nodeSize, { class: 'table-node-selected' })
]
return DecorationSet.create(newState.doc, decorations)
}
}
})
}
function handleModDelete (editor: Editor): boolean {
const { selection } = editor.state.tr
if (selection instanceof CellSelection) {
if (isTableSelected(selection)) {
return editor.commands.deleteTable()
plugins/text-editor-resources/src/components/extension/table/types.ts
+6
View File
@@ -14,9 +14,15 @@
//
import { type Node as ProseMirrorNode } from '@tiptap/pm/model'
import { CellSelection } from '@tiptap/pm/tables'
export interface TableNodeLocation {
pos: number
start: number
node: ProseMirrorNode
}
// This subclass serves as a tag to distinguish between situations where
// the table is selected as a node and when all cells in the table are selected,
// the deletion behavior depends on this.
export class TableSelection extends CellSelection {}
plugins/text-editor-resources/src/components/extension/table/utils.ts
+2 -2
View File
@@ -17,7 +17,7 @@ import { findParentNode } from '@tiptap/core'
import { type Selection, type Transaction } from '@tiptap/pm/state'
import { CellSelection, type Rect, TableMap, addColumn, addRow } from '@tiptap/pm/tables'
import { type TableNodeLocation } from './types'
import { TableSelection, type TableNodeLocation } from './types'
export function insertColumn (table: TableNodeLocation, index: number, tr: Transaction): Transaction {
const map = TableMap.get(table.node)
@@ -71,7 +71,7 @@ export function selectTable (table: TableNodeLocation, tr: Transaction): Transac
const $head = tr.doc.resolve(table.start + map[0])
const $anchor = tr.doc.resolve(table.start + map[map.length - 1])
return tr.setSelection(new CellSelection($anchor, $head))
return tr.setSelection(new TableSelection($anchor, $head))
}
export const isColumnSelected = (columnIndex: number, selection: Selection): boolean => {
plugins/text-editor-resources/src/index.ts
+6 -1
View File
@@ -26,6 +26,7 @@ import {
import { openImage, downloadImage, expandImage, moreImageActions } from './components/extension/imageExt'
import { configureNote, isEditableNote } from './components/extension/note'
import { createInlineComment, shouldShowCreateInlineCommentAction } from './components/extension/inlineComment'
import { isTextStylingEnabled, openBackgroundColorOptions, openTextColorOptions } from './components/extension/colors'
export * from '@hcengineering/presentation/src/types'
export type { EditorKitOptions } from './kits/editor-kit'
@@ -97,8 +98,12 @@ export default async (): Promise<Resources> => ({
IsEditableNote: isEditableNote,
IsEditable: isEditable,
IsHeadingVisible: isHeadingVisible,
IsTextStylingEnabled: isTextStylingEnabled,
CreateInlineComment: createInlineComment,
ShouldShowCreateInlineCommentAction: shouldShowCreateInlineCommentAction
ShouldShowCreateInlineCommentAction: shouldShowCreateInlineCommentAction,
SetBackgroundColor: openBackgroundColorOptions,
SetTextColor: openTextColorOptions
}
})
plugins/text-editor-resources/src/kits/editor-kit.ts
+7 -1
View File
@@ -15,7 +15,7 @@
import { type Class, type Doc, type Ref, type Space } from '@hcengineering/core'
import { getResource } from '@hcengineering/platform'
import { getBlobRef, getClient } from '@hcengineering/presentation'
import { CodeExtension, codeOptions } from '@hcengineering/text'
import { BackgroundColor, CodeExtension, codeOptions, TextColor, TextStyle } from '@hcengineering/text'
import textEditor, { type ActionContext, type ExtensionCreator, type TextEditorMode } from '@hcengineering/text-editor'
import { type AnyExtension, type Editor, Extension } from '@tiptap/core'
import { type Level } from '@tiptap/extension-heading'
@@ -201,6 +201,12 @@ async function buildEditorKit (): Promise<Extension<EditorKitOptions, any>> {
staticKitExtensions.push([400, ParagraphExtension.configure()])
}
if (mode === 'full') {
staticKitExtensions.push([410, TextStyle.configure({})])
staticKitExtensions.push([420, TextColor.configure({})])
staticKitExtensions.push([430, BackgroundColor.configure({ types: ['tableCell'] })])
}
staticKitExtensions.push([
500,
ListKeymapExtension.configure({
plugins/text-editor/src/plugin.ts
+9 -4
View File
@@ -93,8 +93,6 @@ export default plugin(textEditorId, {
CategoryColumn: '' as IntlString,
CategoryCell: '' as IntlString,
Table: '' as IntlString,
TableOptions: '' as IntlString,
SelectTable: '' as IntlString,
Width: '' as IntlString,
Height: '' as IntlString,
Unset: '' as IntlString,
@@ -102,7 +100,12 @@ export default plugin(textEditorId, {
SeparatorLine: '' as IntlString,
TodoList: '' as IntlString,
DrawingBoard: '' as IntlString,
MermaidDiargram: '' as IntlString
MermaidDiargram: '' as IntlString,
TableOptions: '' as IntlString,
SelectTable: '' as IntlString,
SetCellHighlightColor: '' as IntlString,
SetTextColor: '' as IntlString
},
icon: {
Header1: '' as Asset,
@@ -130,6 +133,8 @@ export default plugin(textEditorId, {
Comment: '' as Asset,
SelectTable: '' as Asset,
MergeCells: '' as Asset,
SplitCells: '' as Asset
SplitCells: '' as Asset,
Brush: '' as Asset,
TextStyle: '' as Asset
}
})
plugins/text-editor/src/types.ts
+1 -1
View File
@@ -186,7 +186,7 @@ export interface ActiveDescriptor {
params?: any
}
export type TextEditorActionKind = 'text' | 'image'
export type TextEditorActionKind = 'text' | 'image' | 'table'
/**
* Defines a text action for text action editor
plugins/tracker-resources/src/components/NewIssueHeader.svelte
+3 -29
View File
@@ -13,17 +13,16 @@
// limitations under the License.
-->
<script lang="ts">
import { Analytics } from '@hcengineering/analytics'
import { AccountRole, Ref, Space, getCurrentAccount, hasAccountRole } from '@hcengineering/core'
import { MultipleDraftController, getClient } from '@hcengineering/presentation'
import { TrackerEvents } from '@hcengineering/tracker'
import { ButtonWithDropdown, IconAdd, IconDropdown, SelectPopupValueType, showPopup } from '@hcengineering/ui'
import view from '@hcengineering/view'
import { Project, TrackerEvents } from '@hcengineering/tracker'
import { Analytics } from '@hcengineering/analytics'
import { onDestroy } from 'svelte'
import tracker from '../plugin'
import CreateIssue from './CreateIssue.svelte'
import { importTasks } from '..'
export let currentSpace: Ref<Space> | undefined
@@ -70,7 +69,6 @@
const client = getClient()
let keys: string[] | undefined = undefined
let inputFile: HTMLInputElement
async function dropdownItemSelected (res?: SelectPopupValueType['id']): Promise<void> {
if (res == null) return
@@ -79,39 +77,15 @@
showPopup(tracker.component.CreateProject, {}, 'top', () => {
closed = true
})
} else if (res === tracker.string.Import) {
inputFile.click()
} else {
await newIssue()
}
}
async function fileSelected (): Promise<void> {
const list = inputFile.files
if (list === null || list.length === 0) return
for (let index = 0; index < list.length; index++) {
const file = list.item(index)
if (file !== null && currentSpace != null) {
await importTasks(file, currentSpace as Ref<Project>)
}
}
inputFile.value = ''
}
client.findOne(view.class.Action, { _id: tracker.action.NewIssue }).then((p) => (keys = p?.keyBinding))
void client.findOne(view.class.Action, { _id: tracker.action.NewIssue }).then((p) => (keys = p?.keyBinding))
</script>
<div class="antiNav-subheader">
<input
bind:this={inputFile}
disabled={inputFile == null}
multiple
type="file"
name="file"
id="tasksInput"
accept="application/json"
style="display: none"
on:change={fileSelected}
/>
<ButtonWithDropdown
icon={IconAdd}
justify={'left'}
plugins/tracker-resources/src/index.ts
+2 -170
View File
@@ -14,9 +14,7 @@
//
import { Analytics } from '@hcengineering/analytics'
import chunter, { type ChatMessage } from '@hcengineering/chunter'
import core, {
AccountRole,
type AttachedDoc,
type Attribute,
type Class,
@@ -76,6 +74,7 @@ import StatusEditor from './components/issues/StatusEditor.svelte'
import StatusFilterValuePresenter from './components/issues/StatusFilterValuePresenter.svelte'
import StatusPresenter from './components/issues/StatusPresenter.svelte'
import TitlePresenter from './components/issues/TitlePresenter.svelte'
import LabelsView from './components/LabelsView.svelte'
import EditMilestone from './components/milestones/EditMilestone.svelte'
import MilestoneDatePresenter from './components/milestones/MilestoneDatePresenter.svelte'
import MyIssues from './components/myissues/MyIssues.svelte'
@@ -88,7 +87,6 @@ import SetDueDateActionPopup from './components/SetDueDateActionPopup.svelte'
import SetParentIssueActionPopup from './components/SetParentIssueActionPopup.svelte'
import SettingsRelatedTargets from './components/SettingsRelatedTargets.svelte'
import CreateIssueTemplate from './components/templates/CreateIssueTemplate.svelte'
import LabelsView from './components/LabelsView.svelte'
import {
getIssueIdByIdentifier,
getIssueTitle,
@@ -162,11 +160,7 @@ import ProjectSpacePresenter from './components/projects/ProjectSpacePresenter.s
import { get } from 'svelte/store'
import contact, { AvatarType } from '@hcengineering/contact'
import { personAccountByIdStore, personAccountPersonByIdStore, personByIdStore } from '@hcengineering/contact-resources'
import notification, { type Collaborators } from '@hcengineering/notification'
import { settingId } from '@hcengineering/setting'
import task, { type TaskType } from '@hcengineering/task'
import { getAllStates } from '@hcengineering/task-resources'
import EstimationValueEditor from './components/issues/timereport/EstimationValueEditor.svelte'
import TimePresenter from './components/issues/timereport/TimePresenter.svelte'
@@ -404,167 +398,6 @@ async function deleteMilestone (milestones: Milestone | Milestone[]): Promise<vo
}
}
type ImportIssue = Issue & { activity: ChatMessage[] }
export async function importTasks (tasks: File, space: Ref<Project>): Promise<void> {
const reader = new FileReader()
reader.readAsText(tasks)
const personAccountById = get(personAccountByIdStore)
const personAccountList = Array.from(personAccountById.values())
const personAccountPersonById = get(personAccountPersonByIdStore)
const personList = Array.from(get(personByIdStore).values())
const client = getClient()
const statuses = await client.findAll(tracker.class.IssueStatus, {})
reader.onload = async () => {
let tasksArray: ImportIssue[] = Array.from(JSON.parse(reader.result as string))
const personToImport = Array.from(
new Set(tasksArray.flatMap((t) => [t.createdBy, t.modifiedBy, ...t.activity.flatMap((act) => act.modifiedBy)]))
).filter((x) => x !== undefined) as string[]
const peopleToAdd = personToImport.filter((p) => personList.find((x) => x.name === p) === undefined)
if (peopleToAdd.length > 0) {
console.log('Next people will be created to import properly', peopleToAdd)
for (const personToCreate of peopleToAdd) {
const personId = await client.createDoc(contact.class.Person, contact.space.Contacts, {
name: personToCreate,
avatarType: AvatarType.COLOR,
city: '',
comments: 0,
channels: 0,
attachments: 0
})
await client.createDoc(contact.class.PersonAccount, core.space.Model, {
email: `imported:${personId}`,
person: personId,
role: AccountRole.User
})
}
}
const idsParent: Array<{ id: Ref<Issue>, identifier: string }> = []
while (tasksArray.length > 0) {
let taskParsing: ImportIssue | undefined = tasksArray.find((t: ImportIssue) => t?.parents?.length === 0)
if (taskParsing === undefined) {
taskParsing = tasksArray.find((t: Issue) =>
t?.parents?.every((p) => idsParent.findIndex((par) => par.id === p.parentId) !== -1)
)
}
if (taskParsing != null) {
tasksArray = tasksArray.filter((t) => t._id !== taskParsing?._id)
const proj = await client.findOne(tracker.class.Project, { _id: space })
const modifiedByPerson = personList.find((p) => p.name === taskParsing?.modifiedBy)?._id
const assignee =
taskParsing.assignee !== null ? personList.find((p) => p.name === taskParsing?.assignee)?._id ?? null : null
if (modifiedByPerson === undefined) throw new Error('Person not found')
const modifiedBy = personAccountList.find((pA) => pA.person === modifiedByPerson)?._id
if (modifiedBy === undefined) throw new Error('modifiedBy account not found')
const collaborators = (taskParsing as any)['notification:mixin:Collaborators']?.collaborators
const collaboratorsToImport =
collaborators !== undefined
? collaborators
.map((name: string) => {
const person = personList.find((p) => p.name === name)?._id
if (person === undefined) return undefined
const account = personAccountPersonById.get(person)
return account?._id
})
.filter((c: any) => c !== undefined)
: undefined
const incResult = await client.updateDoc(
tracker.class.Project,
core.space.Space,
space,
{
$inc: { sequence: 1 }
},
true
)
const number = (incResult as any).object.sequence
const identifier = `${proj?.identifier}-${number}`
idsParent.push({ id: taskParsing._id, identifier })
const taskKind = proj?.type !== undefined ? { parent: proj.type } : {}
const kind = (await client.findOne(task.class.TaskType, taskKind)) as TaskType
const status = statuses.find((s) => s.name === taskParsing?.status)?._id
if (status === undefined) throw new Error('status not found')
const taskToCreate = {
title: taskParsing.title,
description: taskParsing.description,
component: taskParsing.component,
milestone: taskParsing.milestone,
number,
status,
priority: taskParsing.priority,
rank: taskParsing.rank,
comments: 0,
subIssues: 0,
dueDate: taskParsing.dueDate,
parents: taskParsing.parents.map((p) => ({
...p,
space,
identifier: idsParent.find((par) => par.id === p.parentId)?.identifier ?? p.identifier
})),
reportedTime: 0,
remainingTime: 0,
estimation: taskParsing.estimation,
reports: 0,
childInfo: taskParsing.childInfo,
identifier,
modifiedBy,
assignee,
kind: kind._id
}
await client.addCollection(
tracker.class.Issue,
space,
taskParsing?.attachedTo ?? tracker.ids.NoParent,
taskParsing._class,
'subIssues',
taskToCreate,
taskParsing._id
)
if (collaboratorsToImport !== undefined) {
await client.createMixin<Doc, Collaborators>(
taskParsing._id,
taskParsing._class,
space,
notification.mixin.Collaborators,
{
collaborators: collaboratorsToImport
}
)
}
// Push activity
if (taskParsing.activity !== undefined) {
const act = taskParsing.activity.sort((a, b) => a.modifiedOn - b.modifiedOn)
for (const activityMessage of act) {
const modifiedByPerson = personList.find((p) => p.name === activityMessage.modifiedBy)?._id
const modifiedBy = personAccountList.find((pA) => pA.person === modifiedByPerson)?._id
if (modifiedBy === undefined) throw new Error('modifiedBy account not found')
await client.addCollection(
chunter.class.ChatMessage,
space,
taskParsing._id,
tracker.class.Issue,
'comments',
{
message: activityMessage.message
},
activityMessage._id,
activityMessage.modifiedOn,
modifiedBy
)
}
}
}
}
}
}
function filterComponents (doc: Component, target: Component): boolean {
return doc.label.toLowerCase().trim() === target.label.toLowerCase().trim() && doc._id !== target._id
}
@@ -702,8 +535,7 @@ export default async (): Promise<Resources> => ({
EditProject: editProject,
DeleteMilestone: deleteMilestone,
DeleteProject: deleteProject,
DeleteIssue: deleteIssue,
ImportIssues: importTasks
DeleteIssue: deleteIssue
},
resolver: {
Location: resolveLocation
pods/fulltext/package.json
+1 -1
View File
@@ -15,7 +15,7 @@
"_phase:docker-build": "rushx docker:build",
"_phase:docker-staging": "rushx docker:staging",
"get-model": "node ../../common/scripts/esbuild.js --entry=src/get-model.ts --keep-names=true --bundle=true --define=MODEL_VERSION --define=VERSION --define=GIT_REVISION && node ./bundle/bundle.js > ./bundle/model.json",
"bundle": "rushx get-model && node ../../common/scripts/esbuild.js --keep-names=true --bundle=true --external=*.node --sourcemap=external",
"bundle": "rushx get-model && node ../../common/scripts/esbuild.js --keep-names=true --bundle=true --external=*.node --external=snappy --sourcemap=external",
"docker:build": "../../common/scripts/docker_build.sh hardcoreeng/fulltext",
"docker:tbuild": "docker build -t hardcoreeng/fulltext . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/fulltext",
"docker:abuild": "docker build -t hardcoreeng/fulltext . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/fulltext",
pods/server/package.json
+6 -3
View File
@@ -17,7 +17,7 @@
"_phase:docker-build": "rushx docker:build",
"_phase:docker-staging": "rushx docker:staging",
"get-model": "node ../../common/scripts/esbuild.js --entry=src/get-model.ts -keep-names=true --define=MODEL_VERSION --define=VERSION --define=GIT_REVISION --bundle=true && node ./bundle/bundle.js > ./bundle/model.json",
"bundle": "rushx get-model && node ../../common/scripts/esbuild.js --entry=src/__start.ts --keep-names=true --bundle=true --sourcemap=external --external=*.node --external=*.node --external=bufferutil --external=snappy --define=MODEL_VERSION --define=VERSION --define=GIT_REVISION --external=utf-8-validate --external=msgpackr-extract",
"bundle": "rushx get-model && node ../../common/scripts/esbuild.js --entry=src/__start.ts --keep-names=true --bundle=true --sourcemap=external --external=*.node --external=bufferutil --external=snappy --define=MODEL_VERSION --define=VERSION --define=GIT_REVISION --external=utf-8-validate --external=msgpackr-extract",
"docker:build": "../../common/scripts/docker_build.sh hardcoreeng/transactor",
"docker:tbuild": "docker build -t hardcoreeng/transactor . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/transactor",
"docker:abuild": "docker build -t hardcoreeng/transactor . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/transactor",
@@ -51,7 +51,8 @@
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5",
"@hcengineering/model-all": "^0.6.0"
"@hcengineering/model-all": "^0.6.0",
"snappyjs": "^0.7.0"
},
"dependencies": {
"@hcengineering/core": "^0.6.32",
@@ -77,6 +78,8 @@
"bufferutil": "^4.0.8",
"msgpackr": "^1.11.2",
"msgpackr-extract": "^3.0.3",
"@hcengineering/postgres": "^0.6.0"
"@hcengineering/postgres": "^0.6.0",
"snappy": "^7.2.2",
"@hcengineering/rpc": "^0.6.5"
}
}
pods/server/src/__tests__/compression.spec.ts
+23
View File
@@ -0,0 +1,23 @@
import { readFile } from 'fs/promises'
import { compress } from 'snappy'
import { RPCHandler } from '@hcengineering/rpc'
describe('compression-tests', () => {
it('check-snappy', async () => {
const modelJSON = (await readFile('./bundle/model.json')).toString()
const txes = JSON.parse(modelJSON)
const compressed = await compress(modelJSON)
console.log(modelJSON.length, compressed.length, compressed.length / modelJSON.length)
expect(compressed.length).toBeLessThan(modelJSON.length)
const rpc = new RPCHandler()
const jsonData = rpc.serialize(txes, true)
const compressed2 = await compress(jsonData)
console.log(jsonData.length, compressed2.length, compressed2.length / jsonData.length)
expect(compressed2.length).toBeLessThan(jsonData.length)
})
})
rush.json
-5
View File
@@ -2176,11 +2176,6 @@
"projectFolder": "workers/branding",
"shouldPublish": false
},
{
"packageName": "@hcengineering/cloud-datalake",
"projectFolder": "workers/datalake",
"shouldPublish": false
},
{
"packageName": "@hcengineering/scripts",
"projectFolder": "common/scripts",
server/core/src/types.ts
-1
View File
@@ -709,7 +709,6 @@ export type ServerFactory = (
ctx: MeasureContext,
pipelineFactory: PipelineFactory,
port: number,
enableCompression: boolean,
accountsUrl: string,
externalStorage: StorageAdapter
) => () => Promise<void>
server/rpc/src/rpc.ts
+1
View File
@@ -50,6 +50,7 @@ export interface HelloResponse extends Response<any> {
lastTx?: string
lastHash?: string // Last model hash
account: Account
useCompression?: boolean
}
function replacer (key: string, value: any): any {
server/server/src/sessionManager.ts
+18 -7
View File
@@ -117,7 +117,8 @@ class TSessionManager implements SessionManager {
stop: () => Promise<string | undefined>
}
| undefined,
readonly accountsUrl: string
readonly accountsUrl: string,
readonly enableCompression: boolean
) {
this.checkInterval = setInterval(() => {
this.handleTick()
@@ -1082,7 +1083,7 @@ class TSessionManager implements SessionManager {
): Promise<void> {
const hello = request as HelloRequest
service.binaryMode = hello.binary ?? false
service.useCompression = hello.compression ?? false
service.useCompression = this.enableCompression ? hello.compression ?? false : false
if (LOGGING_ENABLED) {
ctx.info('hello happen', {
@@ -1109,7 +1110,8 @@ class TSessionManager implements SessionManager {
serverVersion: this.serverVersion,
lastTx: pipeline.context.lastTx,
lastHash: pipeline.context.lastHash,
account: service.getRawAccount(pipeline)
account: service.getRawAccount(pipeline),
useCompression: service.useCompression
}
ws.send(requestCtx, helloResponse, false, false)
}
@@ -1126,9 +1128,18 @@ export function createSessionManager (
stop: () => Promise<string | undefined>
}
| undefined,
accountsUrl: string
accountsUrl: string,
enableCompression: boolean
): SessionManager {
return new TSessionManager(ctx, sessionFactory, timeouts, brandingMap ?? null, profiling, accountsUrl)
return new TSessionManager(
ctx,
sessionFactory,
timeouts,
brandingMap ?? null,
profiling,
accountsUrl,
enableCompression
)
}
/**
@@ -1160,7 +1171,8 @@ export function startSessionManager (
reconnectTimeout: 500
},
opt.profiling,
opt.accountsUrl
opt.accountsUrl,
opt.enableCompression ?? false
)
return {
shutdown: opt.serverFactory(
@@ -1171,7 +1183,6 @@ export function startSessionManager (
ctx,
opt.pipelineFactory,
opt.port,
opt.enableCompression ?? false,
opt.accountsUrl,
opt.externalStorage
),
server/ws/package.json
+2 -1
View File
@@ -51,6 +51,7 @@
"express": "^4.21.2",
"utf-8-validate": "^6.0.4",
"ws": "^8.18.0",
"body-parser": "^1.20.2"
"body-parser": "^1.20.2",
"snappy": "^7.2.2"
}
}
server/ws/src/server_http.ts
+16 -27
View File
@@ -45,7 +45,9 @@ import os from 'os'
import { WebSocketServer, type RawData, type WebSocket } from 'ws'
import 'bufferutil'
import { compress } from 'snappy'
import 'utf-8-validate'
let profiling = false
const rpcHandler = new RPCHandler()
/**
@@ -60,14 +62,12 @@ export function startHttpServer (
ctx: MeasureContext,
pipelineFactory: PipelineFactory,
port: number,
enableCompression: boolean,
accountsUrl: string,
externalStorage: StorageAdapter
): () => Promise<void> {
if (LOGGING_ENABLED) {
ctx.info('starting server on', {
port,
enableCompression,
accountsUrl,
parallel: os.availableParallelism()
})
@@ -324,27 +324,7 @@ export function startHttpServer (
const httpServer = http.createServer(app)
const wss = new WebSocketServer({
noServer: true,
perMessageDeflate: enableCompression
? {
zlibDeflateOptions: {
// See zlib defaults.
chunkSize: 32 * 1024,
memLevel: 1,
level: 1
},
zlibInflateOptions: {
chunkSize: 32 * 1024,
level: 1,
memLevel: 1
},
serverNoContextTakeover: true,
clientNoContextTakeover: true,
// Below options specified as default values.
concurrencyLimit: Math.max(10, os.availableParallelism()), // Limits zlib concurrency for perf.
threshold: 1024 // Size (in bytes) below which messages
// should not be compressed if context takeover is disabled.
}
: false,
perMessageDeflate: false,
skipUTF8Validation: true,
maxPayload: 250 * 1024 * 1024,
clientTracking: false // We do not need to track clients inside clients.
@@ -566,7 +546,7 @@ function createWebsocketClientSocket (
}
ws.send(pongConst)
},
send: (ctx: MeasureContext, msg, binary, compression) => {
send: (ctx: MeasureContext, msg, binary, _compression) => {
const smsg = rpcHandler.serialize(msg, binary)
ctx.measure('send-data', smsg.length)
@@ -574,15 +554,24 @@ function createWebsocketClientSocket (
if (ws.readyState !== ws.OPEN || cs.isClosed) {
return
}
ws.send(smsg, { binary: true, compress: compression }, (err) => {
const handleErr = (err?: Error): void => {
ctx.measure('msg-send-delta', Date.now() - st)
if (err != null) {
if (!`${err.message}`.includes('WebSocket is not open')) {
ctx.error('send error', { err })
Analytics.handleError(err)
}
}
ctx.measure('msg-send-delta', Date.now() - st)
})
}
if (_compression) {
void compress(smsg).then((msg: any) => {
ws.send(msg, { binary: true }, handleErr)
})
} else {
ws.send(smsg, { binary: true }, handleErr)
}
}
}
return cs
services/github/pod-github/src/client.ts
+2 -2
View File
@@ -19,7 +19,7 @@ export async function createPlatformClient (
workspace: string,
timeout: number,
reconnect?: (event: ClientConnectEvent, data: any) => Promise<void>
): Promise<Client> {
): Promise<{ client: Client, endpoint: string }> {
setMetadata(client.metadata.ClientSocketFactory, (url) => {
return new WebSocket(url, {
headers: {
@@ -45,5 +45,5 @@ export async function createPlatformClient (
onConnect: reconnect
})
return connection
return { client: connection, endpoint }
}
services/github/pod-github/src/config.ts
+10 -2
View File
@@ -29,6 +29,9 @@ interface Config {
BrandingPath: string
WorkspaceInactivityInterval: number // Interval in days to stop workspace synchronization if not visited
// Limits
RateLimit: number
}
const envMap: { [key in keyof Config]: string } = {
@@ -55,7 +58,11 @@ const envMap: { [key in keyof Config]: string } = {
SentryDSN: 'SENTRY_DSN',
BrandingPath: 'BRANDING_PATH',
WorkspaceInactivityInterval: 'WORKSPACE_INACTIVITY_INTERVAL'
WorkspaceInactivityInterval: 'WORKSPACE_INACTIVITY_INTERVAL',
// Limits
RateLimit: 'RATE_LIMIT' // Operations per second for one transactor
}
const required: Array<keyof Config> = [
@@ -101,7 +108,8 @@ const config: Config = (() => {
SentryDSN: process.env[envMap.SentryDSN],
BrandingPath: process.env[envMap.BrandingPath] ?? '',
WorkspaceInactivityInterval: parseInt(process.env[envMap.WorkspaceInactivityInterval] ?? '5') // In days
WorkspaceInactivityInterval: parseInt(process.env[envMap.WorkspaceInactivityInterval] ?? '5'), // In days
RateLimit: parseInt(process.env[envMap.RateLimit] ?? '25')
}
const missingEnv = required.filter((key) => params[key] === undefined).map((key) => envMap[key])
services/github/pod-github/src/platform.ts
+14 -2
View File
@@ -15,6 +15,7 @@ import core, {
RateLimiter,
Ref,
systemAccountEmail,
TimeRateLimiter,
TxOperations
} from '@hcengineering/core'
import github, { GithubAuthentication, makeQuery, type GithubIntegration } from '@hcengineering/github'
@@ -73,6 +74,8 @@ export class PlatformWorker {
userManager!: UserManager
rateLimits = new Map<string, TimeRateLimiter>()
private constructor (
readonly ctx: MeasureContext,
readonly app: App,
@@ -83,6 +86,15 @@ export class PlatformWorker {
registerLoaders()
}
getRateLimiter (endpoint: string): TimeRateLimiter {
let limiter = this.rateLimits.get(endpoint)
if (limiter === undefined) {
limiter = new TimeRateLimiter(config.RateLimit)
this.rateLimits.set(endpoint, limiter)
}
return limiter
}
public async initStorage (): Promise<void> {
this.mongoRef = getMongoClient(config.MongoURL)
const mongoClient = await this.mongoRef.getClient()
@@ -230,7 +242,7 @@ export class PlatformWorker {
} else {
let client: Client | undefined
try {
client = await createPlatformClient(oldWorkspace, 30000)
;({ client } = await createPlatformClient(oldWorkspace, 30000))
await this.removeInstallationFromWorkspace(oldWorker, installationId)
await client.close()
} catch (err: any) {
@@ -386,7 +398,7 @@ export class PlatformWorker {
platformClient = this.clients.get(payload.workspace)?.client
if (platformClient === undefined) {
shouldClose = true
platformClient = await createPlatformClient(payload.workspace, 30000)
;({ client: platformClient } = await createPlatformClient(payload.workspace, 30000))
}
const client = new TxOperations(platformClient, payload.accountId)
services/github/pod-github/src/sync/comments.ts
+9
View File
@@ -355,6 +355,10 @@ export class CommentSyncManager implements DocSyncManager {
}
}
isHulyLinkComment (message: string): boolean {
return message.includes('<p>Connected to') && message.includes('Huly&reg;')
}
private async createComment (
info: DocSyncInfo,
messageData: MessageData,
@@ -367,6 +371,11 @@ export class CommentSyncManager implements DocSyncManager {
...messageData,
attachments: 0
}
// Check if it is Connected message.
if ((comment as any).performed_via_github_app !== undefined && this.isHulyLinkComment(comment.body)) {
// No need to create comment on platform.
return
}
await this.client.addCollection(
chunter.class.ChatMessage,
info.space,
services/github/pod-github/src/worker.ts
+115 -106
View File
@@ -35,7 +35,8 @@ import core, {
reduceCalls,
toIdMap,
type Blob,
type MigrationState
type MigrationState,
type TimeRateLimiter
} from '@hcengineering/core'
import github, {
DocSyncInfo,
@@ -336,6 +337,7 @@ export class GithubWorker implements IntegrationManager {
private constructor (
readonly ctx: MeasureContext,
readonly limiter: TimeRateLimiter,
readonly platform: PlatformWorker,
readonly installations: Map<number, InstallationRecord>,
readonly client: Client,
@@ -1152,23 +1154,26 @@ export class GithubWorker implements IntegrationManager {
const _projects = toIdMap(projects)
const _repositories = repositories.map((it) => it._id)
const docs = await this.ctx.with(
'find-doc-sync-info',
{},
(ctx) =>
this._client.findAll<DocSyncInfo>(
github.class.DocSyncInfo,
{
needSync: { $ne: githubSyncVersion },
externalVersion: { $in: [githubExternalSyncVersion, '#'] },
space: { $in: Array.from(_projects.keys()) },
repository: { $in: [null, ..._repositories] }
},
{
limit: 50
}
),
{ _projects, _repositories }
const docs = await this.limiter.exec(
async () =>
await this.ctx.with(
'find-doc-sync-info',
{},
(ctx) =>
this._client.findAll<DocSyncInfo>(
github.class.DocSyncInfo,
{
needSync: { $ne: githubSyncVersion },
externalVersion: { $in: [githubExternalSyncVersion, '#'] },
space: { $in: Array.from(_projects.keys()) },
repository: { $in: [null, ..._repositories] }
},
{
limit: 50
}
),
{ _projects, _repositories }
)
)
//
@@ -1289,98 +1294,100 @@ export class GithubWorker implements IntegrationManager {
})
for (const info of orderedSyncInfo) {
try {
const existing = externalDocs.find((it) => it._id === info._id)
const mapper = this.mappers.find((it) => it._class.includes(info.objectClass))?.mapper
if (mapper === undefined) {
this.ctx.info('No mapper for class', { objectClass: info.objectClass, workspace: this.workspace.name })
await derivedClient.update<DocSyncInfo>(info, {
needSync: githubSyncVersion
})
continue
}
const repo = await this.getRepositoryById(info.repository)
if (repo !== undefined && !repo.enabled) {
continue
}
let parent =
info.parent !== undefined
? parents.find((it) => it.url.toLowerCase() === info.parent?.toLowerCase())
: undefined
if (
parent === undefined &&
existing !== undefined &&
this.client.getHierarchy().isDerived(existing._class, core.class.AttachedDoc)
) {
// Find with attached parent
parent = attachedParents.find((it) => it._id === (existing as AttachedDoc).attachedTo)
}
if (existing !== undefined && existing.space !== info.space) {
// document is moved to non github project, so for github it like delete.
const targetProject = await this.client.findOne(github.mixin.GithubProject, {
_id: existing.space as Ref<GithubProject>
})
if (await mapper.handleDelete(existing, info, derivedClient, false, parent)) {
const h = this._client.getHierarchy()
await derivedClient.remove(info)
if (h.hasMixin(existing, github.mixin.GithubIssue)) {
const mixinData = this._client.getHierarchy().as(existing, github.mixin.GithubIssue)
await this._client.update<GithubIssue>(
mixinData,
{
url: '',
githubNumber: 0,
repository: '' as Ref<GithubIntegrationRepository>
},
false,
Date.now(),
existing.modifiedBy
)
}
continue
}
if (targetProject !== undefined) {
// We need to sync into new project.
await this.limiter.exec(async () => {
try {
const existing = externalDocs.find((it) => it._id === info._id)
const mapper = this.mappers.find((it) => it._class.includes(info.objectClass))?.mapper
if (mapper === undefined) {
this.ctx.info('No mapper for class', { objectClass: info.objectClass, workspace: this.workspace.name })
await derivedClient.update<DocSyncInfo>(info, {
external: null,
current: null,
url: '',
needSync: '',
externalVersion: '',
githubNumber: 0,
repository: null
needSync: githubSyncVersion
})
return
}
}
if (info.deleted === true) {
if (await mapper.handleDelete(existing, info, derivedClient, true)) {
await derivedClient.remove(info)
const repo = await this.getRepositoryById(info.repository)
if (repo !== undefined && !repo.enabled) {
return
}
continue
}
const docUpdate = await this.ctx.withLog(
'sync doc',
{},
(ctx) => mapper.sync(existing, info, parent, derivedClient),
{ url: info.url.toLowerCase(), workspace: this.workspace.name }
)
if (docUpdate !== undefined) {
await derivedClient.update(info, docUpdate)
let parent =
info.parent !== undefined
? parents.find((it) => it.url.toLowerCase() === info.parent?.toLowerCase())
: undefined
if (
parent === undefined &&
existing !== undefined &&
this.client.getHierarchy().isDerived(existing._class, core.class.AttachedDoc)
) {
// Find with attached parent
parent = attachedParents.find((it) => it._id === (existing as AttachedDoc).attachedTo)
}
if (existing !== undefined && existing.space !== info.space) {
// document is moved to non github project, so for github it like delete.
const targetProject = await this.client.findOne(github.mixin.GithubProject, {
_id: existing.space as Ref<GithubProject>
})
if (await mapper.handleDelete(existing, info, derivedClient, false, parent)) {
const h = this._client.getHierarchy()
await derivedClient.remove(info)
if (h.hasMixin(existing, github.mixin.GithubIssue)) {
const mixinData = this._client.getHierarchy().as(existing, github.mixin.GithubIssue)
await this._client.update<GithubIssue>(
mixinData,
{
url: '',
githubNumber: 0,
repository: '' as Ref<GithubIntegrationRepository>
},
false,
Date.now(),
existing.modifiedBy
)
}
return
}
if (targetProject !== undefined) {
// We need to sync into new project.
await derivedClient.update<DocSyncInfo>(info, {
external: null,
current: null,
url: '',
needSync: '',
externalVersion: '',
githubNumber: 0,
repository: null
})
}
}
if (info.deleted === true) {
if (await mapper.handleDelete(existing, info, derivedClient, true)) {
await derivedClient.remove(info)
}
return
}
const docUpdate = await this.ctx.withLog(
'sync doc',
{},
(ctx) => mapper.sync(existing, info, parent, derivedClient),
{ url: info.url.toLowerCase(), workspace: this.workspace.name }
)
if (docUpdate !== undefined) {
await derivedClient.update(info, docUpdate)
}
} catch (err: any) {
Analytics.handleError(err)
this.ctx.error('failed to sync doc', { _id: info._id, objectClass: info.objectClass, error: err })
// Mark to stop processing of document, before restart.
await derivedClient.update<DocSyncInfo>(info, {
error: errorToObj(err),
needSync: githubSyncVersion,
externalVersion: githubExternalSyncVersion
})
}
} catch (err: any) {
Analytics.handleError(err)
this.ctx.error('failed to sync doc', { _id: info._id, objectClass: info.objectClass, error: err })
// Mark to stop processing of document, before restart.
await derivedClient.update<DocSyncInfo>(info, {
error: errorToObj(err),
needSync: githubSyncVersion,
externalVersion: githubExternalSyncVersion
})
}
})
}
}
@@ -1564,15 +1571,17 @@ export class GithubWorker implements IntegrationManager {
): Promise<GithubWorker | undefined> {
ctx.info('Connecting to', { workspace: workspace.workspaceUrl, workspaceId: workspace.workspaceName })
let client: Client | undefined
let endpoint: string | undefined
try {
client = await createPlatformClient(workspace.name, 30000, async (event: ClientConnectEvent) => {
;({ client, endpoint } = await createPlatformClient(workspace.name, 30000, async (event: ClientConnectEvent) => {
reconnect(workspace.name, event)
})
}))
await GithubWorker.checkIntegrations(client, installations)
const worker = new GithubWorker(
ctx,
platformWorker.getRateLimiter(endpoint ?? ''),
platformWorker,
installations,
client,
services/github/server-github-model/src/index.ts
+4
View File
@@ -16,6 +16,10 @@ export function createModel (builder: Builder): void {
trigger: serverGithub.trigger.OnProjectChanges,
isAsync: true
})
builder.createDoc(serverCore.class.Trigger, core.space.Model, {
trigger: serverGithub.trigger.OnGithubBroadcast,
isAsync: false
})
// We should skip activity github mixin stuff.
builder.createDoc(time.class.TodoAutomationHelper, core.space.Model, {
services/github/server-github-resources/src/index.ts
+20 -16
View File
@@ -24,10 +24,28 @@ import { TriggerControl } from '@hcengineering/server-core'
import time, { ToDo } from '@hcengineering/time'
import tracker from '@hcengineering/tracker'
/**
* @public
*/
export async function OnGithubBroadcast (txes: Tx[], control: TriggerControl): Promise<Tx[]> {
// Enhance broadcast to send DocSyncInfo change only to system account.
control.ctx.contextData.broadcast.targets.github = (it) => {
if (TxProcessor.isExtendsCUD(it._class)) {
if ((it as TxCUD<Doc>).objectClass === github.class.DocSyncInfo) {
return [systemAccountEmail]
}
}
}
return []
}
/**
* @public
*/
export async function OnProjectChanges (txes: Tx[], control: TriggerControl): Promise<Tx[]> {
// Enhance broadcast to send DocSyncInfo change only to system account.
await OnGithubBroadcast(txes, control)
const result: Tx[] = []
const cache = new Map<string, any>()
@@ -83,7 +101,8 @@ export async function OnProjectChanges (txes: Tx[], control: TriggerControl): Pr
// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
export default async () => ({
trigger: {
OnProjectChanges
OnProjectChanges,
OnGithubBroadcast
},
functions: {
TodoDoneTester
@@ -206,14 +225,6 @@ function updateSyncDoc (
data
)
)
control.ctx.contextData.broadcast.targets.github = (it) => {
if (control.hierarchy.isDerived(it._class, core.class.TxCUD)) {
if ((it as TxCUD<Doc>).objectClass === github.class.DocSyncInfo) {
return [systemAccountEmail]
}
}
}
}
function createSyncDoc (
@@ -249,11 +260,4 @@ function createSyncDoc (
cud.objectId as Ref<DocSyncInfo>
)
)
control.ctx.contextData.broadcast.targets.github = (it) => {
if (control.hierarchy.isDerived(it._class, core.class.TxCUD)) {
if ((it as TxCUD<Doc>).objectClass === github.class.DocSyncInfo) {
return [systemAccountEmail]
}
}
}
}
services/github/server-github/src/index.ts
+2 -1
View File
@@ -20,7 +20,8 @@ export const serverGithubId = 'server-github' as Plugin
*/
export default plugin(serverGithubId, {
trigger: {
OnProjectChanges: '' as Resource<TriggerFunc>
OnProjectChanges: '' as Resource<TriggerFunc>,
OnGithubBroadcast: '' as Resource<TriggerFunc>
},
functions: {
TodoDoneTester: '' as Resource<TodoDoneTester>
tests/docker-compose.yaml
+1
View File
@@ -139,6 +139,7 @@ services:
- BRANDING_PATH=/var/cfg/branding.json
- FULLTEXT_URL=http://fulltext:4710
- STATS_URL=http://stats:4901
- ENABLE_COMPRESSION=true
collaborator:
image: hardcoreeng/collaborator
links:
workers/datalake/.eslintrc.js
-7
View File
@@ -1,7 +0,0 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}
workers/datalake/jest.config.js
-7
View File
@@ -1,7 +0,0 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
roots: ["./src"],
coverageReporters: ["text-summary", "html"]
}
workers/datalake/package.json
-43
View File
@@ -1,43 +0,0 @@
{
"name": "@hcengineering/cloud-datalake",
"version": "0.6.0",
"main": "lib/index.js",
"types": "types/index.d.ts",
"template": "@hcengineering/cloud-package",
"scripts": {
"deploy": "wrangler deploy",
"dev": "wrangler dev --port 4021",
"start": "wrangler dev --port 4021",
"cf-typegen": "wrangler types",
"build": "compile",
"build:watch": "compile",
"test": "jest --passWithNoTests --silent --forceExit",
"format": "format src",
"_phase:build": "compile transpile src",
"_phase:test": "jest --passWithNoTests --silent --forceExit",
"_phase:format": "format src",
"_phase:validate": "compile validate"
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@cloudflare/workers-types": "^4.20241022.0",
"typescript": "^5.3.3",
"wrangler": "^3.97.0",
"jest": "^29.7.0",
"prettier": "^3.1.0",
"ts-jest": "^29.1.1",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-n": "^15.4.0",
"eslint-plugin-promise": "^6.1.1",
"eslint": "^8.54.0",
"@types/jest": "^29.5.5"
},
"dependencies": {
"aws4fetch": "^1.0.20",
"itty-router": "^5.0.18",
"postgres": "^3.4.5"
}
}
workers/datalake/schema/Dockerfile
-15
View File
@@ -1,15 +0,0 @@
FROM cockroachdb/cockroach:latest
ADD init.sh /cockroach/
RUN chmod a+x /cockroach/init.sh
ADD logs.yaml /cockroach/
ADD optimizations.sql /cockroach/
ADD datalake.sql /cockroach/
WORKDIR /cockroach/
EXPOSE 8080
EXPOSE 26257
ENTRYPOINT ["/cockroach/init.sh"]
workers/datalake/schema/README.md
-5
View File
@@ -1,5 +0,0 @@
# Running Cockroach DB in Docker
```bash
docker run -d -p 8080:8080 -p 26257:26257 cockroach:dev -name cockroach
```
workers/datalake/schema/datalake.sql
-31
View File
@@ -1,31 +0,0 @@
CREATE SCHEMA IF NOT EXISTS blob;
DROP TABLE IF EXISTS blob.blob;
DROP TABLE IF EXISTS blob.data;
DROP TYPE IF EXISTS blob.location;
-- B L O B
CREATE TYPE blob.location AS ENUM ('weur', 'eeur', 'wnam', 'enam', 'apac');
\echo "Creating blob.data..."
CREATE TABLE blob.data (
hash UUID NOT NULL,
location blob.location NOT NULL,
size INT8 NOT NULL,
filename UUID NOT NULL,
type STRING(255) NOT NULL,
CONSTRAINT pk_data PRIMARY KEY (hash, location)
);
\echo "Creating blob.blob..."
CREATE TABLE blob.blob (
workspace STRING(255) NOT NULL,
name STRING(255) NOT NULL,
hash UUID NOT NULL,
location blob.location NOT NULL,
deleted BOOL NOT NULL,
CONSTRAINT pk_blob PRIMARY KEY (workspace, name),
CONSTRAINT fk_data FOREIGN KEY (hash, location) REFERENCES blob.data (hash, location)
);
workers/datalake/schema/init.sh
-9
View File
@@ -1,9 +0,0 @@
#!/usr/bin/env bash
./cockroach start-single-node --insecure --log-config-file=logs.yaml --cache=.25 --background --store=type=mem,size=50%
./cockroach sql --insecure --file optimizations.sql
./cockroach sql --insecure --execute="CREATE DATABASE datalake;"
./cockroach sql --insecure --database=datalake --file datalake.sql
cd /cockroach/cockroach-data/logs
tail -f cockroach.log
workers/datalake/schema/logs.yaml
-67
View File
@@ -1,67 +0,0 @@
file-defaults:
max-file-size: 10MiB
max-group-size: 100MiB
file-permissions: 644
buffered-writes: true
filter: WARNING
format: crdb-v2
redact: false
redactable: true
exit-on-error: true
auditable: false
dir: cockroach-data/logs
fluent-defaults:
filter: WARNING
format: json-fluent-compact
redact: false
redactable: true
exit-on-error: false
auditable: false
http-defaults:
method: POST
unsafe-tls: false
timeout: 0s
disable-keep-alives: false
filter: WARNING
format: json-compact
redact: false
redactable: true
exit-on-error: false
auditable: false
sinks:
file-groups:
default:
channels:
WARNING: all
health:
channels: [HEALTH]
pebble:
channels: [STORAGE]
security:
channels: [PRIVILEGES, USER_ADMIN]
auditable: true
sql-audit:
channels: [SENSITIVE_ACCESS]
auditable: true
sql-auth:
channels: [SESSIONS]
auditable: true
sql-exec:
channels: [SQL_EXEC]
sql-slow:
channels: [SQL_PERF]
sql-slow-internal-only:
channels: [SQL_INTERNAL_PERF]
telemetry:
channels: [TELEMETRY]
max-file-size: 100KiB
max-group-size: 1.0MiB
stderr:
channels: all
filter: NONE
redact: false
redactable: true
exit-on-error: true
capture-stray-errors:
enable: true
max-group-size: 100MiB
workers/datalake/schema/optimizations.sql
-11
View File
@@ -1,11 +0,0 @@
-- see https://www.cockroachlabs.com/docs/v21.2/local-testing.html#use-a-local-single-node-cluster-with-in-memory-storage
SET CLUSTER SETTING kv.raft_log.disable_synchronization_unsafe = true;
SET CLUSTER SETTING kv.range_merge.queue_interval = '50ms';
SET CLUSTER SETTING jobs.registry.interval.gc = '30s';
SET CLUSTER SETTING jobs.registry.interval.cancel = '180s';
SET CLUSTER SETTING jobs.retention_time = '15s';
--SET CLUSTER SETTING schemachanger.backfiller.buffer_increment = '128 KiB';
SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false;
SET CLUSTER SETTING kv.range_split.by_load_merge_delay = '5s';
ALTER RANGE default CONFIGURE ZONE USING "gc.ttlseconds" = 600;
ALTER DATABASE system CONFIGURE ZONE USING "gc.ttlseconds" = 600;
workers/datalake/src/blob.ts
-397
View File
@@ -1,397 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { error, json } from 'itty-router'
import { type BlobDB, withPostgres } from './db'
import { cacheControl, hashLimit } from './const'
import { toUUID } from './encodings'
import { getSha256 } from './hash'
import { selectStorage } from './storage'
import { type BlobRequest, type WorkspaceRequest, type UUID } from './types'
import { copyVideo, deleteVideo } from './video'
import { type MetricsContext, LoggedCache } from './metrics'
export interface BlobMetadata {
lastModified: number
type: string
size: number
name: string
etag: string
}
export function getBlobURL (request: Request, workspace: string, name: string): string {
const path = `/blob/${workspace}/${name}`
return new URL(path, request.url).toString()
}
export async function handleBlobList (
request: WorkspaceRequest,
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext
): Promise<Response> {
const { workspace } = request
const cursor = extractStrParam(request.query.cursor)
const limit = extractIntParam(request.query.limit)
const response = await withPostgres(env, ctx, metrics, (db) => {
return db.listBlobs(workspace, cursor, limit)
})
const blobs = response.blobs.map((blob) => {
const { name, size, type, hash } = blob
return { name, size, type, etag: hash }
})
return json({ blobs, cursor: response.cursor })
}
export async function handleBlobGet (
request: BlobRequest,
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext
): Promise<Response> {
const { workspace, name } = request
const cache = new LoggedCache(caches.default, metrics)
const cacheControl = request.headers.get('Cache-Control') ?? ''
if (!cacheControl.includes('no-cache')) {
const cached = await cache.match(request)
if (cached !== undefined) {
return cached
}
}
const { bucket } = selectStorage(env, workspace)
const blob = await withPostgres(env, ctx, metrics, (db) => {
return db.getBlob({ workspace, name })
})
if (blob === null || blob.deleted) {
return error(404)
}
const range = request.headers.has('Range') ? request.headers : undefined
const object = await bucket.get(blob.filename, { range })
if (object === null) {
return error(404)
}
const headers = r2MetadataHeaders(blob.hash, object)
if (range !== undefined && object?.range !== undefined) {
headers.set('Content-Range', rangeHeader(object.range, object.size))
}
const length = object?.range !== undefined && 'length' in object.range ? object?.range?.length : undefined
const status = length !== undefined && length < object.size ? 206 : 200
if (length !== undefined && length < object.size) {
// for partial content use etag returned by R2
headers.set('ETag', object.httpEtag)
}
const response = new Response(object?.body, { headers, status })
if (response.status === 200) {
if (!cacheControl.includes('no-store')) {
const clone = metrics.withSync('response.clone', () => response.clone())
ctx.waitUntil(cache.put(request, clone))
}
}
return response
}
export async function handleBlobHead (
request: BlobRequest,
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext
): Promise<Response> {
const { workspace, name } = request
const { bucket } = selectStorage(env, workspace)
const blob = await withPostgres(env, ctx, metrics, (db) => {
return db.getBlob({ workspace, name })
})
if (blob === null || blob.deleted) {
return error(404)
}
const head = await bucket.head(blob.filename)
if (head?.httpMetadata === undefined) {
return error(404)
}
const headers = r2MetadataHeaders(blob.hash, head)
return new Response(null, { headers, status: 200 })
}
export async function handleBlobDelete (
request: BlobRequest,
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext
): Promise<Response> {
const { workspace, name } = request
try {
await withPostgres(env, ctx, metrics, (db) => {
return Promise.all([db.deleteBlob({ workspace, name }), deleteVideo(env, workspace, name)])
})
return new Response(null, { status: 204 })
} catch (err: any) {
const message = err instanceof Error ? err.message : String(err)
console.error({ error: 'failed to delete blob:' + message })
return error(500)
}
}
export async function handleUploadFormData (
request: WorkspaceRequest,
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext
): Promise<Response> {
const contentType = request.headers.get('Content-Type')
if (contentType === null || !contentType.includes('multipart/form-data')) {
console.error({ error: 'expected multipart/form-data' })
return error(400, 'expected multipart/form-data')
}
const { workspace } = request
let formData: FormData
try {
formData = await metrics.with('request.formData', () => request.formData())
} catch (err: any) {
const message = err instanceof Error ? err.message : String(err)
console.error({ error: 'failed to parse form data', message })
return error(400, 'failed to parse form data')
}
const files: [File, key: string][] = []
formData.forEach((value: any, key: string) => {
if (typeof value === 'object') files.push([value, key])
})
const result = await Promise.all(
files.map(async ([file, key]) => {
const { name, type, lastModified } = file
try {
const metadata = await withPostgres(env, ctx, metrics, (db) => {
return saveBlob(env, db, file.stream(), file.size, type, workspace, name, lastModified)
})
// TODO this probably should happen via queue, let it be here for now
if (type.startsWith('video/')) {
const blobURL = getBlobURL(request, workspace, name)
await copyVideo(env, blobURL, workspace, name)
}
return { key, metadata }
} catch (err: any) {
const error = err instanceof Error ? err.message : String(err)
console.error('failed to upload blob:', error)
return { key, error }
}
})
)
return json(result)
}
export async function saveBlob (
env: Env,
db: BlobDB,
stream: ReadableStream,
size: number,
type: string,
workspace: string,
name: string,
lastModified: number
): Promise<BlobMetadata> {
const { location, bucket } = selectStorage(env, workspace)
const httpMetadata = { contentType: type, cacheControl, lastModified }
const filename = getUniqueFilename()
const blob = await db.getBlob({ workspace, name })
if (size <= hashLimit) {
const [hashStream, uploadStream] = stream.tee()
const hash = await getSha256(hashStream)
// Check if we have the same blob already
if (blob?.hash === hash && blob?.type === type) {
return { type, size, lastModified, name, etag: hash }
}
const data = await db.getData({ hash, location })
if (data !== null) {
// Lucky boy, nothing to upload, use existing blob
await db.createBlob({ workspace, name, hash, location })
return { type, size, lastModified, name, etag: data.hash }
} else {
await bucket.put(filename, uploadStream, { httpMetadata })
await db.createData({ hash, location, filename, type, size })
await db.createBlob({ workspace, name, hash, location })
return { type, size, lastModified, name, etag: hash }
}
} else {
// For large files we cannot calculate checksum beforehead
// upload file with unique filename and then obtain checksum
const { hash } = await uploadLargeFile(bucket, stream, filename, { httpMetadata })
const data = await db.getData({ hash, location })
if (data !== null) {
// We found an existing blob with the same hash
// we can safely remove the existing blob from storage
await Promise.all([bucket.delete(filename), db.createBlob({ workspace, name, hash, location })])
return { type, size, lastModified, name, etag: hash }
} else {
// Otherwise register a new hash and blob
await db.createData({ hash, location, filename, type, size })
await db.createBlob({ workspace, name, hash, location })
return { type, size, lastModified, name, etag: hash }
}
}
}
export async function handleBlobUploaded (
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext,
workspace: string,
name: string,
filename: UUID
): Promise<BlobMetadata> {
const { location, bucket } = selectStorage(env, workspace)
const object = await bucket.head(filename)
if (object?.httpMetadata === undefined) {
throw Error('blob not found')
}
const hash = object.checksums.md5 !== undefined ? digestToUUID(object.checksums.md5) : (crypto.randomUUID() as UUID)
const size = object.size
const type = object.httpMetadata?.contentType ?? 'application/octet-stream'
await withPostgres(env, ctx, metrics, async (db) => {
const data = await db.getData({ hash, location })
if (data !== null) {
await Promise.all([bucket.delete(filename), db.createBlob({ workspace, name, hash, location })])
} else {
await db.createData({ hash, location, filename, type, size })
await db.createBlob({ workspace, name, hash, location })
}
})
return { type, size, name, etag: hash, lastModified: object.uploaded.getTime() }
}
async function uploadLargeFile (
bucket: R2Bucket,
stream: ReadableStream,
filename: string,
options: R2PutOptions
): Promise<{ hash: UUID }> {
const digestStream = new crypto.DigestStream('SHA-256')
const [digestFS, uploadFS] = stream.tee()
const digestPromise = digestFS.pipeTo(digestStream)
const uploadPromise = bucket.put(filename, uploadFS, options)
await Promise.all([digestPromise, uploadPromise])
const hash = digestToUUID(await digestStream.digest)
return { hash }
}
function getUniqueFilename (): UUID {
return crypto.randomUUID() as UUID
}
function digestToUUID (digest: ArrayBuffer): UUID {
return toUUID(new Uint8Array(digest))
}
function rangeHeader (range: R2Range, size: number): string {
const offset = 'offset' in range ? range.offset : undefined
const length = 'length' in range ? range.length : undefined
const suffix = 'suffix' in range ? range.suffix : undefined
const start = suffix !== undefined ? size - suffix : offset ?? 0
const end = suffix !== undefined ? size : length !== undefined ? start + length : size
return `bytes ${start}-${end - 1}/${size}`
}
function r2MetadataHeaders (hash: string, head: R2Object): Headers {
return head.httpMetadata !== undefined
? new Headers({
'Accept-Ranges': 'bytes',
'Content-Length': head.size.toString(),
'Content-Type': head.httpMetadata.contentType ?? '',
'Content-Security-Policy': "default-src 'none';",
'Cache-Control': head.httpMetadata.cacheControl ?? cacheControl,
'Last-Modified': head.uploaded.toUTCString(),
ETag: hash
})
: new Headers({
'Accept-Ranges': 'bytes',
'Content-Length': head.size.toString(),
'Content-Security-Policy': "default-src 'none';",
'Cache-Control': cacheControl,
'Last-Modified': head.uploaded.toUTCString(),
ETag: hash
})
}
function extractStrParam (value: string | string[] | undefined): string | undefined {
if (Array.isArray(value)) {
return value[0]
}
return value
}
function extractIntParam (value: string | string[] | undefined): number | undefined {
if (value === undefined) {
return undefined
}
if (Array.isArray(value)) {
value = value[0]
}
const intValue = Number.parseInt(value)
if (Number.isInteger(intValue)) {
return intValue
}
return undefined
}
workers/datalake/src/const.ts
-19
View File
@@ -1,19 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export const expires = 86400
export const cacheControl = `public,max-age=${expires}`
export const hashLimit = 1 * 1024 * 1024
workers/datalake/src/cors.ts
-103
View File
@@ -1,103 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type IRequest } from 'itty-router'
// This is a copy of cors.ts from itty-router with following issues fixed:
// - https://github.com/kwhitley/itty-router/issues/242
// - https://github.com/kwhitley/itty-router/issues/249
export interface CorsOptions {
credentials?: true
origin?: boolean | string | string[] | RegExp | ((origin: string) => string | undefined)
maxAge?: number
allowMethods?: string | string[]
allowHeaders?: any
exposeHeaders?: string | string[]
}
export type Preflight = (request: IRequest) => Response | undefined
export type Corsify = (response: Response, request?: IRequest) => Response | undefined
export interface CorsPair {
preflight: Preflight
corsify: Corsify
}
// Create CORS function with default options.
export const cors = (options: CorsOptions = {}): CorsPair => {
// Destructure and set defaults for options.
const { origin = '*', credentials = false, allowMethods = '*', allowHeaders, exposeHeaders, maxAge } = options
const getAccessControlOrigin = (request?: Request): string | null | undefined => {
const requestOrigin = request?.headers.get('origin') // may be null if no request passed
if (requestOrigin === undefined || requestOrigin === null) return requestOrigin
if (origin === true) return requestOrigin
if (origin instanceof RegExp) return origin.test(requestOrigin) ? requestOrigin : undefined
if (Array.isArray(origin)) return origin.includes(requestOrigin) ? requestOrigin : undefined
if (origin instanceof Function) return origin(requestOrigin) ?? undefined
return origin === '*' && credentials ? requestOrigin : (origin as string)
}
const appendHeadersAndReturn = (response: Response, headers: Record<string, any>): Response => {
for (const [key, value] of Object.entries(headers)) {
if (value !== undefined && value !== null && value !== '') {
response.headers.append(key, value)
}
}
return response
}
const preflight = (request: Request): Response | undefined => {
if (request.method === 'OPTIONS') {
const response = new Response(null, { status: 204 })
const allowMethodsHeader = Array.isArray(allowMethods) ? allowMethods.join(',') : allowMethods
const allowHeadersHeader = Array.isArray(allowHeaders) ? allowHeaders.join(',') : allowHeaders
const exposeHeadersHeader = Array.isArray(exposeHeaders) ? exposeHeaders.join(',') : exposeHeaders
return appendHeadersAndReturn(response, {
'access-control-allow-origin': getAccessControlOrigin(request),
'access-control-allow-methods': allowMethodsHeader,
'access-control-expose-headers': exposeHeadersHeader,
'access-control-allow-headers': allowHeadersHeader ?? request.headers.get('access-control-request-headers'),
'access-control-max-age': maxAge,
'access-control-allow-credentials': credentials
})
} // otherwise ignore
}
const corsify = (response: Response, request?: Request): Response | undefined => {
// ignore if already has CORS headers
if (response?.headers?.has('access-control-allow-origin') || response.status === 101) {
return response
}
const responseCopy = new Response(response.body, {
status: response.status,
statusText: response.statusText,
headers: response.headers
})
return appendHeadersAndReturn(responseCopy, {
'access-control-allow-origin': getAccessControlOrigin(request),
'access-control-allow-credentials': credentials
})
}
// Return corsify and preflight methods.
return { corsify, preflight }
}
workers/datalake/src/db.ts
-195
View File
@@ -1,195 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import postgres from 'postgres'
import { type MetricsContext } from './metrics'
import { type Location, type UUID } from './types'
export interface BlobDataId {
hash: UUID
location: Location
}
export interface BlobDataRecord extends BlobDataId {
filename: UUID
size: number
type: string
}
export interface BlobId {
workspace: string
name: string
}
export interface BlobRecord extends BlobId {
hash: UUID
location: Location
deleted: boolean
}
export type BlobWithDataRecord = BlobRecord & BlobDataRecord
export interface ListBlobResult {
cursor: string | undefined
blobs: BlobWithDataRecord[]
}
export async function withPostgres<T> (
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext,
fn: (db: BlobDB) => Promise<T>
): Promise<T> {
const sql = metrics.withSync('db.connect', () => {
return postgres(env.HYPERDRIVE.connectionString, {
connection: {
application_name: 'datalake'
},
fetch_types: false
})
})
const db = new LoggedDB(new PostgresDB(sql), metrics)
try {
return await fn(db)
} finally {
metrics.withSync('db.disconnect', () => {
ctx.waitUntil(sql.end({ timeout: 0 }))
})
}
}
export interface BlobDB {
getData: (dataId: BlobDataId) => Promise<BlobDataRecord | null>
createData: (data: BlobDataRecord) => Promise<void>
listBlobs: (workspace: string, cursor?: string, limit?: number) => Promise<ListBlobResult>
getBlob: (blobId: BlobId) => Promise<BlobWithDataRecord | null>
createBlob: (blob: Omit<BlobRecord, 'filename' | 'deleted'>) => Promise<void>
deleteBlob: (blob: BlobId) => Promise<void>
}
export class PostgresDB implements BlobDB {
constructor (private readonly sql: postgres.Sql) {}
async getData (dataId: BlobDataId): Promise<BlobDataRecord | null> {
const { hash, location } = dataId
const rows = await this.sql<BlobDataRecord[]>`
SELECT hash, location, filename, size, type
FROM blob.data
WHERE hash = ${hash} AND location = ${location}
`
return rows.length > 0 ? rows[0] : null
}
async createData (data: BlobDataRecord): Promise<void> {
const { hash, location, filename, size, type } = data
await this.sql`
UPSERT INTO blob.data (hash, location, filename, size, type)
VALUES (${hash}, ${location}, ${filename}, ${size}, ${type})
`
}
async getBlob (blobId: BlobId): Promise<BlobWithDataRecord | null> {
const { workspace, name } = blobId
try {
const rows = await this.sql<BlobWithDataRecord[]>`
SELECT b.workspace, b.name, b.hash, b.location, b.deleted, d.filename, d.size, d.type
FROM blob.blob AS b
JOIN blob.data AS d ON b.hash = d.hash AND b.location = d.location
WHERE b.workspace = ${workspace} AND b.name = ${name}
`
if (rows.length > 0) {
return rows[0]
}
} catch (err) {
console.error(err)
}
return null
}
async listBlobs (workspace: string, cursor?: string, limit?: number): Promise<ListBlobResult> {
cursor = cursor ?? ''
limit = Math.min(limit ?? 100, 1000)
const rows = await this.sql<BlobWithDataRecord[]>`
SELECT b.workspace, b.name, b.hash, b.location, b.deleted, d.filename, d.size, d.type
FROM blob.blob AS b
JOIN blob.data AS d ON b.hash = d.hash AND b.location = d.location
WHERE b.workspace = ${workspace} AND b.name > ${cursor} AND b.deleted = false
ORDER BY b.workspace, b.name
LIMIT ${limit}
`
return {
cursor: rows.length > 0 ? rows[rows.length - 1].name : undefined,
blobs: rows
}
}
async createBlob (blob: Omit<BlobRecord, 'filename' | 'deleted'>): Promise<void> {
const { workspace, name, hash, location } = blob
await this.sql`
UPSERT INTO blob.blob (workspace, name, hash, location, deleted)
VALUES (${workspace}, ${name}, ${hash}, ${location}, false)
`
}
async deleteBlob (blob: BlobId): Promise<void> {
const { workspace, name } = blob
await this.sql`
UPDATE blob.blob
SET deleted = true
WHERE workspace = ${workspace} AND name = ${name}
`
}
}
export class LoggedDB implements BlobDB {
constructor (
private readonly db: BlobDB,
private readonly ctx: MetricsContext
) {}
async getData (dataId: BlobDataId): Promise<BlobDataRecord | null> {
return await this.ctx.with('db.getData', () => this.db.getData(dataId))
}
async createData (data: BlobDataRecord): Promise<void> {
await this.ctx.with('db.createData', () => this.db.createData(data))
}
async getBlob (blobId: BlobId): Promise<BlobWithDataRecord | null> {
return await this.ctx.with('db.getBlob', () => this.db.getBlob(blobId))
}
async listBlobs (workspace: string, cursor?: string, limit?: number): Promise<ListBlobResult> {
return await this.ctx.with('db.listBlobs', () => this.db.listBlobs(workspace, cursor, limit))
}
async createBlob (blob: Omit<BlobRecord, 'filename' | 'deleted'>): Promise<void> {
await this.ctx.with('db.createBlob', () => this.db.createBlob(blob))
}
async deleteBlob (blob: BlobId): Promise<void> {
await this.ctx.with('db.deleteBlob', () => this.db.deleteBlob(blob))
}
}
workers/datalake/src/encodings.ts
-35
View File
@@ -1,35 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type UUID } from './types'
export const toUUID = (buffer: Uint8Array): UUID => {
const hex = toHex(buffer)
const hex32 = hex.slice(0, 32).padStart(32, '0')
return formatHexAsUUID(hex32)
}
export const toHex = (buffer: Uint8Array): string => {
return Array.from(buffer)
.map((b) => b.toString(16).padStart(2, '0'))
.join('')
}
export function formatHexAsUUID (hexString: string): UUID {
if (hexString.length !== 32) {
throw new Error('Hex string must be exactly 32 characters long.')
}
return hexString.replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, '$1-$2-$3-$4-$5') as UUID
}
workers/datalake/src/hash.ts
-25
View File
@@ -1,25 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { toUUID } from './encodings'
import { type UUID } from './types'
export async function getSha256 (stream: ReadableStream): Promise<UUID> {
const digestStream = new crypto.DigestStream('SHA-256')
await stream.pipeTo(digestStream)
const digest = await digestStream.digest
return toUUID(new Uint8Array(digest))
}
workers/datalake/src/image.ts
-58
View File
@@ -1,58 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { getBlobURL } from './blob'
import { type MetricsContext } from './metrics'
import { type BlobRequest } from './types'
const prefferedImageFormats = ['webp', 'avif', 'jpeg', 'png']
export async function handleImageGet (
request: BlobRequest,
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext
): Promise<Response> {
const {
workspace,
name,
params: { transform }
} = request
const Accept = request.headers.get('Accept') ?? 'image/*'
const image: Record<string, string> = {}
// select format based on Accept header
const formats = Accept.split(',')
for (const format of formats) {
const [type] = format.split(';')
const [clazz, kind] = type.split('/')
if (clazz === 'image' && prefferedImageFormats.includes(kind)) {
image.format = kind
break
}
}
// apply transforms
transform.split(',').reduce((acc, param) => {
const [key, value] = param.split('=')
acc[key] = value
return acc
}, image)
const blobURL = getBlobURL(request, workspace, name)
const imageRequest = new Request(blobURL, { headers: { Accept } })
return await metrics.with('image.transform', () => fetch(imageRequest, { cf: { image, cacheTtl: 3600 } }))
}
workers/datalake/src/index.ts
-145
View File
@@ -1,145 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { WorkerEntrypoint } from 'cloudflare:workers'
import { type IRequest, type IRequestStrict, type RequestHandler, Router, error, html } from 'itty-router'
import { handleBlobDelete, handleBlobGet, handleBlobHead, handleBlobList, handleUploadFormData } from './blob'
import { cors } from './cors'
import { LoggedKVNamespace, LoggedR2Bucket, MetricsContext } from './metrics'
import { handleImageGet } from './image'
import { handleS3Blob } from './s3'
import { handleVideoMetaGet } from './video'
import { handleSignAbort, handleSignComplete, handleSignCreate } from './sign'
import {
handleMultipartUploadStart,
handleMultipartUploadPart,
handleMultipartUploadComplete,
handleMultipartUploadAbort
} from './multipart'
import { type BlobRequest, type WorkspaceRequest } from './types'
const { preflight, corsify } = cors({
maxAge: 86400
})
const router = Router<IRequestStrict, [Env, ExecutionContext], Response>({
before: [preflight],
finally: [corsify]
})
const withWorkspace: RequestHandler<WorkspaceRequest> = (request: WorkspaceRequest) => {
if (request.params.workspace === undefined || request.params.workspace === '') {
return error(400, 'Missing workspace')
}
request.workspace = decodeURIComponent(request.params.workspace)
}
const withBlob: RequestHandler<BlobRequest> = (request: BlobRequest) => {
if (request.params.workspace === undefined || request.params.workspace === '') {
return error(400, 'Missing workspace')
}
if (request.params.name === undefined || request.params.name === '') {
return error(400, 'Missing blob name')
}
request.workspace = decodeURIComponent(request.params.workspace)
request.name = decodeURIComponent(request.params.name)
}
router
.get('/blob/:workspace', withWorkspace, handleBlobList)
.get('/blob/:workspace/:name', withBlob, handleBlobGet)
.get('/blob/:workspace/:name/:filename', withBlob, handleBlobGet)
.head('/blob/:workspace/:name', withBlob, handleBlobHead)
.head('/blob/:workspace/:name/:filename', withBlob, handleBlobHead)
.delete('/blob/:workspace/:name', withBlob, handleBlobDelete)
// Image
.get('/image/:transform/:workspace/:name', withBlob, handleImageGet)
// Video
.get('/video/:workspace/:name/meta', withBlob, handleVideoMetaGet)
// Form Data
.post('/upload/form-data/:workspace', withWorkspace, handleUploadFormData)
// Signed URL
.post('/upload/signed-url/:workspace/:name', withBlob, handleSignCreate)
.put('/upload/signed-url/:workspace/:name', withBlob, handleSignComplete)
.delete('/upload/signed-url/:workspace/:name', withBlob, handleSignAbort)
// Multipart
.post('/upload/multipart/:workspace/:name', withBlob, handleMultipartUploadStart)
.post('/upload/multipart/:workspace/:name/part', withBlob, handleMultipartUploadPart)
.post('/upload/multipart/:workspace/:name/complete', withBlob, handleMultipartUploadComplete)
.post('/upload/multipart/:workspace/:name/abort', withBlob, handleMultipartUploadAbort)
// S3
.post('/upload/s3/:workspace/:name', withBlob, handleS3Blob)
.all('/', () =>
html(
`Huly&reg; Datalake&trade; <a href="https://huly.io">https://huly.io</a>
&copy; 2024 <a href="https://hulylabs.com">Huly Labs</a>`
)
)
.all('*', () => error(404))
export default class DatalakeWorker extends WorkerEntrypoint<Env> {
async fetch (request: IRequest): Promise<Response> {
const start = performance.now()
const context = new MetricsContext()
const env = {
...this.env,
datalake_blobs: new LoggedKVNamespace(this.env.datalake_blobs, context),
DATALAKE_APAC: new LoggedR2Bucket(this.env.DATALAKE_APAC, context),
DATALAKE_EEUR: new LoggedR2Bucket(this.env.DATALAKE_EEUR, context),
DATALAKE_WEUR: new LoggedR2Bucket(this.env.DATALAKE_WEUR, context),
DATALAKE_ENAM: new LoggedR2Bucket(this.env.DATALAKE_ENAM, context),
DATALAKE_WNAM: new LoggedR2Bucket(this.env.DATALAKE_WNAM, context)
}
try {
return await router.fetch(request, env, this.ctx, context).catch(error)
} finally {
const total = performance.now() - start
const ops = context.metrics
const url = `${request.method} ${request.url}`
const message = `total=${total} ` + context.toString()
console.log({ message, total, ops, url })
}
}
async getBlob (workspace: string, name: string): Promise<ArrayBuffer> {
const request = new Request(`https://datalake/blob/${workspace}/${name}`)
const response = await router.fetch(request)
if (!response.ok) {
console.error({ error: 'datalake error: ' + response.statusText, workspace, name })
throw new Error(`Failed to fetch blob: ${response.statusText}`)
}
return await response.arrayBuffer()
}
async putBlob (workspace: string, name: string, data: ArrayBuffer | Blob | string, type: string): Promise<void> {
const request = new Request(`https://datalake/upload/form-data/${workspace}`)
const body = new FormData()
const blob = new Blob([data], { type })
body.set('file', blob, name)
const response = await router.fetch(request, { method: 'POST', body })
if (!response.ok) {
console.error({ error: 'datalake error: ' + response.statusText, workspace, name })
throw new Error(`Failed to fetch blob: ${response.statusText}`)
}
}
}
workers/datalake/src/metrics.ts
-188
View File
@@ -1,188 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the 'License');
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an 'AS IS' BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export interface MetricsData {
name: string
time: number
}
export class MetricsContext {
metrics: Array<MetricsData> = []
async with<T>(name: string, fn: () => Promise<T>): Promise<T> {
const start = performance.now()
try {
return await fn()
} finally {
const time = performance.now() - start
this.metrics.push({ name, time })
}
}
withSync<T>(name: string, fn: () => T): T {
const start = performance.now()
try {
return fn()
} finally {
const time = performance.now() - start
this.metrics.push({ name, time })
}
}
toString (): string {
return this.metrics.map((p) => `${p.name}=${p.time}`).join(' ')
}
}
export class LoggedR2Bucket implements R2Bucket {
constructor (
private readonly bucket: R2Bucket,
private readonly ctx: MetricsContext
) {}
async head (key: string): Promise<R2Object | null> {
return await this.ctx.with('r2.head', () => this.bucket.head(key))
}
async get (
key: string,
options?: R2GetOptions & {
onlyIf?: R2Conditional | Headers
}
): Promise<R2ObjectBody | null> {
return await this.ctx.with('r2.get', () => this.bucket.get(key, options))
}
async put (
key: string,
value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob,
options?: R2PutOptions & {
onlyIf?: R2Conditional | Headers
}
): Promise<R2Object> {
return await this.ctx.with('r2.put', () => this.bucket.put(key, value, options))
}
async createMultipartUpload (key: string, options?: R2MultipartOptions): Promise<R2MultipartUpload> {
return await this.ctx.with('r2.createMultipartUpload', () => this.bucket.createMultipartUpload(key, options))
}
resumeMultipartUpload (key: string, uploadId: string): R2MultipartUpload {
return this.ctx.withSync('r2.resumeMultipartUpload', () => this.bucket.resumeMultipartUpload(key, uploadId))
}
async delete (keys: string | string[]): Promise<void> {
await this.ctx.with('r2.delete', () => this.bucket.delete(keys))
}
async list (options?: R2ListOptions): Promise<R2Objects> {
return await this.ctx.with('r2.list', () => this.bucket.list(options))
}
}
export class LoggedKVNamespace implements KVNamespace {
constructor (
private readonly kv: KVNamespace,
private readonly ctx: MetricsContext
) {}
get (key: string, options?: Partial<KVNamespaceGetOptions<undefined>>): Promise<string | null>
get (key: string, type: 'text'): Promise<string | null>
get<ExpectedValue = unknown>(key: string, type: 'json'): Promise<ExpectedValue | null>
get (key: string, type: 'arrayBuffer'): Promise<ArrayBuffer | null>
get (key: string, type: 'stream'): Promise<ReadableStream | null>
get (key: string, options?: KVNamespaceGetOptions<'text'>): Promise<string | null>
get<ExpectedValue = unknown>(key: string, options?: KVNamespaceGetOptions<'json'>): Promise<ExpectedValue | null>
get (key: string, options?: KVNamespaceGetOptions<'arrayBuffer'>): Promise<ArrayBuffer | null>
get (key: string, options?: KVNamespaceGetOptions<'stream'>): Promise<ReadableStream | null>
async get (key: string, options?: any): Promise<any> {
return await this.ctx.with('kv.get', () => this.kv.get(key, options))
}
getWithMetadata<Metadata = unknown>(
key: string,
options?: Partial<KVNamespaceGetOptions<undefined>>
): Promise<KVNamespaceGetWithMetadataResult<string, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
type: 'text'
): Promise<KVNamespaceGetWithMetadataResult<string, Metadata>>
getWithMetadata<ExpectedValue = unknown, Metadata = unknown>(
key: string,
type: 'json'
): Promise<KVNamespaceGetWithMetadataResult<ExpectedValue, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
type: 'arrayBuffer'
): Promise<KVNamespaceGetWithMetadataResult<ArrayBuffer, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
type: 'stream'
): Promise<KVNamespaceGetWithMetadataResult<ReadableStream, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
options?: KVNamespaceGetOptions<'text'>
): Promise<KVNamespaceGetWithMetadataResult<string, Metadata>>
getWithMetadata<ExpectedValue = unknown, Metadata = unknown>(
key: string,
options?: KVNamespaceGetOptions<'json'>
): Promise<KVNamespaceGetWithMetadataResult<ExpectedValue, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
options?: KVNamespaceGetOptions<'arrayBuffer'>
): Promise<KVNamespaceGetWithMetadataResult<ArrayBuffer, Metadata>>
getWithMetadata<Metadata = unknown>(
key: string,
options?: KVNamespaceGetOptions<'stream'>
): Promise<KVNamespaceGetWithMetadataResult<ReadableStream, Metadata>>
async getWithMetadata (key: string, options?: any): Promise<any> {
return await this.ctx.with('kv.getWithMetadata', () => this.kv.getWithMetadata(key, options))
}
async list<Metadata = unknown>(options?: KVNamespaceListOptions): Promise<KVNamespaceListResult<Metadata, string>> {
return await this.ctx.with('kv.list', () => this.kv.list(options))
}
async put (
key: string,
value: string | ArrayBuffer | ArrayBufferView | ReadableStream,
options?: KVNamespacePutOptions
): Promise<void> {
await this.ctx.with('kv.put', () => this.kv.put(key, value))
}
async delete (key: string): Promise<void> {
await this.ctx.with('kv.delete', () => this.kv.delete(key))
}
}
export class LoggedCache implements Cache {
constructor (
private readonly cache: Cache,
private readonly ctx: MetricsContext
) {}
async match (request: RequestInfo, options?: CacheQueryOptions): Promise<Response | undefined> {
return await this.ctx.with('cache.match', () => this.cache.match(request, options))
}
async delete (request: RequestInfo, options?: CacheQueryOptions): Promise<boolean> {
return await this.ctx.with('cache.delete', () => this.cache.delete(request, options))
}
async put (request: RequestInfo, response: Response): Promise<void> {
await this.ctx.with('cache.put', () => this.cache.put(request, response))
}
}
workers/datalake/src/multipart.ts
-152
View File
@@ -1,152 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { error, json } from 'itty-router'
import { type BlobMetadata } from './blob'
import { withPostgres } from './db'
import { cacheControl } from './const'
import { toUUID } from './encodings'
import { type MetricsContext } from './metrics'
import { selectStorage } from './storage'
import { type BlobRequest, type UUID } from './types'
export interface MultipartUpload {
key: string
uploadId: string
}
export interface MultipartUploadPart {
partNumber: number
etag: string
}
export interface MultipartUploadCompleteRequest {
parts: MultipartUploadPart[]
}
export async function handleMultipartUploadStart (
request: BlobRequest,
env: Env,
ctx: ExecutionContext
): Promise<Response> {
const { workspace } = request
const { bucket } = selectStorage(env, workspace)
const contentType = request.headers.get('content-type') ?? 'application/octet-stream'
const lastModifiedHeader = request.headers.get('last-modified')
const lastModified = lastModifiedHeader !== null ? new Date(lastModifiedHeader).getTime() : Date.now()
const httpMetadata = { contentType, cacheControl, lastModified }
const uuid = crypto.randomUUID() as UUID
const multipart = await bucket.createMultipartUpload(uuid, { httpMetadata })
return json({ key: multipart.key, uploadId: multipart.uploadId })
}
export async function handleMultipartUploadPart (
request: BlobRequest,
env: Env,
ctx: ExecutionContext
): Promise<Response> {
const { workspace } = request
const multipartKey = request.query?.key
const multipartUploadId = request.query?.uploadId
const partNumber = request.query?.partNumber
if (typeof multipartKey !== 'string' || typeof multipartUploadId !== 'string' || typeof partNumber !== 'string') {
return error(400, 'missing key or uploadId or partNumber')
}
if (request.body === null) {
return error(400, 'missing body')
}
const { bucket } = selectStorage(env, workspace)
const upload = bucket.resumeMultipartUpload(multipartKey, multipartUploadId)
const part = await upload.uploadPart(Number.parseInt(partNumber), request.body)
return json({ partNumber: part.partNumber, etag: part.etag })
}
export async function handleMultipartUploadComplete (
request: BlobRequest,
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext
): Promise<Response> {
const { workspace, name } = request
const multipartKey = request.query?.key
const multipartUploadId = request.query?.uploadId
if (typeof multipartKey !== 'string' || typeof multipartUploadId !== 'string') {
return error(400, 'missing key or uploadId')
}
const { parts } = await request.json<MultipartUploadCompleteRequest>()
const { bucket, location } = selectStorage(env, workspace)
const upload = bucket.resumeMultipartUpload(multipartKey, multipartUploadId)
const object = await upload.complete(parts)
const hash =
object.checksums.md5 !== undefined ? toUUID(new Uint8Array(object.checksums.md5)) : (crypto.randomUUID() as UUID)
const type = object.httpMetadata?.contentType ?? 'application/octet-stream'
const size = object.size ?? 0
const filename = multipartKey as UUID
await withPostgres(env, ctx, metrics, async (db) => {
const data = await db.getData({ hash, location })
if (data !== null) {
// blob already exists
await Promise.all([bucket.delete(filename), db.createBlob({ workspace, name, hash, location })])
} else {
// Otherwise register a new hash and blob
await db.createData({ hash, location, filename, type, size })
await db.createBlob({ workspace, name, hash, location })
}
})
const metadata: BlobMetadata = {
type,
size,
name,
etag: hash,
lastModified: object.uploaded.getTime()
}
return json(metadata)
}
export async function handleMultipartUploadAbort (
request: BlobRequest,
env: Env,
ctx: ExecutionContext
): Promise<Response> {
const { workspace } = request
const multipartKey = request.query?.key
const multipartUploadId = request.query?.uploadId
if (typeof multipartKey !== 'string' || typeof multipartUploadId !== 'string') {
return error(400, 'missing key or uploadId')
}
const { bucket } = selectStorage(env, workspace)
const upload = bucket.resumeMultipartUpload(multipartKey, multipartUploadId)
await upload.abort()
return new Response(null, { status: 204 })
}
workers/datalake/src/s3.ts
-77
View File
@@ -1,77 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { AwsClient } from 'aws4fetch'
import { error, json } from 'itty-router'
import { withPostgres } from './db'
import { saveBlob } from './blob'
import { type MetricsContext } from './metrics'
import { type BlobRequest } from './types'
export interface S3UploadPayload {
url: string
region: string
accessKeyId: string
secretAccessKey: string
}
function getS3Client (payload: S3UploadPayload): AwsClient {
return new AwsClient({
service: 's3',
region: payload.region,
accessKeyId: payload.accessKeyId,
secretAccessKey: payload.secretAccessKey
})
}
export async function handleS3Blob (
request: BlobRequest,
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext
): Promise<Response> {
const { workspace, name } = request
const payload = await request.json<S3UploadPayload>()
const client = getS3Client(payload)
return await withPostgres(env, ctx, metrics, async (db) => {
// Ensure the blob does not exist
const blob = await db.getBlob({ workspace, name })
if (blob !== null) {
return new Response(null, { status: 200 })
}
const object = await client.fetch(payload.url)
if (!object.ok || object.status !== 200) {
return error(object.status)
}
if (object.body === null) {
return error(400)
}
const contentType = object.headers.get('content-type') ?? 'application/octet-stream'
const contentLengthHeader = object.headers.get('content-length') ?? '0'
const lastModifiedHeader = object.headers.get('last-modified')
const contentLength = Number.parseInt(contentLengthHeader)
const lastModified = lastModifiedHeader !== null ? new Date(lastModifiedHeader).getTime() : Date.now()
const result = await saveBlob(env, db, object.body, contentLength, contentType, workspace, name, lastModified)
return json(result)
})
}
workers/datalake/src/sign.ts
-137
View File
@@ -1,137 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { AwsClient } from 'aws4fetch'
import { error, json } from 'itty-router'
import { type BlobMetadata, handleBlobUploaded } from './blob'
import { type MetricsContext } from './metrics'
import { type Storage, selectStorage } from './storage'
import { type BlobRequest, type UUID } from './types'
const S3_SIGNED_LINK_TTL = 3600
interface SignBlobInfo {
uuid: UUID
}
function signBlobKey (workspace: string, name: string): string {
return `s/${workspace}/${name}`
}
function getS3Client (storage: Storage): AwsClient {
return new AwsClient({
service: 's3',
region: 'auto',
accessKeyId: storage.bucketAccessKey,
secretAccessKey: storage.bucketSecretKey
})
}
export async function handleSignCreate (
request: BlobRequest,
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext
): Promise<Response> {
const { workspace, name } = request
const storage = selectStorage(env, workspace)
const accountId = env.R2_ACCOUNT_ID
const key = signBlobKey(workspace, name)
const uuid = crypto.randomUUID() as UUID
// Generate R2 object link
const url = new URL(`https://${storage.bucketName}.${accountId}.r2.cloudflarestorage.com`)
url.pathname = uuid
url.searchParams.set('X-Amz-Expires', S3_SIGNED_LINK_TTL.toString())
// Sign R2 object link
let signed: Request
try {
const client = getS3Client(storage)
signed = await metrics.with('s3.sign', () => {
return client.sign(new Request(url, { method: 'PUT' }), { aws: { signQuery: true } })
})
} catch (err: any) {
console.error({ error: 'failed to generate signed url', message: `${err}` })
return error(500, 'failed to generate signed url')
}
// Save upload details
const s3BlobInfo: SignBlobInfo = { uuid }
await env.datalake_blobs.put(key, JSON.stringify(s3BlobInfo), { expirationTtl: S3_SIGNED_LINK_TTL })
const headers = new Headers({
Expires: new Date(Date.now() + S3_SIGNED_LINK_TTL * 1000).toISOString()
})
return new Response(signed.url, { status: 200, headers })
}
export async function handleSignComplete (
request: BlobRequest,
env: Env,
ctx: ExecutionContext,
metrics: MetricsContext
): Promise<Response> {
const { workspace, name } = request
const { bucket } = selectStorage(env, workspace)
const key = signBlobKey(workspace, name)
// Ensure we generated presigned URL earlier
// TODO what if we came after expiration date?
const signBlobInfo = await env.datalake_blobs.get<SignBlobInfo>(key, { type: 'json' })
if (signBlobInfo === null) {
console.error({ error: 'blob sign info not found', workspace, name })
return error(404)
}
// Ensure the blob has been uploaded
const { uuid } = signBlobInfo
const head = await bucket.get(uuid)
if (head === null) {
console.error({ error: 'blob not found', workspace, name, uuid })
return error(400)
}
let metadata: BlobMetadata
try {
metadata = await handleBlobUploaded(env, ctx, metrics, workspace, name, uuid)
} catch (err) {
const message = err instanceof Error ? err.message : String(err)
console.error({ error: message, workspace, name, uuid })
return error(500, 'failed to upload blob')
}
await env.datalake_blobs.delete(key)
return json(metadata)
}
export async function handleSignAbort (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
const { workspace, name } = request
const key = signBlobKey(workspace, name)
// Check if the blob has been uploaded
const s3BlobInfo = await env.datalake_blobs.get<SignBlobInfo>(key, { type: 'json' })
if (s3BlobInfo !== null) {
await env.datalake_blobs.delete(key)
}
return new Response(null, { status: 204 })
}
workers/datalake/src/storage.ts
-76
View File
@@ -1,76 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the 'License');
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an 'AS IS' BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type Location } from './types'
export interface Storage {
location: Location
bucket: R2Bucket
bucketName: string
bucketAccessKey: string
bucketSecretKey: string
}
export function selectStorage (env: Env, workspace: string): Storage {
const location = selectLocation(env, workspace)
switch (location) {
case 'apac':
return {
location,
bucket: env.DATALAKE_APAC,
bucketName: env.DATALAKE_APAC_BUCKET_NAME,
bucketAccessKey: env.DATALAKE_APAC_ACCESS_KEY,
bucketSecretKey: env.DATALAKE_APAC_SECRET_KEY
}
case 'eeur':
return {
location,
bucket: env.DATALAKE_EEUR,
bucketName: env.DATALAKE_EEUR_BUCKET_NAME,
bucketAccessKey: env.DATALAKE_EEUR_ACCESS_KEY,
bucketSecretKey: env.DATALAKE_EEUR_SECRET_KEY
}
case 'weur':
return {
location,
bucket: env.DATALAKE_WEUR,
bucketName: env.DATALAKE_WEUR_BUCKET_NAME,
bucketAccessKey: env.DATALAKE_WEUR_ACCESS_KEY,
bucketSecretKey: env.DATALAKE_WEUR_SECRET_KEY
}
case 'enam':
return {
location,
bucket: env.DATALAKE_ENAM,
bucketName: env.DATALAKE_ENAM_BUCKET_NAME,
bucketAccessKey: env.DATALAKE_ENAM_ACCESS_KEY,
bucketSecretKey: env.DATALAKE_ENAM_SECRET_KEY
}
case 'wnam':
return {
location,
bucket: env.DATALAKE_WNAM,
bucketName: env.DATALAKE_WNAM_BUCKET_NAME,
bucketAccessKey: env.DATALAKE_WNAM_ACCESS_KEY,
bucketSecretKey: env.DATALAKE_WNAM_SECRET_KEY
}
}
}
function selectLocation (env: Env, workspace: string): Location {
// TODO select location based on workspace
return 'weur'
}
workers/datalake/src/types.ts
-43
View File
@@ -1,43 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type IRequestStrict } from 'itty-router'
export type Location = 'weur' | 'eeur' | 'wnam' | 'enam' | 'apac'
export type UUID = string & { __uuid: true }
export type WorkspaceRequest = {
workspace: string
} & IRequestStrict
export type BlobRequest = {
workspace: string
name: string
} & IRequestStrict
export interface CloudflareResponse {
success: boolean
errors: any
messages: any
result: any
}
export interface StreamUploadResponse extends CloudflareResponse {
result: {
uid: string
uploadURL: string
}
}
workers/datalake/src/video.ts
-117
View File
@@ -1,117 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { error, json } from 'itty-router'
import { type BlobRequest, type CloudflareResponse, type StreamUploadResponse } from './types'
export type StreamUploadState = 'ready' | 'error' | 'inprogress' | 'queued' | 'downloading' | 'pendingupload'
// https://developers.cloudflare.com/api/operations/stream-videos-list-videos#response-body
export interface StreamDetailsResponse extends CloudflareResponse {
result: {
uid: string
thumbnail: string
status: {
state: StreamUploadState
}
playback: {
hls: string
dash: string
}
}
}
interface StreamBlobInfo {
streamId: string
}
function streamBlobKey (workspace: string, name: string): string {
return `v/${workspace}/${name}`
}
export async function handleVideoMetaGet (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise<Response> {
const { workspace, name } = request
const key = streamBlobKey(workspace, name)
const streamInfo = await env.datalake_blobs.get<StreamBlobInfo>(key, { type: 'json' })
if (streamInfo === null) {
return error(404)
}
const url = `https://api.cloudflare.com/client/v4/accounts/${env.STREAMS_ACCOUNT_ID}/stream/${streamInfo.streamId}`
const streamRequest = new Request(url, {
headers: {
Authorization: `Bearer ${env.STREAMS_AUTH_KEY}`,
'Content-Type': 'application/json'
}
})
const streamResponse = await fetch(streamRequest)
const stream = await streamResponse.json<StreamDetailsResponse>()
if (stream.success) {
return json({
status: stream.result.status.state,
thumbnail: stream.result.thumbnail,
hls: stream.result.playback.hls
})
} else {
return error(500, { errors: stream.errors })
}
}
export async function copyVideo (env: Env, source: string, workspace: string, name: string): Promise<void> {
const key = streamBlobKey(workspace, name)
const url = `https://api.cloudflare.com/client/v4/accounts/${env.STREAMS_ACCOUNT_ID}/stream/copy`
const request = new Request(url, {
method: 'POST',
headers: {
Authorization: `Bearer ${env.STREAMS_AUTH_KEY}`,
'Content-Type': 'application/json'
},
body: JSON.stringify({ url: source, meta: { name } })
})
const response = await fetch(request)
const upload = await response.json<StreamUploadResponse>()
if (upload.success) {
const streamInfo: StreamBlobInfo = {
streamId: upload.result.uid
}
await env.datalake_blobs.put(key, JSON.stringify(streamInfo))
}
}
export async function deleteVideo (env: Env, workspace: string, name: string): Promise<void> {
const key = streamBlobKey(workspace, name)
const streamInfo = await env.datalake_blobs.get<StreamBlobInfo>(key, { type: 'json' })
if (streamInfo !== null) {
const url = `https://api.cloudflare.com/client/v4/accounts/${env.STREAMS_ACCOUNT_ID}/stream/${streamInfo.streamId}`
const request = new Request(url, {
method: 'DELETE',
headers: {
Authorization: `Bearer ${env.STREAMS_AUTH_KEY}`,
'Content-Type': 'application/json'
}
})
await Promise.all([fetch(request), env.datalake_blobs.delete(key)])
}
}
workers/datalake/tsconfig.json
-12
View File
@@ -1,12 +0,0 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"declarationDir": "./types",
"tsBuildInfoFile": ".build/build.tsbuildinfo",
"types": ["@cloudflare/workers-types", "jest"],
"lib": ["esnext"]
}
}
workers/datalake/worker-configuration.d.ts
Vendored
-31
View File
@@ -1,31 +0,0 @@
// Generated by Wrangler on Sat Jul 06 2024 18:52:21 GMT+0200 (Central European Summer Time)
// by running `wrangler types`
interface Env {
datalake_blobs: KVNamespace;
DATALAKE_APAC: R2Bucket;
DATALAKE_EEUR: R2Bucket;
DATALAKE_WEUR: R2Bucket;
DATALAKE_ENAM: R2Bucket;
DATALAKE_WNAM: R2Bucket;
HYPERDRIVE: Hyperdrive;
STREAMS_ACCOUNT_ID: string;
STREAMS_AUTH_KEY: string;
R2_ACCOUNT_ID: string;
DB_URL: string;
DATALAKE_APAC_ACCESS_KEY: string;
DATALAKE_APAC_SECRET_KEY: string;
DATALAKE_APAC_BUCKET_NAME: string;
DATALAKE_EEUR_ACCESS_KEY: string;
DATALAKE_EEUR_SECRET_KEY: string;
DATALAKE_EEUR_BUCKET_NAME: string;
DATALAKE_WEUR_ACCESS_KEY: string;
DATALAKE_WEUR_SECRET_KEY: string;
DATALAKE_WEUR_BUCKET_NAME: string;
DATALAKE_ENAM_ACCESS_KEY: string;
DATALAKE_ENAM_SECRET_KEY: string;
DATALAKE_ENAM_BUCKET_NAME: string;
DATALAKE_WNAM_ACCESS_KEY: string;
DATALAKE_WNAM_SECRET_KEY: string;
DATALAKE_WNAM_BUCKET_NAME: string;
}
workers/datalake/wrangler.toml
-110
View File
@@ -1,110 +0,0 @@
#:schema node_modules/wrangler/config-schema.json
name = "datalake-worker"
main = "src/index.ts"
compatibility_date = "2024-09-23"
compatibility_flags = ["nodejs_compat"]
keep_vars = true
kv_namespaces = [
{ binding = "datalake_blobs", id = "64144eb146fd45febc928d44419ebb39", preview_id = "31c6f6e76e7e4524a59f87a4f381de82" }
]
r2_buckets = [
{ binding = "DATALAKE_APAC", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_EEUR", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_WEUR", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_ENAM", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" },
{ binding = "DATALAKE_WNAM", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" }
]
[[hyperdrive]]
binding = "HYPERDRIVE"
id = "055e968f3067414eaa30467d8a9c5021"
localConnectionString = "postgresql://root:roach@localhost:26257/datalake"
[observability]
enabled = true
head_sampling_rate = 1
[vars]
DATALAKE_EEUR_BUCKET_NAME = "datalake-eeur"
# DATALAKE_EEUR_ACCESS_KEY = ""
# DATALAKE_EEUR_SECRET_KEY = ""
DATALAKE_WEUR_BUCKET_NAME = "datalake-weur"
# DATALAKE_WEUR_ACCESS_KEY = ""
# DATALAKE_WEUR_SECRET_KEY = ""
DATALAKE_APAC_BUCKET_NAME = "datalake-apac"
# DATALAKE_APAC_ACCESS_KEY = ""
# DATALAKE_APAC_SECRET_KEY = ""
DATALAKE_ENAM_BUCKET_NAME = "datalake-enam"
# DATALAKE_ENAM_ACCESS_KEY = ""
# DATALAKE_ENAM_SECRET_KEY = ""
DATALAKE_WNAM_BUCKET_NAME = "datalake-wnam"
# DATALAKE_WNAM_ACCESS_KEY = ""
# DATALAKE_WNAM_SECRET_KEY = ""
# STREAMS_ACCOUNT_ID = ""
# STREAMS_AUTH_KEY = ""
# R2_ACCOUNT_ID = ""
[env.staging]
name = "datalake-worker-staging"
kv_namespaces = [
{ binding = "datalake_blobs", id = "d52f8b6a9a1e495a8d6625d14d8f3188" }
]
r2_buckets = [
{ binding = "DATALAKE_APAC", bucket_name = "datalake-staging-wnam" },
{ binding = "DATALAKE_EEUR", bucket_name = "datalake-staging-wnam" },
{ binding = "DATALAKE_WEUR", bucket_name = "datalake-staging-weur" },
{ binding = "DATALAKE_ENAM", bucket_name = "datalake-staging-wnam" },
{ binding = "DATALAKE_WNAM", bucket_name = "datalake-staging-wnam" }
]
hyperdrive = [
{ binding = "HYPERDRIVE", id = "1f713bb2ec1b464cb663983feca4c89e" }
]
[env.staging.vars]
DATALAKE_EEUR_BUCKET_NAME = "datalake-staging-weur"
DATALAKE_WEUR_BUCKET_NAME = "datalake-staging-wnam"
DATALAKE_APAC_BUCKET_NAME = "datalake-staging-wnam"
DATALAKE_ENAM_BUCKET_NAME = "datalake-staging-wnam"
DATALAKE_WNAM_BUCKET_NAME = "datalake-staging-wnam"
[env.staging.observability]
enabled = true
head_sampling_rate = 1
[env.dev]
name = "datalake-worker-dev"
kv_namespaces = [
{ binding = "datalake_blobs", id = "5acdcea18a504ab3a2d73556314b0047" }
]
r2_buckets = [
{ binding = "DATALAKE_APAC", bucket_name = "datalake-dev" },
{ binding = "DATALAKE_EEUR", bucket_name = "datalake-dev" },
{ binding = "DATALAKE_WEUR", bucket_name = "datalake-dev" },
{ binding = "DATALAKE_ENAM", bucket_name = "datalake-dev" },
{ binding = "DATALAKE_WNAM", bucket_name = "datalake-dev" }
]
hyperdrive = [
{ binding = "HYPERDRIVE", id = "055e968f3067414eaa30467d8a9c5021", localConnectionString = "postgresql://root:roach@localhost:26257/datalake" }
]
[env.dev.vars]
DATALAKE_EEUR_BUCKET_NAME = "datalake-dev"
DATALAKE_WEUR_BUCKET_NAME = "datalake-dev"
DATALAKE_APAC_BUCKET_NAME = "datalake-dev"
DATALAKE_ENAM_BUCKET_NAME = "datalake-dev"
DATALAKE_WNAM_BUCKET_NAME = "datalake-dev"
[env.dev.observability]
enabled = true
head_sampling_rate = 1
workers/transactor/package.json
+4 -2
View File
@@ -39,7 +39,8 @@
"ts-jest": "^29.1.1",
"typescript": "^5.3.3",
"wrangler": "^3.97.0",
"esbuild": "^0.24.2"
"esbuild": "^0.24.2",
"@types/snappyjs": "^0.7.1"
},
"dependencies": {
"@hcengineering/core": "^0.6.32",
@@ -54,6 +55,7 @@
"@hcengineering/server-pipeline": "^0.6.0",
"@hcengineering/server-token": "^0.6.11",
"@hcengineering/storage": "^0.6.0",
"itty-router": "^5.0.18"
"itty-router": "^5.0.18",
"snappyjs": "^0.7.0"
}
}
workers/transactor/src/transactor.ts
+10 -3
View File
@@ -29,6 +29,7 @@ import serverPlugin, { decodeToken, type Token } from '@hcengineering/server-tok
import { DurableObject } from 'cloudflare:workers'
import { promisify } from 'util'
import { gzip } from 'zlib'
import { compress } from 'snappyjs'
// Approach usefull only for separate build, after model-all bundle phase is executed.
import { createPostgreeDestroyAdapter, createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
@@ -111,7 +112,8 @@ export class Transactor extends DurableObject<Env> {
reconnectTimeout: 3000
},
undefined,
this.accountsUrl
this.accountsUrl,
env.ENABLE_COMPRESSION === 'true'
)
})
}
@@ -269,13 +271,18 @@ export class Transactor extends DurableObject<Env> {
return rpcHandler.readRequest(buffer, binary)
},
data: () => data,
send: (ctx: MeasureContext, msg, binary, compression) => {
const smsg = rpcHandler.serialize(msg, binary)
send: (ctx: MeasureContext, msg, binary, _compression) => {
let smsg = rpcHandler.serialize(msg, binary)
ctx.measure('send-data', smsg.length)
if (ws.readyState !== WebSocket.OPEN || cs.isClosed) {
return
}
if (_compression) {
smsg = compress(smsg)
}
ws.send(smsg)
},
sendPong: () => {
workers/transactor/worker-configuration.d.ts
Vendored
+2
View File
@@ -12,4 +12,6 @@ interface Env {
DB_URL: string | undefined
STATS_URL: string | undefined
ENABLE_COMPRESSION: string | undefined
}