WEBDIENSTE/huly-platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2026-09-13 21:27:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

UBERF-9764: Adjust gmail for new accounts (#8681)

Browse Source 
Signed-off-by: Artem Savchenko <armisav@gmail.com>
This commit is contained in:
Artyom Savchenko
2025-04-28 11:37:30 +05:00
committed by GitHub
parent 9efefe3693
commit ac249c8273
59 changed files with 4326 additions and 966 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/config/rush/pnpm-lock.yaml
Generated
+186 -6
View File
@@ -385,6 +385,9 @@ importers:
'@rush-temp/kanban':
specifier: file:./projects/kanban.tgz
version: file:projects/kanban.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.5.3)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.5.3)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/kvs-client':
specifier: file:./projects/kvs-client.tgz
version: file:projects/kvs-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
'@rush-temp/lead':
specifier: file:./projects/lead.tgz
version: file:projects/lead.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
@@ -771,7 +774,7 @@ importers:
version: file:projects/pod-github.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)(utf-8-validate@6.0.4)(y-prosemirror@1.2.15(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(y-protocols@1.0.6(yjs@13.6.23))(yjs@13.6.23))
'@rush-temp/pod-gmail':
specifier: file:./projects/pod-gmail.tgz
version: file:projects/pod-gmail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
version: file:projects/pod-gmail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)
'@rush-temp/pod-inbound-mail':
specifier: file:./projects/pod-inbound-mail.tgz
version: file:projects/pod-inbound-mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))
@@ -1870,6 +1873,9 @@ importers:
jest-environment-jsdom:
specifier: ^29.7.0
version: 29.7.0(bufferutil@4.0.8)(utf-8-validate@6.0.4)
jest-fetch-mock:
specifier: ^3.0.3
version: 3.0.3(encoding@0.1.13)
jimp:
specifier: ^0.16.1
version: 0.16.13
@@ -1942,6 +1948,9 @@ importers:
mongodb:
specifier: ^6.12.0
version: 6.12.0(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)
mongodb-memory-server:
specifier: ^10.1.4
version: 10.1.4(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)
morgan:
specifier: ^1.10.0
version: 1.10.0
@@ -4313,6 +4322,10 @@ packages:
resolution: {integrity: sha512-a1aAx7zBxeZjaejI86Ho69hHcEm7F4dkdD9rVCv2BVHzjG/1C8mq7pv+QWHFGoxkiQwwm9MatVV4IMCSjDjAZw==, tarball: file:projects/kanban.tgz}
version: 0.0.0
'@rush-temp/kvs-client@file:projects/kvs-client.tgz':
resolution: {integrity: sha512-UHZpGqCKncG8BGVHJPeceusj86Knh9SjhNvp972sLvA/FY89ph6Rpz+34X6aI3D5Cu3048ePmwszH81PKphNrQ==, tarball: file:projects/kvs-client.tgz}
version: 0.0.0
'@rush-temp/lead-assets@file:projects/lead-assets.tgz':
resolution: {integrity: sha512-VVUGNaGtZaGVHj7V/o6b0kmXeLwYxQA3LwuWvW0vbv2xdMsfZO/QC6ShBZPUSrzYPjfOWTjvS6lhH2alTanW7A==, tarball: file:projects/lead-assets.tgz}
version: 0.0.0
@@ -4826,7 +4839,7 @@ packages:
version: 0.0.0
'@rush-temp/pod-gmail@file:projects/pod-gmail.tgz':
resolution: {integrity: sha512-HyyHadfmgT1OP7pdln0WJvK6cGPTYVVvxq21JBHKshu40WojCB6dLdAch/YrTO0aSS7UzluuLmYnnCn5JGvfcg==, tarball: file:projects/pod-gmail.tgz}
resolution: {integrity: sha512-mWpD7TVaoiHBcGFgqt2KoPioV7Uc/9Y7DXu3LKiwZBrUX6X+WR/77RcKq/gNGPbwNxCKOc+dDsscF5NKMhIqSg==, tarball: file:projects/pod-gmail.tgz}
version: 0.0.0
'@rush-temp/pod-inbound-mail@file:projects/pod-inbound-mail.tgz':
@@ -5482,7 +5495,7 @@ packages:
version: 0.0.0
'@rush-temp/tool@file:projects/tool.tgz':
resolution: {integrity: sha512-zqh0++u5ETaUNlwkipP+8CeICqrFYEDBO4sAjFRyLPd1SHZbanUpQRAg58t4C+Z7q4Owqr149ee/hCGWPult3w==, tarball: file:projects/tool.tgz}
resolution: {integrity: sha512-k0KXvAT1ulcbCo/O0bYbPxjZHX2I7BAGMr7Ntdrt1xLcUBsQ3WSOUw9AxCLU4+PiJmIHQVpFEhJPNc06CeFfCA==, tarball: file:projects/tool.tgz}
version: 0.0.0
'@rush-temp/tracker-assets@file:projects/tracker-assets.tgz':
@@ -5510,7 +5523,7 @@ packages:
version: 0.0.0
'@rush-temp/ui-next@file:projects/ui-next.tgz':
resolution: {integrity: sha512-7yi52BTXL9htk9CapPOLg7a26v5GVVYSKPWqEbWrEQgin1JvjL7F7qv3EcIGVtdcCnfkvATeDXT8HLvZNg9DwA==, tarball: file:projects/ui-next.tgz}
resolution: {integrity: sha512-bisoYF6n3HWSicnaOnc1DhInOtgSsIW77QjtWRtuq3lmJMzKQ8u+RF2wqa6gUaw3Qa/qcbE+HyzkJGQyCVB0og==, tarball: file:projects/ui-next.tgz}
version: 0.0.0
'@rush-temp/ui@file:projects/ui.tgz':
@@ -7137,6 +7150,9 @@ packages:
async-mutex@0.3.2:
resolution: {integrity: sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==}
async-mutex@0.5.0:
resolution: {integrity: sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==}
async@3.2.5:
resolution: {integrity: sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==}
@@ -7644,6 +7660,9 @@ packages:
resolution: {integrity: sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==}
engines: {node: ^12.20.0 || >=14}
commondir@1.0.1:
resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==}
compare-version@0.1.2:
resolution: {integrity: sha512-pJDh5/4wrEnXX/VWRZvruAGHkzKdr46z11OlTPN+VrATlWWhSKewNCJ1futCO5C7eJB3nPMFZA1LeYtcFboZ2A==}
engines: {node: '>=0.10.0'}
@@ -9064,6 +9083,10 @@ packages:
resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==}
engines: {node: '>= 0.8'}
find-cache-dir@3.3.2:
resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==}
engines: {node: '>=8'}
find-up@3.0.0:
resolution: {integrity: sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==}
engines: {node: '>=6'}
@@ -9112,6 +9135,15 @@ packages:
debug:
optional: true
follow-redirects@1.15.9:
resolution: {integrity: sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==}
engines: {node: '>=4.0'}
peerDependencies:
debug: '*'
peerDependenciesMeta:
debug:
optional: true
for-each@0.3.3:
resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==}
@@ -10049,6 +10081,9 @@ packages:
resolution: {integrity: sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
jest-fetch-mock@3.0.3:
resolution: {integrity: sha512-Ux1nWprtLrdrH4XwE7O7InRY6psIi3GOsqNESJgMJ+M5cv4A8Lh7SN9d2V2kKRZ8ebAfcd1LNyZguAOb6JiDqw==}
jest-get-type@29.6.3:
resolution: {integrity: sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
@@ -10651,6 +10686,10 @@ packages:
resolution: {integrity: sha512-8vBuFF/I/+OSLRmdf2wwFCJCz+nSn0m6DPvGH1fS/KiQoSaR+sETbov0eIk9KhEKy8CYqIkIAnbohxT/4H0kuA==}
engines: {node: '>=12'}
make-dir@3.1.0:
resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==}
engines: {node: '>=8'}
make-dir@4.0.0:
resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==}
engines: {node: '>=10'}
@@ -10903,6 +10942,14 @@ packages:
mongodb-connection-string-url@3.0.0:
resolution: {integrity: sha512-t1Vf+m1I5hC2M5RJx/7AtxgABy1cZmIPQRMXw+gEIPn/cZNF3Oiy+l0UIypUwVB5trcWHq3crg2g3uAR9aAwsQ==}
mongodb-memory-server-core@10.1.4:
resolution: {integrity: sha512-o8fgY7ZalEd8pGps43fFPr/hkQu1L8i6HFEGbsTfA2zDOW0TopgpswaBCqDr0qD7ptibyPfB5DmC+UlIxbThzA==}
engines: {node: '>=16.20.1'}
mongodb-memory-server@10.1.4:
resolution: {integrity: sha512-+oKQ/kc3CX+816oPFRtaF0CN4vNcGKNjpOQe4bHo/21A3pMD+lC7Xz1EX5HP7siCX4iCpVchDMmCOFXVQSGkUg==}
engines: {node: '>=16.20.1'}
mongodb@6.12.0:
resolution: {integrity: sha512-RM7AHlvYfS7jv7+BXund/kR64DryVI+cHbVAy9P61fnb1RcWZqOW1/Wj2YhqMCx+MuYhqTRGv7AwHBzmsCKBfA==}
engines: {node: '>=16.20.1'}
@@ -10997,6 +11044,10 @@ packages:
resolution: {integrity: sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==}
engines: {node: '>= 0.4.0'}
new-find-package-json@2.0.0:
resolution: {integrity: sha512-lDcBsjBSMlj3LXH2v/FW3txlh2pYTjmbOXPYJD93HI5EwuLzI11tdHSIpUMmfq/IOsldj4Ps8M8flhm+pCK4Ew==}
engines: {node: '>=12.22.0'}
next-tick@1.1.0:
resolution: {integrity: sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==}
@@ -11801,6 +11852,9 @@ packages:
promise-inflight@1.0.1:
resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==}
promise-polyfill@8.3.0:
resolution: {integrity: sha512-H5oELycFml5yto/atYqmjyigJoAo3+OXwolYiH7OfQuYlAqhxNvTfiNMbV9hsC6Yp83yE5r2KTVmtrG6R9i6Pg==}
promise-retry@2.0.1:
resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==}
engines: {node: '>=10'}
@@ -12214,6 +12268,9 @@ packages:
sanitize-html@2.15.0:
resolution: {integrity: sha512-wIjst57vJGpLyBP8ioUbg6ThwJie5SuSIjHxJg53v5Fg+kUK+AXlb7bK3RNXpp315MvwM+0OBGCV6h5pPHsVhA==}
sanitize-html@2.16.0:
resolution: {integrity: sha512-0s4caLuHHaZFVxFTG74oW91+j6vW7gKbGD6CD2+miP73CE6z6YtOBN0ArtLd2UGyi4IC7K47v3ENUbQX4jV3Mg==}
sass-loader@13.3.3:
resolution: {integrity: sha512-mt5YN2F1MOZr3d/wBRcZxeFgwgkH44wVc2zohO2YF6JiOMkiXe4BYRZpSu2sO1g71mo/j16txzUhsKZlqjVGzA==}
engines: {node: '>= 14.15.0'}
@@ -13713,6 +13770,10 @@ packages:
yauzl@2.10.0:
resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==}
yauzl@3.2.0:
resolution: {integrity: sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==}
engines: {node: '>=12'}
yjs@13.6.23:
resolution: {integrity: sha512-ExtnT5WIOVpkL56bhLeisG/N5c4fmzKn4k0ROVfJa5TY2QHbH7F0Wu2T5ZhR7ErsFWQEFafyrnSI8TPKVF9Few==}
engines: {node: '>=16.0.0', npm: '>=8.0.0'}
@@ -19372,6 +19433,34 @@ snapshots:
- supports-color
- ts-node
'@rush-temp/kvs-client@file:projects/kvs-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3)
cross-env: 7.0.3
esbuild: 0.24.2
eslint: 8.56.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3)
eslint-plugin-import: 2.29.1(eslint@8.56.0)
eslint-plugin-n: 15.7.0(eslint@8.56.0)
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))
jest-fetch-mock: 3.0.3(encoding@0.1.13)
prettier: 3.2.5
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3)
typescript: 5.7.3
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
- babel-jest
- babel-plugin-macros
- encoding
- node-notifier
- supports-color
- ts-node
'@rush-temp/lead-assets@file:projects/lead-assets.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
dependencies:
'@types/jest': 29.5.12
@@ -22053,17 +22142,19 @@ snapshots:
- utf-8-validate
- y-prosemirror
'@rush-temp/pod-gmail@file:projects/pod-gmail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))':
'@rush-temp/pod-gmail@file:projects/pod-gmail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)':
dependencies:
'@tsconfig/node16': 1.0.4
'@types/cors': 2.8.17
'@types/express': 4.17.21
'@types/jest': 29.5.12
'@types/node': 20.11.19
'@types/sanitize-html': 2.15.0
'@types/uuid': 8.3.4
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3)
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
cors: 2.8.5
cross-env: 7.0.3
dotenv: 16.0.3
esbuild: 0.24.2
eslint: 8.56.0
@@ -22082,7 +22173,9 @@ snapshots:
jwt-simple: 0.5.6
mongodb: 6.12.0(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)
prettier: 3.2.5
sanitize-html: 2.16.0
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.3.3)
ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3)
ts-node-dev: 2.0.0(@types/node@20.11.19)(typescript@5.3.3)
typescript: 5.3.3
uuid: 8.3.2
@@ -22103,7 +22196,6 @@ snapshots:
- snappy
- socks
- supports-color
- ts-node
'@rush-temp/pod-inbound-mail@file:projects/pod-inbound-mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))':
dependencies:
@@ -26977,6 +27069,7 @@ snapshots:
libphonenumber-js: 1.10.56
mime-types: 2.1.35
mongodb: 6.12.0(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)
mongodb-memory-server: 10.1.4(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)
msgpackr: 1.11.2
msgpackr-extract: 3.0.3
postgres: 3.4.5
@@ -29730,6 +29823,10 @@ snapshots:
dependencies:
tslib: 2.7.0
async-mutex@0.5.0:
dependencies:
tslib: 2.8.1
async@3.2.5: {}
asynckit@0.4.0: {}
@@ -30326,6 +30423,8 @@ snapshots:
commander@9.5.0: {}
commondir@1.0.1: {}
compare-version@0.1.2: {}
component-emitter@1.3.1: {}
@@ -32105,6 +32204,12 @@ snapshots:
statuses: 2.0.1
unpipe: 1.0.0
find-cache-dir@3.3.2:
dependencies:
commondir: 1.0.1
make-dir: 3.1.0
pkg-dir: 4.2.0
find-up@3.0.0:
dependencies:
locate-path: 3.0.0
@@ -32140,6 +32245,10 @@ snapshots:
follow-redirects@1.15.6: {}
follow-redirects@1.15.9(debug@4.4.0):
optionalDependencies:
debug: 4.4.0
for-each@0.3.3:
dependencies:
is-callable: 1.2.7
@@ -33252,6 +33361,13 @@ snapshots:
jest-mock: 29.7.0
jest-util: 29.7.0
jest-fetch-mock@3.0.3(encoding@0.1.13):
dependencies:
cross-fetch: 3.1.8(encoding@0.1.13)
promise-polyfill: 8.3.0
transitivePeerDependencies:
- encoding
jest-get-type@29.6.3: {}
jest-haste-map@29.7.0:
@@ -34052,6 +34168,10 @@ snapshots:
dependencies:
'@jridgewell/sourcemap-codec': 1.4.15
make-dir@3.1.0:
dependencies:
semver: 6.3.1
make-dir@4.0.0:
dependencies:
semver: 7.6.3
@@ -34339,6 +34459,44 @@ snapshots:
'@types/whatwg-url': 11.0.4
whatwg-url: 13.0.0
mongodb-memory-server-core@10.1.4(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3):
dependencies:
async-mutex: 0.5.0
camelcase: 6.3.0
debug: 4.4.0
find-cache-dir: 3.3.2
follow-redirects: 1.15.9(debug@4.4.0)
https-proxy-agent: 7.0.5
mongodb: 6.12.0(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)
new-find-package-json: 2.0.0
semver: 7.6.3
tar-stream: 3.1.7
tslib: 2.8.1
yauzl: 3.2.0
transitivePeerDependencies:
- '@aws-sdk/credential-providers'
- '@mongodb-js/zstd'
- gcp-metadata
- kerberos
- mongodb-client-encryption
- snappy
- socks
- supports-color
mongodb-memory-server@10.1.4(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3):
dependencies:
mongodb-memory-server-core: 10.1.4(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)
tslib: 2.8.1
transitivePeerDependencies:
- '@aws-sdk/credential-providers'
- '@mongodb-js/zstd'
- gcp-metadata
- kerberos
- mongodb-client-encryption
- snappy
- socks
- supports-color
mongodb@6.12.0(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3):
dependencies:
'@mongodb-js/saslprep': 1.1.9
@@ -34416,6 +34574,12 @@ snapshots:
netmask@2.0.2: {}
new-find-package-json@2.0.0:
dependencies:
debug: 4.4.0
transitivePeerDependencies:
- supports-color
next-tick@1.1.0: {}
nice-try@1.0.5: {}
@@ -35201,6 +35365,8 @@ snapshots:
promise-inflight@1.0.1: {}
promise-polyfill@8.3.0: {}
promise-retry@2.0.1:
dependencies:
err-code: 2.0.3
@@ -35714,6 +35880,15 @@ snapshots:
parse-srcset: 1.0.2
postcss: 8.5.3
sanitize-html@2.16.0:
dependencies:
deepmerge: 4.3.1
escape-string-regexp: 4.0.0
htmlparser2: 8.0.2
is-plain-object: 5.0.0
parse-srcset: 1.0.2
postcss: 8.5.3
sass-loader@13.3.3(sass@1.71.1)(webpack@5.97.1):
dependencies:
neo-async: 2.6.2
@@ -37616,6 +37791,11 @@ snapshots:
buffer-crc32: 0.2.13
fd-slicer: 1.1.0
yauzl@3.2.0:
dependencies:
buffer-crc32: 0.2.13
pend: 1.2.0
yjs@13.6.23:
dependencies:
lib0: 0.2.99
dev/tool/package.json
+3 -1
View File
@@ -55,7 +55,8 @@
"@types/request": "~2.48.8",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5"
"@types/jest": "^29.5.5",
"mongodb-memory-server": "^10.1.4"
},
"dependencies": {
"@elastic/elasticsearch": "^7.17.14",
@@ -160,6 +161,7 @@
"@hcengineering/collaboration": "^0.6.0",
"@hcengineering/datalake": "^0.6.0",
"@hcengineering/s3": "^0.6.0",
"@hcengineering/kvs-client": "^0.6.0",
"commander": "^8.1.0",
"csv-parse": "~5.1.0",
"email-addresses": "^5.0.0",
dev/tool/src/__start.ts
+9
View File
@@ -103,4 +103,13 @@ export function getAccountDBUrl (): string {
return url
}
export function getKvsUrl (): string {
const url = process.env.KVS_URL
if (url === undefined) {
console.error('please provide KVS_URL')
process.exit(1)
}
return url
}
devTool(prepareTools)
dev/tool/src/__tests__/gmail.test.ts
+311
View File
@@ -0,0 +1,311 @@
import { getAccountClient } from '@hcengineering/server-client'
import { getClient as getKvsClient } from '@hcengineering/kvs-client'
import { generateToken } from '@hcengineering/server-token'
import { MongoMemoryServer } from 'mongodb-memory-server'
import { MongoClient, type Db } from 'mongodb'
import { performGmailAccountMigrations } from '../gmail'
// Mock dependencies
jest.mock('@hcengineering/server-client')
jest.mock('@hcengineering/server-token')
jest.mock('@hcengineering/kvs-client')
describe('Gmail Migrations', () => {
// Setup MongoDB in-memory server
let mongoServer: MongoMemoryServer
let mongoClient: MongoClient
let db: Db
// Mock implementations
const mockAccountClient = {
listWorkspaces: jest.fn(),
findFullSocialIdBySocialKey: jest.fn(),
getIntegration: jest.fn(),
createIntegration: jest.fn(),
getIntegrationSecret: jest.fn(),
addIntegrationSecret: jest.fn(),
updateIntegrationSecret: jest.fn()
}
const mockKvsClient = {
getValue: jest.fn(),
setValue: jest.fn(),
deleteKey: jest.fn(),
listKeys: jest.fn()
}
beforeAll(async () => {
mongoServer = await MongoMemoryServer.create()
const uri = mongoServer.getUri()
mongoClient = await MongoClient.connect(uri)
db = mongoClient.db('test-db')
// Setup mocks
;(getAccountClient as jest.Mock).mockReturnValue(mockAccountClient)
;(getKvsClient as jest.Mock).mockReturnValue(mockKvsClient)
;(generateToken as jest.Mock).mockReturnValue('mock-token')
})
afterAll(async () => {
await mongoClient.close()
await mongoServer.stop()
})
beforeEach(async () => {
// Reset mocks
jest.clearAllMocks()
// Setup collections
await db.collection('tokens').deleteMany({})
await db.collection('histories').deleteMany({})
// Reset mock implementations
mockAccountClient.listWorkspaces.mockReset()
mockAccountClient.findFullSocialIdBySocialKey.mockReset()
mockAccountClient.getIntegration.mockReset()
mockAccountClient.createIntegration.mockReset()
mockAccountClient.getIntegrationSecret.mockReset()
mockAccountClient.addIntegrationSecret.mockReset()
mockAccountClient.updateIntegrationSecret.mockReset()
mockKvsClient.getValue.mockReset()
mockKvsClient.setValue.mockReset()
}, 10000)
it('should migrate tokens to new integration format', async () => {
// Setup test data
const workspace1 = { uuid: 'ws1', name: 'Workspace 1' }
const workspace2 = { uuid: 'ws2', dataId: 'oldWs2', name: 'Workspace 2' }
// Mock workspace list
mockAccountClient.listWorkspaces.mockResolvedValue([workspace1, workspace2])
// Setup tokens in DB
await db.collection('tokens').insertMany([
{
userId: 'user1@example.com',
workspace: 'ws1',
token: 'token1',
refresh_token: 'refresh1',
access_token: 'access1'
},
{
userId: 'user2@example.com',
workspace: 'oldWs2',
token: 'token2',
refresh_token: 'refresh2',
access_token: 'access2'
}
])
// Mock social ID lookup
mockAccountClient.findFullSocialIdBySocialKey.mockImplementation((key) => {
if (key === 'email:user1@example.com') {
return Promise.resolve({ _id: 'social1', personUuid: 'person1' })
} else if (key === 'email:user2@example.com') {
return Promise.resolve({ _id: 'social2', personUuid: 'person2' })
}
return Promise.resolve(undefined)
})
// Mock integration checks
mockAccountClient.getIntegration.mockResolvedValue(null)
mockAccountClient.getIntegrationSecret.mockResolvedValue(null)
// Run migration
await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url')
// Verify tokens were migrated
expect(mockAccountClient.createIntegration).toHaveBeenCalledTimes(2)
expect(mockAccountClient.addIntegrationSecret).toHaveBeenCalledTimes(2)
// Check that tokens were migrated with correct data
const calls = mockAccountClient.addIntegrationSecret.mock.calls
expect(calls).toContainEqual([
expect.objectContaining({
socialId: 'social1',
workspaceUuid: 'ws1',
secret: expect.stringContaining('user1@example.com')
})
])
expect(calls).toContainEqual([
expect.objectContaining({
socialId: 'social2',
workspaceUuid: 'ws2',
secret: expect.stringContaining('user2@example.com')
})
])
})
it('should migrate with oids and github ids', async () => {
// Setup test data
const workspace1 = { uuid: 'ws1', name: 'Workspace 1' }
const workspace2 = { uuid: 'ws2', dataId: 'oldWs2', name: 'Workspace 2' }
// Mock workspace list
mockAccountClient.listWorkspaces.mockResolvedValue([workspace1, workspace2])
// Setup tokens in DB
await db.collection('tokens').insertMany([
{
userId: 'github:user1',
workspace: 'ws1',
token: 'token1',
refresh_token: 'refresh1',
access_token: 'access1'
},
{
userId: 'openid:user2',
workspace: 'oldWs2',
token: 'token2',
refresh_token: 'refresh2',
access_token: 'access2'
}
])
// Mock social ID lookup
mockAccountClient.findFullSocialIdBySocialKey.mockImplementation((key) => {
if (key === 'github:user1') {
return Promise.resolve({ _id: 'social1', personUuid: 'person1' })
} else if (key === 'oidc:user2') {
return Promise.resolve({ _id: 'social2', personUuid: 'person2' })
}
return Promise.resolve(undefined)
})
// Mock integration checks
mockAccountClient.getIntegration.mockResolvedValue(null)
mockAccountClient.getIntegrationSecret.mockResolvedValue(null)
// Run migration
await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url')
// Verify tokens were migrated
expect(mockAccountClient.createIntegration).toHaveBeenCalledTimes(2)
expect(mockAccountClient.addIntegrationSecret).toHaveBeenCalledTimes(2)
// Check that tokens were migrated with correct data
const calls = mockAccountClient.addIntegrationSecret.mock.calls
expect(calls).toContainEqual([
expect.objectContaining({
socialId: 'social1',
workspaceUuid: 'ws1',
secret: expect.stringContaining('github:user1')
})
])
expect(calls).toContainEqual([
expect.objectContaining({
socialId: 'social2',
workspaceUuid: 'ws2',
secret: expect.stringContaining('openid:user2')
})
])
})
it('should migrate history records to KVS', async () => {
// Setup test data
const workspace1 = { uuid: 'ws1', name: 'Workspace 1' }
// Mock workspace list
mockAccountClient.listWorkspaces.mockResolvedValue([workspace1])
// Setup histories in DB
await db.collection('histories').insertMany([
{
userId: 'user1@example.com',
workspace: 'ws1',
token: 'token1',
historyId: 'history1'
}
])
// Mock social ID lookup
mockAccountClient.findFullSocialIdBySocialKey.mockImplementation((key) => {
if (key === 'email:user1@example.com') {
return Promise.resolve({ _id: 'social1', personUuid: 'person1' })
}
return Promise.resolve(undefined)
})
// Mock KVS client responses
mockKvsClient.getValue.mockResolvedValue(null)
// Run migration
await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url')
// Verify KVS calls
expect(mockKvsClient.setValue).toHaveBeenCalledWith(
'history:ws1:social1',
expect.objectContaining({
historyId: 'history1',
email: 'user1@example.com',
userId: 'person1',
workspace: 'ws1'
})
)
})
it('should handle errors gracefully', async () => {
// Mock console.error to capture errors
const originalConsoleError = console.error
console.error = jest.fn()
// Setup test data that will cause errors
mockAccountClient.listWorkspaces.mockResolvedValue([])
mockAccountClient.findFullSocialIdBySocialKey.mockRejectedValue(new Error('Network error'))
// Insert some data
await db.collection('tokens').insertOne({
userId: 'error@example.com',
workspace: 'non-existent',
token: 'token-error'
})
// Run migration
await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url')
// Should not throw but log errors
expect(console.error).toHaveBeenCalled()
// Restore console.error
console.error = originalConsoleError
})
it('should update existing integration secrets', async () => {
// Setup test data
const workspace = { uuid: 'ws1', name: 'Workspace 1' }
// Mock workspace list
mockAccountClient.listWorkspaces.mockResolvedValue([workspace])
// Setup token in DB
await db.collection('tokens').insertOne({
userId: 'user1@example.com',
workspace: 'ws1',
token: 'token1',
refresh_token: 'refresh1'
})
// Mock social ID lookup
mockAccountClient.findFullSocialIdBySocialKey.mockResolvedValue({ _id: 'social1', personUuid: 'person1' })
// Mock existing integration and token
mockAccountClient.getIntegration.mockResolvedValue({ _id: 'integration1' })
mockAccountClient.getIntegrationSecret.mockResolvedValue({
scope: 'old-scope',
token_type: 'Bearer'
})
// Run migration
await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url')
// Verify update was called instead of add
expect(mockAccountClient.createIntegration).not.toHaveBeenCalled()
expect(mockAccountClient.addIntegrationSecret).not.toHaveBeenCalled()
expect(mockAccountClient.updateIntegrationSecret).toHaveBeenCalledWith(
expect.objectContaining({
secret: expect.stringContaining('refresh1')
})
)
})
})
dev/tool/src/gmail.ts
+225
View File
@@ -0,0 +1,225 @@
import {
type AccountUuid,
type PersonId,
type SocialId,
type WorkspaceInfoWithStatus,
type WorkspaceUuid,
buildSocialIdString,
systemAccountUuid
} from '@hcengineering/core'
import { getAccountClient } from '@hcengineering/server-client'
import { getClient as getKvsClient } from '@hcengineering/kvs-client'
import { generateToken } from '@hcengineering/server-token'
import { getSocialKeyByOldEmail } from '@hcengineering/model-core'
import type { Db } from 'mongodb'
// Old token and history types
interface Credentials {
refresh_token?: string | null
expiry_date?: number | null
access_token?: string | null
token_type?: string | null
id_token?: string | null
scope?: string
}
interface User {
userId: string
workspace: WorkspaceUuid
token: string
}
type History = User & {
historyId: string
}
// Updated token and history types
interface UserV2 {
userId: AccountUuid
email?: string
socialId: SocialId
workspace: WorkspaceUuid
token: string
}
export type TokenV2 = UserV2 & Credentials
export type HistoryV2 = UserV2 & {
historyId: string
}
type Token = User & Credentials
interface WorkspaceInfoProvider {
getWorkspaceInfo: (workspaceUuid: WorkspaceUuid) => Promise<WorkspaceInfoWithStatus | undefined>
}
const GMAIL_INTEGRATION = 'gmail'
const TOKEN_TYPE = 'token'
export async function performGmailAccountMigrations (db: Db, region: string | null, kvsUrl: string): Promise<void> {
console.log('Start Gmail migrations')
const token = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'admin', admin: 'true' })
const accountClient = getAccountClient(token)
const allWorkpaces = await accountClient.listWorkspaces(region)
const byId = new Map(allWorkpaces.map((it) => [it.uuid, it]))
const oldNewIds = new Map(allWorkpaces.map((it) => [it.dataId ?? it.uuid, it]))
const workspaceProvider: WorkspaceInfoProvider = {
getWorkspaceInfo: async (workspaceUuid: WorkspaceUuid) => {
const ws = oldNewIds.get(workspaceUuid as any) ?? byId.get(workspaceUuid as any)
if (ws == null) {
console.error('No workspace found for token', workspaceUuid)
return undefined
}
return ws
}
}
await migrateGmailIntegrations(db, token, workspaceProvider)
await migrateGmailHistory(db, token, kvsUrl, workspaceProvider)
console.log('Finished Gmail migrations')
}
async function migrateGmailIntegrations (
db: Db,
token: string,
workspaceProvider: WorkspaceInfoProvider
): Promise<void> {
try {
console.log('Start Gmail account migrations')
const gmailToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'gmail' })
const accountClient = getAccountClient(token)
const gmailAccountClient = getAccountClient(gmailToken)
const tokens = db.collection<Token>('tokens')
const allTokens = await tokens.find({}).toArray()
for (const token of allTokens) {
try {
const ws = await workspaceProvider.getWorkspaceInfo(token.workspace as any)
if (ws == null) {
continue
}
token.workspace = ws.uuid
const socialKey = buildSocialIdString(getSocialKeyByOldEmail(token.userId))
const socialId =
socialKey !== undefined ? await accountClient.findFullSocialIdBySocialKey(socialKey) : undefined
if (socialId == null) {
console.error('No socialId found for token', token)
continue
}
// Check/create integration in account
const existing = await gmailAccountClient.getIntegration({
kind: GMAIL_INTEGRATION,
workspaceUuid: ws?.uuid,
socialId: socialId._id
})
if (existing == null) {
await gmailAccountClient.createIntegration({
kind: GMAIL_INTEGRATION,
workspaceUuid: ws?.uuid,
socialId: socialId._id
})
}
const existingToken = await gmailAccountClient.getIntegrationSecret({
key: TOKEN_TYPE,
kind: GMAIL_INTEGRATION,
socialId: socialId._id,
workspaceUuid: ws?.uuid
})
const newToken: TokenV2 = {
...token,
workspace: ws?.uuid,
email: token.userId,
userId: socialId.personUuid as AccountUuid,
socialId
}
if (existingToken == null) {
await gmailAccountClient.addIntegrationSecret({
key: TOKEN_TYPE,
kind: GMAIL_INTEGRATION,
socialId: socialId._id,
secret: JSON.stringify(newToken),
workspaceUuid: token.workspace
})
} else {
const updatedToken = {
...existingToken,
...newToken
}
await gmailAccountClient.updateIntegrationSecret({
key: TOKEN_TYPE,
kind: GMAIL_INTEGRATION,
socialId: socialId._id,
secret: JSON.stringify(updatedToken),
workspaceUuid: token.workspace
})
}
} catch (e) {
console.error('Error migrating token', token, e)
}
}
console.log('Gmail integrations migrations done, integration count:', allTokens.length)
} catch (e) {
console.error('Error migrating tokens', e)
}
}
async function migrateGmailHistory (
db: Db,
token: string,
kvsUrl: string,
workspaceProvider: WorkspaceInfoProvider
): Promise<void> {
try {
console.log('Start Gmail history migrations')
const accountClient = getAccountClient(token)
const history = db.collection<History>('histories')
const allHistories = await history.find({}).toArray()
const kvsClient = getKvsClient(kvsUrl, token)
for (const history of allHistories) {
try {
const socialKey = buildSocialIdString(getSocialKeyByOldEmail(history.userId))
const socialId =
socialKey !== undefined ? await accountClient.findFullSocialIdBySocialKey(socialKey) : undefined
if (socialId == null) {
console.error('No socialId found for history', history)
continue
}
// Update/create history in KVS
const ws = await workspaceProvider.getWorkspaceInfo(history.workspace as any)
if (ws == null) {
continue
}
const historyKey = getHistoryKey(ws.uuid, socialId._id)
const existingHistory = await kvsClient.getValue<HistoryV2>(historyKey)
const updatedHistory: HistoryV2 = {
...(existingHistory ?? history),
email: history.userId,
workspace: ws.uuid,
userId: socialId.personUuid as AccountUuid,
socialId
}
await kvsClient.setValue(historyKey, updatedHistory)
} catch (e) {
console.error('Error migrating history', history, e)
}
}
console.log('Finished migrating gmail history, count:', allHistories.length)
} catch (e) {
console.error('Error migrating gmail history', e)
}
}
function getHistoryKey (workspace: WorkspaceUuid, userId: PersonId): string {
return `history:${workspace}:${userId}`
}
dev/tool/src/index.ts
+17 -1
View File
@@ -89,13 +89,14 @@ import {
type QueueWorkspaceMessage,
type StorageAdapter
} from '@hcengineering/server-core'
import { getAccountDBUrl, getMongoDBUrl } from './__start'
import { getAccountDBUrl, getKvsUrl, getMongoDBUrl } from './__start'
// import { fillGithubUsers, fixAccountEmails, renameAccount } from './account'
import { changeConfiguration } from './configuration'
import { performGithubAccountMigrations } from './github'
import { migrateCreatedModifiedBy, ensureGlobalPersonsForLocalAccounts, moveAccountDbFromMongoToPG } from './db'
import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
import { performGmailAccountMigrations } from './gmail'
const colorConstants = {
colorRed: '\u001b[31m',
@@ -2429,6 +2430,21 @@ export function devTool (
await queue.createTopics(parseInt(cmd.tx ?? '1'))
})
program
.command('migrate-gmail-account')
.option('--db <db>', 'DB name', 'gmail-service')
.option('--region <region>', 'DB region')
.action(async (cmd: { db: string, region?: string }) => {
const mongodbUri = getMongoDBUrl()
const client = getMongoClient(mongodbUri)
const _client = await client.getClient()
const kvsUrl = getKvsUrl()
await performGmailAccountMigrations(_client.db(cmd.db), cmd.region ?? null, kvsUrl)
await _client.close()
client.close()
})
extendProgram?.(program)
program.parse(process.argv)
packages/kvs-client/.eslintrc.js
+7
View File
@@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}
packages/kvs-client/.npmignore
+4
View File
@@ -0,0 +1,4 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/
packages/kvs-client/config/rig.json
+4
View File
@@ -0,0 +1,4 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}
packages/kvs-client/jest.config.js
+7
View File
@@ -0,0 +1,7 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
roots: ["./src"],
coverageReporters: ["text-summary", "html"]
}
packages/kvs-client/jest.setup.js
+2
View File
@@ -0,0 +1,2 @@
// Set up fetch mock
require('jest-fetch-mock').enableMocks()
packages/kvs-client/package.json
+58
View File
@@ -0,0 +1,58 @@
{
"name": "@hcengineering/kvs-client",
"version": "0.6.0",
"main": "lib/index.js",
"svelte": "src/index.ts",
"types": "types/index.d.ts",
"files": [
"lib/**/*",
"types/**/*",
"tsconfig.json"
],
"author": "Hardcore Engineering Inc.",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"format": "format src",
"test": "jest --passWithNoTests --silent",
"_phase:build": "compile transpile src",
"_phase:test": "jest --passWithNoTests --silent",
"_phase:format": "format src",
"_phase:validate": "compile validate"
},
"devDependencies": {
"cross-env": "~7.0.3",
"@hcengineering/platform-rig": "^0.6.0",
"@types/node": "~20.11.16",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"esbuild": "^0.24.2",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"jest": "^29.7.0",
"jest-fetch-mock": "^3.0.3",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5"
},
"dependencies": {
"@hcengineering/core": "^0.6.32",
"@hcengineering/platform": "^0.6.11"
},
"repository": "https://github.com/hcengineering/platform",
"publishConfig": {
"registry": "https://npm.pkg.github.com"
},
"exports": {
".": {
"types": "./types/index.d.ts",
"require": "./lib/index.js",
"import": "./lib/index.js"
}
}
}
packages/kvs-client/src/__tests__/client.test.ts
+201
View File
@@ -0,0 +1,201 @@
import { getClient } from '../client'
import { KeyValueClient } from '../types'
import fetchMock from 'jest-fetch-mock'
// Enable fetch mocks
fetchMock.enableMocks()
describe('KeyValueClient', () => {
let client: KeyValueClient
const namespace = 'test-ns'
beforeEach(() => {
// Reset fetch mocks for each test
fetchMock.resetMocks()
// Default successful response for all fetch calls
fetchMock.mockResponse(JSON.stringify({ key: 'value' }), {
status: 200,
headers: { 'Content-Type': 'application/json' }
})
client = getClient(namespace, 'https://api.example.com', 'test-token')
})
afterEach(() => {
fetchMock.resetMocks()
})
it('should throw error when baseUrl is empty', () => {
expect(() => getClient('test-ns', '')).toThrow('Key-value API URL not specified')
})
it('should throw error when namespace is empty', () => {
expect(() => getClient('', 'https://api.example.com')).toThrow('Namespace not specified')
})
describe('setValue', () => {
it('should send POST request with correct headers and body', async () => {
// Mock 204 response for setValue
fetchMock.mockResponseOnce('', { status: 204 })
const key = 'test-key'
const value = { data: 'test-value' }
await client.setValue(key, value)
expect(fetchMock).toHaveBeenCalledTimes(1)
expect(fetchMock).toHaveBeenCalledWith(
'https://api.example.com/api/test-ns/test-key',
expect.objectContaining({
method: 'POST',
headers: expect.objectContaining({
'Content-Type': 'application/json',
Authorization: 'Bearer test-token',
Connection: 'keep-alive'
}),
body: JSON.stringify(value)
})
)
})
it('should throw error when response status is not 204', async () => {
fetchMock.mockResponseOnce(JSON.stringify({ error: 'Server error' }), { status: 500 })
await expect(client.setValue('key', 'value')).rejects.toThrow('Failed to store value')
})
})
describe('getValue', () => {
it('should send GET request with correct headers', async () => {
const testData = { message: 'test data' }
fetchMock.mockResponseOnce(JSON.stringify(testData), {
status: 200,
headers: { 'Content-Type': 'application/json' }
})
const result = await client.getValue('key')
expect(fetchMock).toHaveBeenCalledTimes(1)
expect(fetchMock).toHaveBeenCalledWith(
'https://api.example.com/api/test-ns/key',
expect.objectContaining({
method: 'GET',
headers: expect.objectContaining({
Authorization: 'Bearer test-token',
Connection: 'keep-alive'
})
})
)
expect(result).toEqual(testData)
})
it('should return null when key does not exist (404)', async () => {
fetchMock.mockResponseOnce('', { status: 404 })
const result = await client.getValue('key')
expect(result).toBeNull()
})
it('should throw error for non-404 error status', async () => {
fetchMock.mockResponseOnce(JSON.stringify({ error: 'Server error' }), { status: 500 })
await expect(client.getValue('key')).rejects.toThrow('Request failed')
})
})
describe('deleteKey', () => {
it('should send DELETE request with correct headers', async () => {
fetchMock.mockResponseOnce('', { status: 204 })
await client.deleteKey('key')
expect(fetchMock).toHaveBeenCalledTimes(1)
expect(fetchMock).toHaveBeenCalledWith(
'https://api.example.com/api/test-ns/key',
expect.objectContaining({
method: 'DELETE',
headers: expect.objectContaining({
Authorization: 'Bearer test-token',
Connection: 'keep-alive'
})
})
)
})
it('should not throw error when key does not exist (404)', async () => {
fetchMock.mockResponseOnce('', { status: 404 })
await expect(client.deleteKey('key')).resolves.toBeUndefined()
})
it('should throw error for status other than 204 or 404', async () => {
fetchMock.mockResponseOnce(JSON.stringify({ error: 'Server error' }), { status: 500 })
await expect(client.deleteKey('key')).rejects.toThrow('Failed to delete key')
})
})
describe('listKeys', () => {
it('should send GET request to namespace endpoint', async () => {
const testData = { key1: 'value1', key2: 'value2' }
fetchMock.mockResponseOnce(JSON.stringify(testData), {
status: 200,
headers: { 'Content-Type': 'application/json' }
})
const result = await client.listKeys()
expect(fetchMock).toHaveBeenCalledTimes(1)
expect(fetchMock).toHaveBeenCalledWith(
'https://api.example.com/api/test-ns',
expect.objectContaining({ method: 'GET' })
)
expect(result).toEqual(testData)
})
it('should include prefix query parameter when provided', async () => {
fetchMock.mockResponseOnce(JSON.stringify({}), {
status: 200,
headers: { 'Content-Type': 'application/json' }
})
await client.listKeys('prefix')
expect(fetchMock).toHaveBeenCalledWith('https://api.example.com/api/test-ns?prefix=prefix', expect.anything())
})
it('should encode namespace and prefix properly', async () => {
// Create client with a namespace that needs encoding
const specialClient = getClient('name/space', 'https://api.example.com', 'test-token')
fetchMock.mockResponseOnce(JSON.stringify({}), {
status: 200,
headers: { 'Content-Type': 'application/json' }
})
await specialClient.listKeys('pre/fix')
expect(fetchMock).toHaveBeenCalledWith(
'https://api.example.com/api/name%2Fspace?prefix=pre%2Ffix',
expect.anything()
)
})
})
describe('retry logic', () => {
it('should retry on network errors', async () => {
// First call throws network error, second succeeds
fetchMock.mockRejectOnce(new Error('Network error')).mockResponseOnce(JSON.stringify({ key: 'value' }), {
status: 200,
headers: { 'Content-Type': 'application/json' }
})
const result = await client.getValue('key')
expect(fetchMock).toHaveBeenCalledTimes(2)
expect(result).toEqual({ key: 'value' })
})
})
})
packages/kvs-client/src/client.ts
+200
View File
@@ -0,0 +1,200 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { concatLink } from '@hcengineering/core'
import { PlatformError } from '@hcengineering/platform'
import { KeyValueClient } from './types'
/**
* Get a KeyValueClient instance
* @param namespace - Namespace for the key-value operations
* @param baseUrl - URL of the key-value API server
* @param token - Optional authorization token
* @param retryTimeoutMs - Optional timeout for retrying failed requests
* @returns KeyValueClient instance
* @public
*/
export function getClient (namespace: string, baseUrl: string, token?: string, retryTimeoutMs?: number): KeyValueClient {
if (baseUrl === undefined) {
throw new Error('Key-value API URL not specified')
}
return new KeyValueClientImpl(namespace, baseUrl, token, retryTimeoutMs)
}
class KeyValueClientImpl implements KeyValueClient {
private readonly requestInit: RequestInit
constructor (
private readonly namespace: string,
private readonly baseUrl: string,
private readonly token?: string,
private readonly retryTimeoutMs: number = 5000
) {
if (baseUrl === '') {
throw new Error('Key-value API URL not specified')
}
if (namespace === '' || namespace === undefined) {
throw new Error('Namespace not specified')
}
const isBrowser = typeof window !== 'undefined'
this.requestInit = {
keepalive: true,
headers: {
...(this.token === undefined
? {}
: {
Authorization: 'Bearer ' + this.token
})
},
...(isBrowser ? { credentials: 'include' } : {})
}
}
async setValue<T>(key: string, value: T): Promise<void> {
const url = this.buildUrl(key)
await this.sendRequest(url, {
method: 'POST',
body: JSON.stringify(value),
contentType: 'application/json',
errorMessage: 'Failed to store value'
})
}
async getValue<T>(key: string): Promise<T | null> {
const url = this.buildUrl(key)
return await this.sendRequest<T>(url, {
method: 'GET',
returnNullOn404: true
})
}
async deleteKey (key: string): Promise<void> {
const url = this.buildUrl(key)
await this.sendRequest(url, {
method: 'DELETE',
acceptNotFound: true,
errorMessage: 'Failed to delete key'
})
}
async listKeys<T>(prefix?: string): Promise<Record<string, T> | null> {
let url = this.buildUrl()
if (prefix !== undefined) {
url += `?prefix=${encodeURIComponent(prefix)}`
}
return await this.sendRequest<Record<string, T>>(url, {
method: 'GET',
errorMessage: 'Failed to list keys'
})
}
private buildUrl (key?: string): string {
const baseApiUrl = concatLink(this.baseUrl, `/api/${encodeURIComponent(this.namespace)}`)
return key !== undefined ? concatLink(baseApiUrl, encodeURIComponent(key)) : baseApiUrl
}
private async sendRequest<T>(
url: string,
options: {
method: string
body?: string
contentType?: string
returnNullOn404?: boolean
acceptNotFound?: boolean
errorMessage?: string
}
): Promise<T | null> {
const headers = {
...this.requestInit.headers,
...(options.contentType !== undefined && options.contentType !== ''
? { 'Content-Type': options.contentType }
: {})
}
const response = await this.fetchWithRetry(url, {
...this.requestInit,
method: options.method,
headers,
body: options.body
})
if (options.returnNullOn404 === true && response.status === 404) {
return null
}
if (options.acceptNotFound === true && response.status === 404) {
return null
}
if (!response.ok) {
try {
const errorBody = await response.json()
if (errorBody?.error != null) {
throw new PlatformError(errorBody?.error)
}
} catch (e) {
// Ignore JSON parsing errors
}
throw new Error(options.errorMessage ?? 'Request failed')
}
// Parse JSON response when needed
const contentType = response.headers.get('content-type')
if (
response.status !== 204 &&
(options.method === 'GET' || (contentType != null && contentType.includes('application/json')))
) {
return await response.json()
}
return null
}
private async fetchWithRetry (url: string, init: RequestInit): Promise<Response> {
const timeout = Date.now() + this.retryTimeoutMs
const connectionErrorCodes = ['ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND']
let intervalMs = 25
while (true) {
try {
const headers = {
...init.headers,
Connection: 'keep-alive'
}
return await fetch(url, { ...init, headers })
} catch (err: any) {
// Check if this is a network error we should retry
const errMessage: string = err?.message ?? ''
const isNetworkError =
connectionErrorCodes.includes(err?.cause?.code) ||
errMessage.includes('Network error') ||
errMessage.includes('network')
if (!isNetworkError || timeout < Date.now()) {
throw err
}
await new Promise<void>((resolve) => setTimeout(resolve, intervalMs))
if (intervalMs < 1000) {
intervalMs += 100
}
}
}
}
}
packages/kvs-client/src/index.ts
+17
View File
@@ -0,0 +1,17 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export * from './client'
export * from './types'
packages/kvs-client/src/types.ts
+49
View File
@@ -0,0 +1,49 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
/**
* Client for interacting with the key-value store API
* @public
*/
export interface KeyValueClient {
/**
* Store a value for a key in a namespace
* @param key - The key to store the value under
* @param value - The value to store
* @returns Promise that resolves when the value is stored
*/
setValue: <T>(key: string, value: T) => Promise<void>
/**
* Retrieve a value for a key in a namespace
* @param key - The key to retrieve the value for
* @returns Promise that resolves to the value, or null if not found
*/
getValue: <T>(key: string) => Promise<T | null>
/**
* Delete a key-value pair in a namespace
* @param key - The key to delete
* @returns Promise that resolves when the key is deleted
*/
deleteKey: (key: string) => Promise<void>
/**
* List all key-value pairs in a namespace, optionally filtered by prefix
* @param prefix - Optional prefix to filter keys by
* @returns Promise that resolves to an object with keys and their values
*/
listKeys: <T>(prefix?: string) => Promise<Record<string, T> | null>
}
packages/kvs-client/tsconfig.json
+10
View File
@@ -0,0 +1,10 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"declarationDir": "./types",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}
plugins/gmail-assets/lang/cs.json
+2 -1
View File
@@ -38,6 +38,7 @@
"ConfigLabel": "E-mail",
"ConfigDescription": "Rozšíření pro integraci e-mailů Gmail",
"GooglePrivacy": "Použití a přenos informací získaných z Google API do jakékoli jiné aplikace bude v souladu s <a href=\"https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes\" target=\"_blank\">zásadami Google API Services User Data Policy</a>, včetně požadavků na omezené použití.",
"EmailNotificationsDescription": "Dostávejte osobní oznámení na e-mail."
"EmailNotificationsDescription": "Dostávejte osobní oznámení na e-mail.",
"FailedToConnect": "Nepodařilo se připojit k Gmail"
}
}
plugins/gmail-assets/lang/de.json
+2 -1
View File
@@ -38,6 +38,7 @@
"ConfigLabel": "E-Mail",
"ConfigDescription": "Erweiterung für Gmail-Integration",
"GooglePrivacy": "Die Nutzung und Übertragung von Informationen, die Huly von Google-APIs erhält, an andere Apps erfolgt gemäß der <a href=\"https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes\" target=\"_blank\">Google API Services User Data Policy</a>, einschließlich der Anforderungen zur eingeschränkten Nutzung.",
"EmailNotificationsDescription": "Erhalten Sie persönliche Benachrichtigungen per E-Mail."
"EmailNotificationsDescription": "Erhalten Sie persönliche Benachrichtigungen per E-Mail.",
"FailedToConnect": "Fehler beim Verbinden mit Gmail"
}
}
plugins/gmail-assets/lang/en.json
+2 -1
View File
@@ -38,6 +38,7 @@
"ConfigLabel": "Email",
"ConfigDescription": "Extension for Gmail email integration",
"GooglePrivacy": "Huly’s use and transfer of information received from Google APIs to any other app will adhere to <a href=\"https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes\" target=\"_blank\">Google API Services User Data Policy</a>, including the Limited Use requirements.",
"EmailNotificationsDescription": "Receive personal notifications on email."
"EmailNotificationsDescription": "Receive personal notifications on email.",
"FailedToConnect": "Failed to connect to Gmail"
}
}
plugins/gmail-assets/lang/es.json
+2 -1
View File
@@ -38,6 +38,7 @@
"ConfigLabel": "Correo Electrónico",
"ConfigDescription": "Extensión para la integración de correo electrónico de Gmail",
"GooglePrivacy": "Huly’s use and transfer of information received from Google APIs to any other app will adhere to <a href=\"https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes\" target=\"_blank\">Google API Services User Data Policy</a>, including the Limited Use requirements.",
"EmailNotificationsDescription": "Reciba notificaciones personales por correo electrónico."
"EmailNotificationsDescription": "Reciba notificaciones personales por correo electrónico.",
"FailedToConnect": "Error al conectar con Gmail"
}
}
plugins/gmail-assets/lang/fr.json
+2 -1
View File
@@ -38,6 +38,7 @@
"ConfigLabel": "Email",
"ConfigDescription": "Extension pour l'intégration des emails Gmail",
"GooglePrivacy": "L'utilisation et le transfert des informations reçues des API Google par Huly à toute autre application respecteront les <a href=\"https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes\" target=\"_blank\">règles d'utilisation des données utilisateur des services API Google</a>, y compris les exigences d'utilisation limitée.",
"EmailNotificationsDescription": "Recevez des notifications personnelles par e-mail."
"EmailNotificationsDescription": "Recevez des notifications personnelles par e-mail.",
"FailedToConnect": "Impossible de se connecter au compte Gmail"
}
}
plugins/gmail-assets/lang/it.json
+2 -1
View File
@@ -38,6 +38,7 @@
"ConfigLabel": "Email",
"ConfigDescription": "Estensione per l'integrazione delle email di Gmail",
"GooglePrivacy": "L'uso e il trasferimento delle informazioni ricevute dalle API di Google a qualsiasi altra app di Huly adempirà alla <a href=\"https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes\" target=\"_blank\">Politica sui dati degli utenti dei servizi API di Google</a>, inclusi i requisiti di utilizzo limitato.",
"EmailNotificationsDescription": "Ricevi notifiche personali sulle email."
"EmailNotificationsDescription": "Ricevi notifiche personali sulle email.",
"FailedToConnect": "Impossibile connettere l'account Gmail"
}
}
plugins/gmail-assets/lang/pt.json
+2 -1
View File
@@ -38,6 +38,7 @@
"ConfigLabel": "Email",
"ConfigDescription": "Extensão para a integração de email do Gmail",
"GooglePrivacy": "Huly’s use and transfer of information received from Google APIs to any other app will adhere to <a href=\"https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes\" target=\"_blank\">Google API Services User Data Policy</a>, including the Limited Use requirements.",
"EmailNotificationsDescription": "Receba notificações pessoais por e-mail."
"EmailNotificationsDescription": "Receba notificações pessoais por e-mail.",
"FailedToConnect": "Não foi possível conectar a conta do Gmail"
}
}
plugins/gmail-assets/lang/ru.json
+2 -1
View File
@@ -38,6 +38,7 @@
"ConfigLabel": "Электронная почта",
"ConfigDescription": "Расширение по работе с Gmail электронной почтой",
"GooglePrivacy": "Использование и передача информации, полученной Huly от Google API, будет соответствовать <a href=\"https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes\" target=\"_blank\">Политике использования данных пользователей Google API</a>, включая требования ограниченного использования.",
"EmailNotificationsDescription": "Получайте персональные уведомления на электронную почту."
"EmailNotificationsDescription": "Получайте персональные уведомления на электронную почту.",
"FailedToConnect": "Не удалось подключиться к Gmail"
}
}
plugins/gmail-assets/lang/zh.json
+2 -1
View File
@@ -38,6 +38,7 @@
"ConfigLabel": "电子邮件",
"ConfigDescription": "Gmail 邮件集成扩展",
"GooglePrivacy": "Huly 从 Google API 接收的信息的使用和传输将遵守 <a href=\"https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes\" target=\"_blank\">Google API 服务用户数据政策</a>，包括有限使用要求。",
"EmailNotificationsDescription": "透過電子郵件接收個人通知。"
"EmailNotificationsDescription": "透過電子郵件接收個人通知。",
"FailedToConnect": "無法連接 Gmail 帳戶。請檢查您的憑證並重試。"
}
}
plugins/gmail-resources/src/components/Chats.svelte
+14 -39
View File
@@ -16,14 +16,12 @@
<script lang="ts">
/* eslint-disable @typescript-eslint/no-unused-vars */
import { Channel, Contact } from '@hcengineering/contact'
import { employeeByIdStore } from '@hcengineering/contact-resources'
import { Ref, SortingOrder } from '@hcengineering/core'
import { Message, SharedMessage } from '@hcengineering/gmail'
import { InboxNotificationsClientImpl } from '@hcengineering/notification-resources'
import { createQuery, getClient } from '@hcengineering/presentation'
import plugin, { Button, Icon, IconShare, Label, Scroller } from '@hcengineering/ui'
import { Integration } from '@hcengineering/setting'
import gmail from '../plugin'
import { convertMessages } from '../utils'
import Messages from './Messages.svelte'
@@ -33,7 +31,6 @@
export let channel: Channel
export let newMessage: boolean
export let enabled: boolean
export let allIntegrations: Integration[]
let plainMessages: Message[] = []
let newMessages: Message[] = []
@@ -76,28 +73,19 @@
const client = getClient()
async function share (): Promise<void> {
// TODO: FIXME
throw new Error('Not implemented')
// const selectedMessages = messages.filter((m) => selected.has(m._id as string as Ref<SharedMessage>))
// await client.addCollection(
// gmail.class.SharedMessages,
// object.space,
// object._id,
// object._class,
// 'gmailSharedMessages',
// {
// messages: convertMessages(
// object,
// channel,
// selectedMessages,
// allIntegrations,
// $personAccountByIdStore,
// $employeeByIdStore
// )
// }
// )
// await inboxClient.readDoc(channel._id)
// clear()
const selectedMessages = messages.filter((m) => selected.has(m._id as string as Ref<SharedMessage>))
await client.addCollection(
gmail.class.SharedMessages,
object.space,
object._id,
object._class,
'gmailSharedMessages',
{
messages: convertMessages(object, channel, selectedMessages)
}
)
await inboxClient.readDoc(channel._id)
clear()
}
function clear (): void {
@@ -140,20 +128,7 @@
{#if messages && messages.length > 0}
<div class="antiVSpacer x2" />
<Scroller padding={'.5rem 1rem'}>
<!-- TODO: FIXME -->
<!-- <Messages
messages={convertMessages(
object,
channel,
messages,
allIntegrations,
$personAccountByIdStore,
$employeeByIdStore
)}
{selectable}
bind:selected
on:select
/> -->
<Messages messages={convertMessages(object, channel, messages)} {selectable} bind:selected on:select />
<div class="antiVSpacer x2" />
</Scroller>
<div class="antiVSpacer x2" />
plugins/gmail-resources/src/components/Connect.svelte
+34 -17
View File
@@ -14,8 +14,9 @@
// limitations under the License.
-->
<script lang="ts">
import { Analytics } from '@hcengineering/analytics'
import { getMetadata, translate } from '@hcengineering/platform'
import { Button, Html, IconClose, Label, themeStore } from '@hcengineering/ui'
import { Button, Html, IconClose, Label, themeStore, IconError } from '@hcengineering/ui'
import { createEventDispatcher } from 'svelte'
import gmail from '../plugin'
import { concatLink } from '@hcengineering/core'
@@ -25,25 +26,34 @@
let connecting = false
const gmailUrl = getMetadata(gmail.metadata.GmailURL) ?? ''
let error = false
async function sendRequest (): Promise<void> {
connecting = true
const link = concatLink(gmailUrl, '/signin')
const url = new URL(link)
url.search = new URLSearchParams({
redirectURL: (getMetadata(presentation.metadata.FrontUrl) ?? window.location.origin) + window.location.pathname
}).toString()
try {
connecting = true
const link = concatLink(gmailUrl, '/signin')
const url = new URL(link)
url.search = new URLSearchParams({
redirectURL: (getMetadata(presentation.metadata.FrontUrl) ?? window.location.origin) + window.location.pathname
}).toString()
const res = await fetch(url.toString(), {
method: 'GET',
headers: {
Authorization: 'Bearer ' + getMetadata(presentation.metadata.Token),
'Content-Type': 'application/json'
}
})
const redirectTo = await res.text()
window.open(redirectTo)
dispatch('close')
const res = await fetch(url.toString(), {
method: 'GET',
headers: {
Authorization: 'Bearer ' + getMetadata(presentation.metadata.Token),
'Content-Type': 'application/json'
}
})
const redirectTo = await res.text()
window.open(redirectTo)
dispatch('close')
} catch (e: any) {
Analytics.handleError(e)
console.error(e)
error = true
} finally {
connecting = false
}
}
let label = ''
@@ -73,6 +83,13 @@
<Html value={label} />
</div>
{#if error}
<div class="flex-row-center gap-2 pt-2">
<IconError size={'medium'} />
<Label label={gmail.string.FailedToConnect} />
</div>
{/if}
<div class="footer">
<Button label={gmail.string.Connect} kind={'primary'} disabled={connecting} on:click={sendRequest} />
</div>
plugins/gmail-resources/src/components/Main.svelte
+27 -25
View File
@@ -16,8 +16,7 @@
<script lang="ts">
/* eslint-disable @typescript-eslint/no-unused-vars */
import contact, { Channel, Contact, getName } from '@hcengineering/contact'
import { employeeByIdStore } from '@hcengineering/contact-resources'
import { Ref } from '@hcengineering/core'
import { Ref, getCurrentAccount } from '@hcengineering/core'
import { Message, SharedMessage } from '@hcengineering/gmail'
import { InboxNotificationsClientImpl } from '@hcengineering/notification-resources'
import { getResource } from '@hcengineering/platform'
@@ -49,8 +48,8 @@
let currentMessage: SharedMessage | undefined = undefined
let newMessage: boolean = false
const allIntegrations: Integration[] = []
const integrations: Integration[] = []
let allIntegrations: Integration[] = []
let integrations: Integration[] = []
let selectedIntegration: Integration | undefined = undefined
channel && inboxClient.forceReadDoc(channel._id, channel._class)
@@ -96,20 +95,30 @@
$: templateProvider && selectedIntegration && templateProvider.set(setting.class.Integration, selectedIntegration)
// TODO: FIXME
settingsQuery.query(setting.class.Integration, { type: gmail.integrationType.Gmail }, (res) => {
// allIntegrations = res.filter((p) => !p.disabled && p.value !== '')
// integrations = allIntegrations.filter((p) => p.createdBy === me || p.shared?.includes(me))
// selectedIntegration = integrations.find((p) => p.createdBy === me) ?? integrations[0]
})
settingsQuery.query(
setting.class.Integration,
{
type: gmail.integrationType.Gmail,
disabled: false
},
(res) => {
allIntegrations = res.filter((p) => !p.disabled && p.value !== '')
const account = getCurrentAccount()
const socialIds = getCurrentAccount().socialIds
const isAvailable = (p: Integration): boolean => {
const isOwner = p.createdBy !== undefined && socialIds.includes(p.createdBy)
const shared = p.shared?.includes(account.uuid) ?? false
return isOwner || shared
}
integrations = allIntegrations.filter(isAvailable)
selectedIntegration = integrations.find((p) => p.createdBy === account.primarySocialId) ?? integrations[0]
}
)
// TODO: FIXME
// $: gmailMessage &&
// channel &&
// object &&
// convertMessage(object, channel, gmailMessage, allIntegrations, $personAccountByIdStore, $employeeByIdStore).then(
// (p) => (currentMessage = p)
// )
$: gmailMessage &&
channel &&
object &&
convertMessage(object, channel, gmailMessage).then((p) => (currentMessage = p))
</script>
{#if channel && object}
@@ -154,14 +163,7 @@
{:else if currentMessage}
<FullMessage {currentMessage} bind:newMessage on:close={back} />
{:else}
<Chats
{object}
{channel}
bind:newMessage
{allIntegrations}
enabled={integrations.length > 0}
on:select={selectHandler}
/>
<Chats {object} {channel} bind:newMessage enabled={integrations.length > 0} on:select={selectHandler} />
{/if}
</Dialog>
{/if}
plugins/gmail-resources/src/plugin.ts
+2 -1
View File
@@ -43,6 +43,7 @@ export default mergeIds(gmailId, gmail, {
Email: '' as IntlString,
HaveWrittenEmail: '' as IntlString,
NewIncomingMessage: '' as IntlString,
GooglePrivacy: '' as IntlString
GooglePrivacy: '' as IntlString,
FailedToConnect: '' as IntlString
}
})
plugins/gmail-resources/src/utils.ts
+47 -71
View File
@@ -1,9 +1,10 @@
// import { getName as getContactName } from '@hcengineering/contact'
import contact, { type Channel, type Contact, type Employee } from '@hcengineering/contact'
import { type Client, type Doc, type IdMap, type Ref } from '@hcengineering/core'
import { get } from 'svelte/store'
import { getName as getContactName } from '@hcengineering/contact'
import contact, { type Channel, type Contact } from '@hcengineering/contact'
import { personByPersonIdStore, employeeBySocialKeyStore } from '@hcengineering/contact-resources'
import { buildSocialIdString, type PersonId, SocialIdType, type Client, type Doc, type Ref } from '@hcengineering/core'
import { type Message, type SharedMessage } from '@hcengineering/gmail'
import { getClient } from '@hcengineering/presentation'
import { type Integration } from '@hcengineering/setting'
import gmail from './plugin'
export function getTime (time: number): string {
@@ -61,78 +62,53 @@ export async function checkHasEmail (doc: Doc | Doc[] | undefined): Promise<bool
return true
}
// const EMAIL_REGEX =
// /(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))/
const EMAIL_REGEX =
/(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))/
export function convertMessages (
object: Contact,
channel: Channel,
messages: Message[],
integrations: Integration[],
accounts: any,
employees: IdMap<Employee>
): SharedMessage[] {
// TODO: FIXME
throw new Error('Not implemented')
// return messages.map((m) => {
// return {
// ...m,
// _id: m._id as string as Ref<SharedMessage>,
// sender: getName(object, channel, m, integrations, accounts, employees, true),
// receiver: getName(object, channel, m, integrations, accounts, employees, false)
// }
// })
export function convertMessages (object: Contact, channel: Channel, messages: Message[]): SharedMessage[] {
return messages.map((m) => {
return {
...m,
_id: m._id as string as Ref<SharedMessage>,
sender: getName(object, channel, m, true),
receiver: getName(object, channel, m, false)
}
})
}
export async function convertMessage (
object: Contact,
channel: Channel,
message: Message,
integrations: Integration[],
accounts: any,
employees: IdMap<Employee>
): Promise<SharedMessage> {
// TODO: FIXME
throw new Error('Not implemented')
// return {
// ...message,
// _id: message._id as string as Ref<SharedMessage>,
// sender: getName(object, channel, message, integrations, accounts, employees, true),
// receiver: getName(object, channel, message, integrations, accounts, employees, false)
// }
export async function convertMessage (object: Contact, channel: Channel, message: Message): Promise<SharedMessage> {
return {
...message,
_id: message._id as string as Ref<SharedMessage>,
sender: getName(object, channel, message, true),
receiver: getName(object, channel, message, false)
}
}
export function getName (
object: Contact,
channel: Channel,
message: Message,
integrations: Integration[],
accounts: any,
employees: IdMap<Employee>,
sender: boolean
): string {
// TODO: FIXME
throw new Error('Not implemented')
// const h = getClient().getHierarchy()
// if (message._class === gmail.class.NewMessage) {
// if (!sender) return `${getContactName(h, object)} (${channel.value})`
// const from = message.from ?? message.createdBy ?? message.modifiedBy
// const account = accounts.get(from)
// const emp = account != null ? employees.get(account?.person as Ref<Employee>) : undefined
// const integration = integrations.find((p) => p.createdBy === from)
// const email = integration?.value ?? integrations[0]?.value
// return emp != null ? `${getContactName(h, emp)} (${email})` : email
// }
// if (message.incoming === sender) {
// return `${getContactName(h, object)} (${channel.value})`
// } else {
// const account = accounts.get(message.modifiedBy)
// const emp = account != null ? employees.get(account?.person as Ref<Employee>) : undefined
// const value = message.incoming ? message.to : message.from
// const email = value.match(EMAIL_REGEX)
// const emailVal = email?.[0] ?? value
// return emp != null ? `${getContactName(h, emp)} (${emailVal})` : emailVal
// }
export function getName (object: Contact, channel: Channel, message: Message, sender: boolean): string {
const h = getClient().getHierarchy()
if (message._class === gmail.class.NewMessage) {
if (!sender) return `${getContactName(h, object)} (${channel.value})`
return getPersonName(message.from ?? message.createdBy ?? message.modifiedBy)
}
if (message.incoming === sender) {
return `${getContactName(h, object)} (${channel.value})`
} else {
return getPersonName(message.modifiedBy)
}
}
export function getPersonName (emailOrId: string): string {
const personName = get(personByPersonIdStore).get(emailOrId as PersonId)?.name
if (personName != null) return personName
const emailSearch = emailOrId.match(EMAIL_REGEX)
const email = emailSearch?.[0]
if (email != null && email !== '') {
const socialId = buildSocialIdString({ type: SocialIdType.EMAIL, value: email })
const name = get(employeeBySocialKeyStore).get(socialId)?.name
if (name != null) return name
}
return emailOrId
}
export async function MessageTitleProvider (client: Client, ref: Ref<Message>, doc?: Message): Promise<string> {
rush.json
+5
View File
@@ -2472,6 +2472,11 @@
"packageName": "@hcengineering/model-achievement",
"projectFolder": "models/achievement",
"shouldPublish": false
},
{
"packageName": "@hcengineering/kvs-client",
"projectFolder": "packages/kvs-client",
"shouldPublish": false
}
]
}
server/account/src/serviceOperations.ts
+3 -3
View File
@@ -549,7 +549,7 @@ export async function addSocialIdToPerson (
const { person, type, value, confirmed, displayValue } = params
const { extra } = decodeTokenVerbose(ctx, token)
verifyAllowedServices(['github', 'telegram-bot'], extra)
verifyAllowedServices(['github', 'telegram-bot', 'gmail'], extra)
return await addSocialId(db, person, type, value, confirmed, displayValue)
}
@@ -564,7 +564,7 @@ export async function updateSocialId (
const { personId, displayValue } = params
const { extra } = decodeTokenVerbose(ctx, token)
verifyAllowedServices(['telegram-bot'], extra)
verifyAllowedServices(['telegram-bot', 'gmail'], extra)
const socialId = await db.socialId.findOne({ _id: personId })
if (socialId != null) {
@@ -815,7 +815,7 @@ export async function findFullSocialIdBySocialKey (
params: { socialKey: string }
): Promise<SocialId | null> {
const { extra } = decodeTokenVerbose(ctx, token)
verifyAllowedServices(['telegram-bot'], extra)
verifyAllowedServices(['telegram-bot', 'gmail'], extra)
const { socialKey } = params
server/account/src/utils.ts
+2 -2
View File
@@ -583,7 +583,7 @@ export async function selectWorkspace (
throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspaceUrl }))
}
if (accountUuid !== systemAccountUuid) {
if (accountUuid !== systemAccountUuid && meta !== undefined) {
void setTimezoneIfNotDefined(ctx, db, accountUuid, account, meta)
}
@@ -1475,7 +1475,7 @@ export async function setTimezoneIfNotDefined (
}
// Move to config?
export const integrationServices = ['github', 'telegram-bot', 'telegram', 'mailbox', 'caldav']
export const integrationServices = ['github', 'telegram-bot', 'telegram', 'mailbox', 'caldav', 'gmail']
export async function findExistingIntegration (
account: AccountUuid,
services/gmail/pod-gmail/.gitignore
+1
View File
@@ -0,0 +1 @@
.env
services/gmail/pod-gmail/README.md
+116
View File
@@ -0,0 +1,116 @@
# Pod Gmail Service
A service that provides Gmail integration capabilities for the Hardcore Engineering Platform. This service handles Gmail authentication, message synchronization, and workspace management.
## Overview
The pod-gmail service is responsible for:
- Managing Gmail client connections
- Synchronizing emails across workspaces
- Handling user authentication and token management
- Providing Gmail integration features for the platform
## Prerequisites
- Node.js 16 or higher
- Gmail API credentials
- Hardcore Engineering Platform dependencies
## Installation
1. Install dependencies:
```bash
npm install
```
2. Configure environment variables:
- Create a `.env` file with necessary credentials
- Configure Gmail API credentials
## Example .env Configuration
Below is an example `.env` file for running the service locally:
```env
PORT=8093
ACCOUNTS_URL="http://127.0.0.1:3000"
SECRET="secret"
WATCH_TOPIC_NAME="test"
Credentials={"web":{"client_id":"get_from_your_gmail_account","client_secret":"get_from_your_gmail_account","redirect_uris":["http://localhost:8088/signin/code"]}}
MINIO_ENDPOINT="localhost"
MINIO_ACCESS_KEY="minioadmin"
MINIO_SECRET_KEY="minioadmin"
KVS_URL="http://localhost:8098"
```
Make sure to replace the values with your actual credentials and configuration.
## Development
### Running Locally
```bash
rushx run-local
```
### Building
```bash
rushx build
```
### Testing
```bash
rushx test
```
## Docker Support
The service can be built using Docker:
```bash
# Build Docker image
rushx docker:build
```
## Configuration
The service requires the following configurations:
- Gmail API credentials
- Workspace-specific settings
- Rate limiting configurations
## API Integration
The service integrates with:
- Gmail API for email operations
- Hardcore Engineering Platform core services
- Server core components
## Dependencies
### Core Dependencies
- @hcengineering/core
- @hcengineering/server-core
- @hcengineering/server-client
- @hcengineering/server-token
- @hcengineering/gmail
### External Dependencies
- googleapis
- google-auth-library
- express
- cors
## License
This project is licensed under the Eclipse Public License, Version 2.0. See the LICENSE file for details.
## Contributing
Contributions are welcome! Please read our contributing guidelines before submitting pull requests.
## Support
For support, please contact the Hardcore Engineering team or open an issue in the repository.
services/gmail/pod-gmail/package.json
+8 -2
View File
@@ -35,6 +35,7 @@
"@types/cors": "^2.8.12",
"@types/express": "^4.17.13",
"@types/node": "~20.11.16",
"@types/sanitize-html": "^2.15.0",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"@typescript-eslint/parser": "^6.11.0",
"esbuild": "^0.24.2",
@@ -50,10 +51,13 @@
"prettier": "^3.1.0",
"ts-node-dev": "^2.0.0",
"typescript": "^5.3.3",
"@types/uuid": "^8.3.1"
"@types/uuid": "^8.3.1",
"ts-node": "^10.8.0",
"cross-env": "~7.0.3"
},
"dependencies": {
"@hcengineering/attachment": "^0.6.14",
"@hcengineering/account-client": "^0.6.0",
"@hcengineering/client": "^0.6.18",
"@hcengineering/client-resources": "^0.6.27",
"@hcengineering/contact": "^0.6.24",
@@ -66,6 +70,7 @@
"@hcengineering/server-client": "^0.6.0",
"@hcengineering/server-storage": "^0.6.0",
"@hcengineering/server-token": "^0.6.11",
"@hcengineering/kvs-client": "^0.6.0",
"cors": "^2.8.5",
"dotenv": "~16.0.0",
"express": "^4.21.2",
@@ -77,6 +82,7 @@
"jwt-simple": "^0.5.6",
"mongodb": "^6.12.0",
"uuid": "^8.3.2",
"@hcengineering/analytics-service": "^0.6.0"
"@hcengineering/analytics-service": "^0.6.0",
"sanitize-html": "^2.15.0"
}
}
services/gmail/pod-gmail/src/__mocks__/config.ts
+12
View File
@@ -0,0 +1,12 @@
const config = {
Port: 8087,
AccountsURL: 'http://localhost:8080',
ServiceID: 'gmail-service-test',
Secret: 'test-secret',
Credentials: 'test-credentials',
WATCH_TOPIC_NAME: 'test-watch-topic',
FooterMessage: '<br><br><p>Sent via <a href="https://huly.io">Huly</a></p>',
InitLimit: 50
}
export default config
services/gmail/pod-gmail/src/__tests__/attachments.test.ts
+223
View File
@@ -0,0 +1,223 @@
import {
MeasureContext,
Ref,
Blob,
TxOperations,
WorkspaceUuid,
Doc,
Space,
Class,
PersonId
} from '@hcengineering/core'
import { StorageAdapter } from '@hcengineering/server-core'
import { gmail_v1 } from 'googleapis'
import { AttachmentHandler } from '../message/attachments'
import type { AttachedFile } from '../types'
import attachment, { Attachment } from '@hcengineering/attachment'
import { decode64, encode64 } from '../base64'
jest.mock('../config')
/* eslint-disable @typescript-eslint/unbound-method */
describe('AttachmentHandler', () => {
const mockCtx: MeasureContext = {
contextData: {},
newChild: jest.fn(),
with: jest.fn(),
withSync: jest.fn(),
withLog: jest.fn(),
logger: {
info: jest.fn(),
error: jest.fn(),
warn: jest.fn(),
logOperation: jest.fn(),
childLogger: jest.fn(),
close: jest.fn()
},
measure: jest.fn(),
error: jest.fn(),
info: jest.fn(),
warn: jest.fn(),
end: jest.fn()
}
const mockWorkspaceId = 'test-workspace' as WorkspaceUuid
const mockStorageAdapter = {
put: jest.fn(),
read: jest.fn()
} as unknown as StorageAdapter
const mockGmail = {
messages: {
attachments: {
get: jest.fn()
}
}
} as unknown as gmail_v1.Resource$Users
const mockClient = {
addCollection: jest.fn(),
findAll: jest.fn()
} as unknown as TxOperations
const getBaseAttachment = (): Attachment => ({
_id: 'test-attachment' as Ref<Attachment>,
name: 'test.txt',
type: 'text/plain',
file: 'test-file' as Ref<Blob>,
size: 0,
lastModified: 0,
attachedTo: '' as Ref<Doc>,
attachedToClass: '' as Ref<Class<Doc>>,
collection: '',
space: '' as Ref<Space>,
modifiedOn: 0,
modifiedBy: '' as PersonId,
_class: '' as Ref<Class<Attachment>>
})
let attachmentHandler: AttachmentHandler
beforeEach(() => {
attachmentHandler = new AttachmentHandler(mockCtx, mockWorkspaceId, mockStorageAdapter, mockGmail, mockClient)
})
describe('addAttachement', () => {
it('should not add attachment if it already exists', async () => {
const file: AttachedFile = {
file: 'test-file',
name: 'test.txt',
type: 'text/plain',
size: 100,
lastModified: Date.now()
}
const message = { _id: 'test-message', _class: 'test-class', space: 'test-space' }
const currentAttachments: Attachment[] = [
{
...getBaseAttachment(),
name: 'test.txt',
lastModified: file.lastModified
}
]
await attachmentHandler.addAttachement(file, message, currentAttachments)
expect(mockStorageAdapter.put).not.toHaveBeenCalled()
expect(mockClient.addCollection).not.toHaveBeenCalled()
})
it('should add new attachment', async () => {
const file: AttachedFile = {
file: 'test-file',
name: 'test.txt',
type: 'text/plain',
size: 100,
lastModified: Date.now()
}
const message = { _id: 'test-message', _class: 'test-class', space: 'test-space' }
const currentAttachments: Attachment[] = []
await attachmentHandler.addAttachement(file, message, currentAttachments)
expect(mockStorageAdapter.put).toHaveBeenCalled()
expect(mockClient.addCollection).toHaveBeenCalledWith(
attachment.class.Attachment,
message.space,
message._id,
message._class,
'attachments',
expect.any(Object)
)
})
})
describe('getPartFiles', () => {
it('should return empty array for undefined part', async () => {
const result = await attachmentHandler.getPartFiles(undefined, 'test-message')
expect(result).toEqual([])
})
it('should get files from attachment', async () => {
const part: gmail_v1.Schema$MessagePart = {
filename: 'test.txt',
mimeType: 'text/plain',
body: {
attachmentId: 'test-attachment',
size: 100
}
}
const mockAttachment = {
data: 'test-data',
size: 100
}
;(mockGmail.messages.attachments.get as jest.Mock).mockResolvedValue({ data: mockAttachment })
const result = await attachmentHandler.getPartFiles(part, 'test-message')
expect(result).toEqual([
{
file: 'test-data',
name: 'test.txt',
type: 'text/plain',
size: 100,
lastModified: expect.any(Number)
}
])
})
it('should get files from body data', async () => {
const part: gmail_v1.Schema$MessagePart = {
filename: 'test.txt',
mimeType: 'text/plain',
body: {
data: 'test-data',
size: 100
}
}
const result = await attachmentHandler.getPartFiles(part, 'test-message')
expect(result).toEqual([
{
file: 'test-data',
name: 'test.txt',
type: 'text/plain',
size: 100,
lastModified: expect.any(Number)
}
])
})
})
describe('makeAttachmentsBody', () => {
it('should create message body with attachments', async () => {
const message = {
to: 'test@example.com',
subject: 'Test Subject',
content: 'Test Content',
_id: 'test-message'
}
const from = 'from@example.com'
const attachment: Attachment = {
...getBaseAttachment(),
_id: 'test-attachment' as Ref<Attachment>,
name: 'test.txt',
type: 'text/plain',
file: 'test-file' as Ref<Blob>
}
const mockAttachments: Attachment[] = [attachment]
;(mockClient.findAll as jest.Mock).mockResolvedValue(mockAttachments)
;(mockStorageAdapter.read as jest.Mock).mockResolvedValue([Buffer.from('test-data', 'utf8')])
const result = await attachmentHandler.makeAttachmentsBody(message, from)
const decodedResult = decode64(result)
expect(decodedResult).toContain('Content-Type: multipart/mixed')
expect(decodedResult).toContain('To: test@example.com')
expect(decodedResult).toContain('From: from@example.com')
expect(decodedResult).toContain(`Subject: =?UTF-8?B?${encode64('Test Subject')}?= \n`)
expect(decodedResult).toContain('Test Content')
expect(decodedResult).toContain('test.txt')
expect(decodedResult).toContain('Content-Disposition: attachment; filename="test.txt"')
expect(decodedResult).toContain('<br><br><p>Sent via <a href="https://huly.io">Huly</a></p>')
})
})
})
services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts
+345
View File
@@ -0,0 +1,345 @@
import {
AccountUuid,
PersonId,
WorkspaceUuid,
type SocialId,
type SocialIdType,
type MeasureContext
} from '@hcengineering/core'
import { GmailClient } from '../gmail'
import { ProjectCredentials, ProjectCredentialsData, Token, User } from '../types'
import { TokenStorage } from '../tokens'
import { SyncManager } from '../message/sync'
// Mock dependencies
const mockedGoogleAuth = {
generateAuthUrl: jest.fn().mockReturnValue('https://auth-url.com'),
getToken: jest.fn().mockResolvedValue({ tokens: { access_token: 'new-token' } }),
setCredentials: jest.fn(),
refreshAccessToken: jest.fn().mockResolvedValue({ credentials: { access_token: 'refreshed-token' } }),
revokeCredentials: jest.fn().mockResolvedValue({})
}
const mockedGmailUsers = {
getProfile: jest.fn().mockResolvedValue({ data: { emailAddress: 'test@example.com' } }),
messages: {
send: jest.fn().mockResolvedValue({}),
list: jest.fn().mockResolvedValue({ data: { messages: [] } })
},
watch: jest.fn().mockResolvedValue({}),
stop: jest.fn().mockResolvedValue({})
}
// Mock all imports before they're used
jest.mock('@hcengineering/core', () => {
return {
AccountUuid: String,
PersonId: String,
WorkspaceUuid: String,
// Provide missing properties
configUserAccountUuid: 'test-user-id',
core: {
space: {
Workspace: 'workspace'
}
},
TxOperations: jest.fn().mockImplementation(() => ({
findAll: jest.fn().mockResolvedValue([]),
findOne: jest.fn().mockResolvedValue(undefined),
createDoc: jest.fn().mockResolvedValue({}),
update: jest.fn().mockResolvedValue({}),
remove: jest.fn().mockResolvedValue({}),
updateDoc: jest.fn().mockResolvedValue({}),
tx: jest.fn().mockResolvedValue({})
}))
}
})
jest.mock('googleapis', () => ({
gmail_v1: {},
google: {
auth: {
OAuth2: jest.fn().mockImplementation(() => mockedGoogleAuth)
},
gmail: jest.fn().mockImplementation(() => ({
users: mockedGmailUsers
}))
}
}))
jest.mock('../tokens')
jest.mock('../message/message')
jest.mock('../message/sync')
jest.mock('../message/attachments')
jest.mock('@hcengineering/server-client', () => ({
getAccountClient: jest.fn()
}))
jest.mock('../accounts', () => ({
getOrCreateSocialId: jest.fn().mockResolvedValue({ _id: 'test-social-id' })
}))
jest.mock('../gmail/utils', () => ({
getEmail: jest.fn().mockResolvedValue('test@example.com')
}))
jest.mock('../integrations', () => ({
createIntegrationIfNotEsixts: jest.fn().mockResolvedValue({ _id: 'test-integration-id' }),
disableIntegration: jest.fn(),
removeIntegration: jest.fn()
}))
jest.mock('../utils', () => ({
addFooter: jest.fn().mockImplementation((content: string) => content),
isToken: jest.fn().mockImplementation((obj: any) => 'token' in obj),
serviceToken: jest.fn().mockReturnValue('service-token'),
getKvsClient: jest.fn().mockImplementation(() => ({
getValue: jest.fn(),
setValue: jest.fn().mockImplementation(() => Promise.resolve(undefined)),
deleteKey: jest.fn().mockImplementation(() => Promise.resolve(undefined)),
listKeys: jest.fn()
}))
}))
// Mock gmail module
jest.mock('@hcengineering/gmail', () => ({
integrationType: {
Gmail: 'gmail'
},
class: {
NewMessage: 'class.NewMessage'
}
}))
// Mock setting module
jest.mock('@hcengineering/setting', () => ({
class: {
Integration: 'class.Integration'
}
}))
// Mock config
jest.mock('../config', () => ({
WATCH_TOPIC_NAME: 'test-topic'
}))
describe('GmailClient', () => {
const mockContext: MeasureContext = {
info: jest.fn(),
error: jest.fn(),
measure: jest.fn().mockImplementation((name: string, fn: () => any) => fn()),
with: jest.fn().mockImplementation((data: any, fn: () => any) => fn())
} as any
const mockWorkspaceId = 'test-workspace' as WorkspaceUuid
const mockUserId = 'test-user-id' as AccountUuid
// Fix the SocialId type issue
const mockSocialId: SocialId = {
_id: 'test-social-id' as PersonId,
value: 'test@example.com',
type: 'email' as SocialIdType,
key: 'email'
}
const credentialsData: ProjectCredentialsData = {
client_id: 'client-id',
client_secret: 'client-secret',
redirect_uris: ['https://redirect-uri.com'],
project_id: 'test-project',
auth_uri: 'https://auth-uri.com',
token_uri: 'https://token-uri.com',
auth_provider_x509_cert_url: 'https://cert-url.com'
}
const mockCredentials: ProjectCredentials = {
web: credentialsData
}
const mockUser: User = {
userId: mockUserId,
workspace: mockWorkspaceId,
socialId: mockSocialId,
email: 'test@example.com',
token: 'user-token'
}
const mockToken: Token = {
userId: mockUserId,
workspace: mockWorkspaceId,
socialId: mockSocialId,
token: 'test-token',
access_token: 'test-access-token',
refresh_token: 'test-refresh-token',
scope: 'test-scope',
token_type: 'Bearer',
expiry_date: 123456789
}
const mockClient = {
findAll: jest.fn().mockResolvedValue([]),
findOne: jest.fn().mockResolvedValue(undefined),
createDoc: jest.fn().mockResolvedValue({}),
update: jest.fn().mockResolvedValue({}),
remove: jest.fn().mockResolvedValue({}),
updateDoc: jest.fn().mockResolvedValue({}),
tx: jest.fn().mockResolvedValue({})
}
const mockWorkspaceClient = {
subscribeMessages: jest.fn().mockResolvedValue({}),
signoutBySocialId: jest.fn().mockResolvedValue({})
}
const mockStorageAdapter = {}
beforeEach(() => {
jest.clearAllMocks()
// Fix unbound method errors by using the proper mock assignment approach
jest.spyOn(TokenStorage.prototype, 'getToken').mockResolvedValue(mockToken)
jest.spyOn(TokenStorage.prototype, 'saveToken').mockResolvedValue({} as any)
jest.spyOn(TokenStorage.prototype, 'deleteToken').mockResolvedValue()
jest.spyOn(SyncManager.prototype, 'sync').mockResolvedValue()
})
it('should create a GmailClient instance', async () => {
const client = await GmailClient.create(
mockContext as any,
mockCredentials,
mockUser,
mockClient as any,
mockWorkspaceClient as any,
mockWorkspaceId,
mockStorageAdapter as any
)
expect(client).toBeInstanceOf(GmailClient)
expect(TokenStorage).toHaveBeenCalled()
expect(mockContext.info).toHaveBeenCalledWith('Created gmail client', expect.any(Object))
})
it('should create a GmailClient instance with auth code', async () => {
const client = await GmailClient.create(
mockContext as any,
mockCredentials,
mockUser,
mockClient as any,
mockWorkspaceClient as any,
mockWorkspaceId,
mockStorageAdapter as any,
'auth-code'
)
expect(client).toBeInstanceOf(GmailClient)
expect(mockedGoogleAuth.setCredentials).toHaveBeenCalled()
expect(mockContext.info).toHaveBeenCalledWith('Created gmail client', expect.any(Object))
})
it('should generate auth URL', async () => {
const client = await GmailClient.create(
mockContext as any,
mockCredentials,
mockUser,
mockClient as any,
mockWorkspaceClient as any,
mockWorkspaceId,
mockStorageAdapter as any
)
const authUrl = client.getAuthUrl('https://callback-url.com')
expect(authUrl).toBe('https://auth-url.com')
})
it('should sign out and clean up resources', async () => {
const client = await GmailClient.create(
mockContext as any,
mockCredentials,
mockUser,
mockClient as any,
mockWorkspaceClient as any,
mockWorkspaceId,
mockStorageAdapter as any
)
await client.signout()
// Fix unbound method reference by using the spy
expect(jest.spyOn(TokenStorage.prototype, 'deleteToken')).toHaveBeenCalledWith(mockSocialId._id)
expect(mockContext.info).toHaveBeenCalledWith('Deactivate gmail client', expect.any(Object))
})
it('should create a message', async () => {
const client = await GmailClient.create(
mockContext as any,
mockCredentials,
mockUser,
mockClient as any,
mockWorkspaceClient as any,
mockWorkspaceId,
mockStorageAdapter as any
)
const newMessage = {
_id: 'test-message-id',
_class: 'class.NewMessage',
space: 'space',
status: 'new',
from: mockSocialId._id,
to: 'recipient@example.com',
subject: 'Test Subject',
content: '<p>Test content</p>'
}
// Mock internal TxOperations instance
const mockTxOperations = {
findAll: jest.fn().mockResolvedValue([]),
findOne: jest.fn().mockResolvedValue(undefined),
createDoc: jest.fn().mockResolvedValue({}),
update: jest.fn().mockResolvedValue({}),
remove: jest.fn().mockResolvedValue({}),
updateDoc: jest.fn().mockResolvedValue({}),
tx: jest.fn().mockResolvedValue({})
}
// Access and replace the private TxOperations instance
Object.defineProperty(client, 'client', {
value: mockTxOperations,
writable: true
})
await client.createMessage(newMessage as any)
expect(mockTxOperations.updateDoc).toHaveBeenCalledWith('class.NewMessage', 'space', 'test-message-id', {
status: 'sent'
})
})
it('should sync messages', async () => {
const client = await GmailClient.create(
mockContext as any,
mockCredentials,
mockUser,
mockClient as any,
mockWorkspaceClient as any,
mockWorkspaceId,
mockStorageAdapter as any
)
await client.sync()
// Fix unbound method reference by using the spy
expect(jest.spyOn(SyncManager.prototype, 'sync')).toHaveBeenCalledWith(mockSocialId._id, 'test@example.com')
})
it('should initialize integration', async () => {
const client = await GmailClient.create(
mockContext as any,
mockCredentials,
mockUser,
mockClient as any,
mockWorkspaceClient as any,
mockWorkspaceId,
mockStorageAdapter as any
)
await client.initIntegration()
expect(mockContext.info).toHaveBeenCalledWith('Init integration', expect.any(Object))
})
})
services/gmail/pod-gmail/src/__tests__/message.test.ts
+227
View File
@@ -0,0 +1,227 @@
import { type MeasureContext, PersonId, TxOperations, AttachedData } from '@hcengineering/core'
import { type GaxiosResponse } from 'gaxios'
import { gmail_v1 } from 'googleapis'
import { type Message } from '@hcengineering/gmail'
import { type Channel } from '../types'
import { AttachmentHandler } from '../message/attachments'
import { MessageManager, sanitizeText } from '../message/message'
/* eslint-disable @typescript-eslint/consistent-type-assertions */
/* eslint-disable @typescript-eslint/unbound-method */
jest.mock('../config')
describe('MessageManager', () => {
let messageManager: MessageManager
let mockCtx: MeasureContext
let mockClient: TxOperations
let mockAttachmentHandler: AttachmentHandler
let mockSocialId: PersonId
let mockWorkspace: { getChannel: (email: string) => Channel | undefined }
beforeEach(() => {
mockCtx = {
measure: jest.fn(),
with: jest.fn()
} as unknown as MeasureContext
mockClient = {
findOne: jest.fn(),
findAll: jest.fn(),
tx: jest.fn()
} as unknown as TxOperations
mockAttachmentHandler = {
getPartFiles: jest.fn(),
addAttachement: jest.fn()
} as unknown as AttachmentHandler
mockSocialId = 'test-social-id' as PersonId
mockWorkspace = {
getChannel: jest.fn()
}
messageManager = new MessageManager(mockCtx, mockClient, mockAttachmentHandler, mockSocialId, mockWorkspace)
})
describe('saveMessage', () => {
const createMockGmailResponse = (): GaxiosResponse<gmail_v1.Schema$Message> => ({
config: {},
data: {
id: 'test-message-id',
internalDate: '1234567890',
payload: {
headers: [
{ name: 'From', value: 'sender@example.com' },
{ name: 'To', value: 'recipient@example.com' },
{ name: 'Message-ID', value: 'test-message-id' },
{ name: 'Subject', value: 'Test Subject' }
],
parts: [
{
mimeType: 'text/plain',
body: { data: 'dGVzdCBjb250ZW50' }
},
{
mimeType: 'text/html',
body: { data: 'PHA+dGVzdCBjb250ZW50PC9wPg==' }
}
]
}
},
status: 200,
statusText: 'OK',
headers: {},
request: {
responseURL: 'https://example.com'
}
})
const createMockMessage = (): AttachedData<Message> => ({
messageId: 'test-message-id',
textContent: 'test content',
subject: 'Test Subject',
content: 'test content',
sendOn: 1234567890,
from: 'sender@example.com',
to: 'recipient@example.com',
copy: [],
incoming: true
})
it('should save message with attachments', async () => {
const mockMessage = createMockGmailResponse()
const mockChannel = { _id: 'test-channel-id' } as Channel
mockWorkspace.getChannel = jest.fn().mockReturnValue(mockChannel)
mockClient.findOne = jest.fn().mockResolvedValue(undefined)
mockAttachmentHandler.getPartFiles = jest.fn().mockResolvedValue([])
await messageManager.saveMessage(mockMessage, 'test@example.com')
expect(mockClient.tx).toHaveBeenCalled()
expect(mockAttachmentHandler.getPartFiles).toHaveBeenCalled()
})
it('should update existing message', async () => {
const mockMessage = createMockGmailResponse()
const mockChannel = { _id: 'test-channel-id' } as Channel
const existingMessage = createMockMessage()
mockWorkspace.getChannel = jest.fn().mockReturnValue(mockChannel)
mockClient.findOne = jest.fn().mockResolvedValue(existingMessage)
mockAttachmentHandler.getPartFiles = jest.fn().mockResolvedValue([])
await messageManager.saveMessage(mockMessage, 'test@example.com')
expect(mockClient.tx).toHaveBeenCalled()
expect(mockAttachmentHandler.getPartFiles).toHaveBeenCalled()
})
it('should not save message when no channels found', async () => {
const mockMessage = createMockGmailResponse()
mockWorkspace.getChannel = jest.fn().mockReturnValue(undefined)
await messageManager.saveMessage(mockMessage, 'test@example.com')
expect(mockClient.tx).not.toHaveBeenCalled()
expect(mockAttachmentHandler.getPartFiles).not.toHaveBeenCalled()
})
it('should handle incoming messages correctly', async () => {
const mockMessage = createMockGmailResponse()
const mockChannel = { _id: 'test-channel-id' } as Channel
mockWorkspace.getChannel = jest.fn().mockReturnValue(mockChannel)
mockClient.findOne = jest.fn().mockResolvedValue(undefined)
mockAttachmentHandler.getPartFiles = jest.fn().mockResolvedValue([])
await messageManager.saveMessage(mockMessage, 'recipient@example.com')
expect(mockWorkspace.getChannel).toHaveBeenCalledWith('sender@example.com')
})
it('should handle outgoing messages correctly', async () => {
const mockMessage = createMockGmailResponse()
const mockChannel = { _id: 'test-channel-id' } as Channel
mockWorkspace.getChannel = jest.fn().mockReturnValue(mockChannel)
mockClient.findOne = jest.fn().mockResolvedValue(undefined)
mockAttachmentHandler.getPartFiles = jest.fn().mockResolvedValue([])
await messageManager.saveMessage(mockMessage, 'sender@example.com')
expect(mockWorkspace.getChannel).toHaveBeenCalledWith('recipient@example.com')
})
})
})
describe('sanitizeHtml', () => {
it('should remove all HTML tags', () => {
const html = '<p>This is <b>bold</b> and <i>italic</i> text</p>'
const expected = 'This is bold and italic text'
expect(sanitizeText(html)).toBe(expected)
})
it('should handle nested tags properly', () => {
const html = '<div><p>Nested <span>tags <strong>should</strong> be</span> removed</p></div>'
const expected = 'Nested tags should be removed'
expect(sanitizeText(html)).toBe(expected)
})
it('should handle malformed HTML', () => {
const html = 'Text with <unclosed tag and <p>paragraph</p>'
const expected = 'Text with paragraph'
expect(sanitizeText(html)).toBe(expected)
})
it('should handle scripts and potential XSS attacks', () => {
const html = 'Hello <script>alert("XSS");</script> world'
const expected = 'Hello world'
expect(sanitizeText(html)).toBe(expected)
})
it('should remove style tags and attributes', () => {
const html = '<p style="color:red">Styled text</p>'
const expected = 'Styled text'
expect(sanitizeText(html)).toBe(expected)
})
it('should handle empty input', () => {
expect(sanitizeText('')).toBe('')
})
})
describe('sanitizeText', () => {
it('should remove all HTML tags', () => {
const html = '<p>This is <b>bold</b> and <i>italic</i> text</p>'
const expected = 'This is bold and italic text'
expect(sanitizeText(html)).toBe(expected)
})
it('should handle nested tags properly', () => {
const html = '<div><p>Nested <span>tags <strong>should</strong> be</span> removed</p></div>'
const expected = 'Nested tags should be removed'
expect(sanitizeText(html)).toBe(expected)
})
it('should handle malformed HTML', () => {
const html = 'Text with <unclosed tag and <p>paragraph</p>'
const expected = 'Text with paragraph'
expect(sanitizeText(html)).toBe(expected)
})
it('should handle scripts and potential XSS attacks', () => {
const html = 'Hello <script>alert("XSS");</script> world'
const expected = 'Hello world'
expect(sanitizeText(html)).toBe(expected)
})
it('should remove style tags and attributes', () => {
const html = '<p style="color:red">Styled text</p>'
const expected = 'Styled text'
expect(sanitizeText(html)).toBe(expected)
})
it('should handle empty input', () => {
expect(sanitizeText('')).toBe('')
})
})
services/gmail/pod-gmail/src/__tests__/sync.test.ts
+337
View File
@@ -0,0 +1,337 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { MeasureContext, PersonId } from '@hcengineering/core'
import { SyncManager, SyncMutex } from '../message/sync'
import { MessageManager } from '../message/message'
import { RateLimiter } from '../rateLimiter'
jest.mock('../config')
jest.mock('../message/message')
jest.mock('../utils', () => {
const originalModule = jest.requireActual('../utils')
return {
...originalModule,
getKvsClient: jest.fn().mockImplementation(() => mockKeyValueClientInstance)
}
})
// Create mock types
interface MockKeyValueClient {
getValue: jest.Mock
setValue: jest.Mock
deleteKey: jest.Mock
listKeys: jest.Mock
}
// Create a shared mock instance that can be referenced in the mock above
const mockKeyValueClientInstance = {
getValue: jest.fn(),
setValue: jest.fn().mockImplementation(() => Promise.resolve(undefined)),
deleteKey: jest.fn().mockImplementation(() => Promise.resolve(undefined)),
listKeys: jest.fn()
}
describe('SyncManager', () => {
// Mocked dependencies
let mockCtx: MeasureContext
let mockMessageManager: jest.Mocked<MessageManager>
let mockGmail: any // Using any for easier mocking
let mockKeyValueClient: MockKeyValueClient
// Test constants
const workspace = 'test-workspace'
const userId = 'user1' as PersonId
const userEmail = 'user1@example.com'
const historyId = '12345'
// Initialize SyncManager instance for each test
let syncManager: SyncManager
beforeEach(() => {
// Setup mocks
mockCtx = {
info: jest.fn(),
error: jest.fn()
} as unknown as jest.Mocked<MeasureContext>
mockMessageManager = {
saveMessage: jest.fn().mockResolvedValue(undefined)
} as unknown as jest.Mocked<MessageManager>
mockGmail = {
history: {
list: jest.fn().mockImplementation(() =>
Promise.resolve({
data: {
history: [],
nextPageToken: undefined
}
})
)
},
messages: {
list: jest.fn().mockImplementation(() =>
Promise.resolve({
data: {
messages: [],
nextPageToken: undefined
}
})
),
get: jest.fn().mockImplementation(() =>
Promise.resolve({
data: {
id: 'test-id',
historyId
}
})
)
}
}
mockKeyValueClient = {
getValue: jest.fn(),
setValue: jest.fn().mockImplementation(() => Promise.resolve(undefined)),
deleteKey: jest.fn().mockImplementation(() => Promise.resolve(undefined)),
listKeys: jest.fn()
}
// Create SyncManager instance
syncManager = new SyncManager(
mockCtx,
mockMessageManager,
mockGmail,
workspace,
mockKeyValueClient as any,
new RateLimiter(100, 100)
)
})
afterEach(() => {
jest.clearAllMocks()
})
describe('getHistory', () => {
it('should retrieve history from key-value store', async () => {
// Arrange
const mockHistory = {
historyId,
userId,
workspace
}
mockKeyValueClient.getValue.mockResolvedValue(mockHistory)
// Act
const result = await (syncManager as any).getHistory(userId)
// Assert
expect(mockKeyValueClient.getValue).toHaveBeenCalledWith(`history:${workspace}:${userId}`)
expect(result).toEqual(mockHistory)
})
it('should return null when history not found', async () => {
// Arrange
mockKeyValueClient.getValue.mockResolvedValue(null)
// Act
const result = await (syncManager as any).getHistory(userId)
// Assert
expect(result).toBeNull()
})
})
describe('setHistoryId', () => {
it('should store history in key-value store', async () => {
// Act
await (syncManager as any).setHistoryId(userId, historyId)
// Assert
expect(mockKeyValueClient.setValue).toHaveBeenCalledWith(`history:${workspace}:${userId}`, {
historyId,
userId,
workspace
})
})
})
describe('clearHistory', () => {
it('should delete history from key-value store', async () => {
// Act
await (syncManager as any).clearHistory(userId)
// Assert
expect(mockKeyValueClient.deleteKey).toHaveBeenCalledWith(`history:${workspace}:${userId}`)
})
})
describe('sync', () => {
it('should perform full sync when no history exists', async () => {
// Arrange
mockKeyValueClient.getValue.mockResolvedValue(null)
const spyFullSync = jest.spyOn(syncManager, 'fullSync').mockResolvedValue(undefined)
// Act
await syncManager.sync(userId, userEmail)
// Assert
expect(spyFullSync).toHaveBeenCalledWith(userId, userEmail)
})
it('should perform part sync when history exists', async () => {
// Arrange
mockKeyValueClient.getValue.mockResolvedValue({
historyId,
userId,
workspace
})
// Act
await syncManager.sync(userId, userEmail)
// Assert
expect(mockGmail.history.list).toHaveBeenCalledWith({
userId: 'me',
historyTypes: ['messageAdded'],
startHistoryId: historyId,
pageToken: undefined
})
})
})
describe('getMessage', () => {
it('should retrieve a message by ID', async () => {
// Arrange
const messageId = 'msg123'
const mockResponse = {
data: {
id: messageId,
historyId
}
}
mockGmail.messages.get.mockImplementation(() => Promise.resolve(mockResponse))
// Act
const result = await (syncManager as any).getMessage(messageId)
// Assert
expect(mockGmail.messages.get).toHaveBeenCalledWith({
id: messageId,
userId: 'me',
format: 'FULL'
})
expect(result).toEqual(mockResponse)
})
})
})
describe('SyncMutex', () => {
let syncMutex: SyncMutex
beforeEach(() => {
syncMutex = new SyncMutex()
})
it('should allow sequential locking and unlocking', async () => {
// Lock first time
const release1 = await syncMutex.lock('test-key')
release1()
// Lock second time
const release2 = await syncMutex.lock('test-key')
release2()
// If we got here without hanging, the test passes
expect(true).toBe(true)
})
it('should queue up multiple requests for the same key', async () => {
const results: number[] = []
// Start 3 concurrent lock operations
const promise1 = (async () => {
const release = await syncMutex.lock('test-key')
results.push(1)
await new Promise((resolve) => setTimeout(resolve, 10))
release()
})()
const promise2 = (async () => {
const release = await syncMutex.lock('test-key')
results.push(2)
await new Promise((resolve) => setTimeout(resolve, 5))
release()
})()
const promise3 = (async () => {
const release = await syncMutex.lock('test-key')
results.push(3)
release()
})()
// Wait for all promises to resolve
await Promise.all([promise1, promise2, promise3])
// The operations should have happened in order
expect(results).toEqual([1, 2, 3])
})
it('should allow concurrent operations on different keys', async () => {
const results: string[] = []
// Lock two different keys concurrently
const promise1 = (async () => {
const release = await syncMutex.lock('key1')
results.push('key1-locked')
await new Promise((resolve) => setTimeout(resolve, 20))
results.push('key1-unlocked')
release()
})()
const promise2 = (async () => {
const release = await syncMutex.lock('key2')
results.push('key2-locked')
await new Promise((resolve) => setTimeout(resolve, 10))
results.push('key2-unlocked')
release()
})()
// Wait for both promises to resolve
await Promise.all([promise1, promise2])
// key2 operations should complete before key1 due to shorter timeout
expect(results.indexOf('key2-locked')).toBeLessThan(results.indexOf('key1-unlocked'))
})
it('should release the lock properly even if an error occurs', async () => {
try {
const release = await syncMutex.lock('test-key')
try {
throw new Error('Test error')
} finally {
release()
}
} catch (error) {
// Ignore the error
}
// Should be able to acquire the lock again
const release = await syncMutex.lock('test-key')
release()
expect(true).toBe(true)
})
})
services/gmail/pod-gmail/src/__tests__/tokens.test.ts
+139
View File
@@ -0,0 +1,139 @@
import { AccountUuid, PersonId, WorkspaceUuid } from '@hcengineering/core'
import { AccountClient, IntegrationSecret } from '@hcengineering/account-client'
import { getAccountClient } from '@hcengineering/server-client'
import { TokenStorage } from '../tokens'
import { GMAIL_INTEGRATION, SecretType, Token } from '../types'
import type { MeasureContext } from '@hcengineering/core'
// Mock the getAccountClient function
jest.mock('@hcengineering/server-client', () => ({
getAccountClient: jest.fn()
}))
describe('TokenStorage', () => {
const mockSocialId = 'test-social-id' as PersonId
const mockWorkspace = 'test-workspace' as WorkspaceUuid
const mockPlatformToken = 'test-platform-token'
const mockUserId = 'test-user' as AccountUuid
const mockToken: Token = {
userId: mockUserId,
socialId: { _id: '123' } as any,
workspace: mockWorkspace,
token: 'test-token',
access_token: 'test-access-token',
refresh_token: 'test-refresh-token',
scope: 'test-scope',
token_type: 'Bearer',
expiry_date: 123456789
}
let tokenStorage: TokenStorage
let mockAccountClient: jest.Mocked<AccountClient>
// Add mock MeasureContext
const mockMeasureContext: MeasureContext = {
info: jest.fn(),
error: jest.fn()
} as any
beforeEach(() => {
// Create mock AccountClient
mockAccountClient = {
getIntegrationSecret: jest.fn(),
addIntegrationSecret: jest.fn(),
updateIntegrationSecret: jest.fn(),
deleteIntegrationSecret: jest.fn()
} as any
// Set up getAccountClient mock
;(getAccountClient as jest.Mock).mockReturnValue(mockAccountClient)
// Create TokenStorage instance with MeasureContext
tokenStorage = new TokenStorage(mockMeasureContext, mockWorkspace, mockPlatformToken)
})
afterEach(() => {
jest.clearAllMocks()
})
describe('getToken', () => {
it('should return null when no token exists', async () => {
mockAccountClient.getIntegrationSecret.mockResolvedValue(null)
const result = await tokenStorage.getToken(mockSocialId)
expect(result).toBeNull()
expect(mockAccountClient.getIntegrationSecret).toHaveBeenCalledWith({
key: SecretType.TOKEN,
kind: GMAIL_INTEGRATION,
socialId: mockSocialId,
workspaceUuid: mockWorkspace
})
})
it('should return parsed token when token exists', async () => {
const mockSecret: IntegrationSecret = {
socialId: mockSocialId,
kind: GMAIL_INTEGRATION,
workspaceUuid: mockWorkspace,
key: SecretType.TOKEN,
secret: JSON.stringify(mockToken)
}
mockAccountClient.getIntegrationSecret.mockResolvedValue(mockSecret)
const result = await tokenStorage.getToken(mockSocialId)
expect(result).toEqual(mockToken)
})
})
describe('saveToken', () => {
it('should update token when it exists', async () => {
const existingToken = { existing: 'token' }
const mockSecret: IntegrationSecret = {
socialId: mockSocialId,
kind: GMAIL_INTEGRATION,
workspaceUuid: mockWorkspace,
key: SecretType.TOKEN,
secret: JSON.stringify(existingToken)
}
mockAccountClient.getIntegrationSecret.mockResolvedValue(mockSecret)
await tokenStorage.saveToken(mockSocialId, mockToken)
// Expect merged token (existing token properties + new token properties)
const expectedMergedToken = {
...existingToken,
...mockToken
}
expect(mockAccountClient.updateIntegrationSecret).toHaveBeenCalledWith({
key: SecretType.TOKEN,
kind: GMAIL_INTEGRATION,
socialId: mockSocialId,
secret: JSON.stringify(expectedMergedToken),
workspaceUuid: mockWorkspace
})
expect(mockAccountClient.addIntegrationSecret).not.toHaveBeenCalled()
})
it('should add new token when it does not exist', async () => {
mockAccountClient.getIntegrationSecret.mockResolvedValue(null)
await tokenStorage.saveToken(mockSocialId, mockToken)
expect(mockAccountClient.addIntegrationSecret).toHaveBeenCalledWith({
key: SecretType.TOKEN,
kind: GMAIL_INTEGRATION,
socialId: mockSocialId,
secret: JSON.stringify(mockToken),
workspaceUuid: mockWorkspace
})
expect(mockAccountClient.updateIntegrationSecret).not.toHaveBeenCalled()
})
})
describe('deleteToken', () => {
it('should delete the token', async () => {
await tokenStorage.deleteToken(mockSocialId)
expect(mockAccountClient.deleteIntegrationSecret).toHaveBeenCalledWith({
key: SecretType.TOKEN,
kind: GMAIL_INTEGRATION,
socialId: mockSocialId,
workspaceUuid: mockWorkspace
})
})
})
})
services/gmail/pod-gmail/src/accounts.ts
+74
View File
@@ -0,0 +1,74 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { AccountUuid, Person, PersonId, SocialId, SocialIdType, buildSocialIdString } from '@hcengineering/core'
import { getAccountClient } from '@hcengineering/server-client'
import { generateToken } from '@hcengineering/server-token'
import { serviceToken } from './utils'
export async function getAccountPerson (account: AccountUuid): Promise<Person | undefined> {
try {
const accountClient = getAccountClient(generateToken(account))
return await accountClient.getPerson()
} catch (e) {
console.error(e)
}
return undefined
}
export async function getAccountSocialIds (account: AccountUuid): Promise<SocialId[]> {
try {
const accountClient = getAccountClient(generateToken(account))
return await accountClient.getSocialIds()
} catch (e) {
console.error(e)
}
return []
}
export async function getOrCreateSocialIdByEmail (
token: string,
account: AccountUuid,
email: string
): Promise<PersonId> {
const client = getAccountClient(token)
const socialId = await client.findFullSocialIdBySocialKey(
buildSocialIdString({ type: SocialIdType.EMAIL, value: email })
)
if (socialId != null) return socialId._id
return await client.addSocialIdToPerson(account, SocialIdType.EMAIL, email, true)
}
export async function getOrCreateSocialId (account: AccountUuid, email: string): Promise<SocialId> {
const accountClient = getAccountClient(serviceToken())
let socialId = await accountClient.findFullSocialIdBySocialKey(
buildSocialIdString({ type: SocialIdType.EMAIL, value: email })
)
if (socialId == null) {
await accountClient.addSocialIdToPerson(account, SocialIdType.EMAIL, email, true)
socialId = await accountClient.findFullSocialIdBySocialKey(
buildSocialIdString({ type: SocialIdType.EMAIL, value: email })
)
if (socialId == null) {
throw new Error('Cannot create social id')
}
}
if (socialId.personUuid !== account) {
throw new Error('Social id connected to another account')
}
return socialId
}
services/gmail/pod-gmail/src/config.ts
+8 -11
View File
@@ -13,12 +13,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { config as dotenvConfig } from 'dotenv'
dotenvConfig()
interface Config {
Port: number
MongoURI: string
MongoDB: string
AccountsURL: string
ServiceID: string
Secret: string
@@ -26,21 +26,19 @@ interface Config {
WATCH_TOPIC_NAME: string
FooterMessage: string
InitLimit: number
KvsUrl: string
}
const envMap: { [key in keyof Config]: string } = {
Port: 'PORT',
MongoURI: 'MONGO_URI',
MongoDB: 'MONGO_DB',
AccountsURL: 'ACCOUNTS_URL',
ServiceID: 'SERVICE_ID',
Secret: 'SECRET',
Credentials: 'Credentials',
WATCH_TOPIC_NAME: 'WATCH_TOPIC_NAME',
FooterMessage: 'FOOTER_MESSAGE',
InitLimit: 'INIT_LIMIT'
InitLimit: 'INIT_LIMIT',
KvsUrl: 'KVS_URL'
}
const parseNumber = (str: string | undefined): number | undefined => (str !== undefined ? Number(str) : undefined)
@@ -48,15 +46,14 @@ const parseNumber = (str: string | undefined): number | undefined => (str !== un
const config: Config = (() => {
const params: Partial<Config> = {
Port: parseNumber(process.env[envMap.Port]) ?? 8087,
MongoDB: process.env[envMap.MongoDB] ?? 'gmail-service',
MongoURI: process.env[envMap.MongoURI],
AccountsURL: process.env[envMap.AccountsURL],
ServiceID: process.env[envMap.ServiceID] ?? 'gmail-service',
Secret: process.env[envMap.Secret],
Credentials: process.env[envMap.Credentials],
WATCH_TOPIC_NAME: process.env[envMap.WATCH_TOPIC_NAME],
InitLimit: parseNumber(process.env[envMap.InitLimit]) ?? 50,
FooterMessage: process.env[envMap.FooterMessage] ?? '<br><br><p>Sent via <a href="https://huly.io">Huly</a></p>'
FooterMessage: process.env[envMap.FooterMessage] ?? '<br><br><p>Sent via <a href="https://huly.io">Huly</a></p>',
KvsUrl: process.env[envMap.KvsUrl]
}
const missingEnv = (Object.keys(params) as Array<keyof Config>)
services/gmail/pod-gmail/src/gmail.ts
+229 -574
View File
@@ -13,44 +13,30 @@
// limitations under the License.
//
import attachment, { Attachment } from '@hcengineering/attachment'
import core, {
AttachedData,
Blob,
Client,
Data,
MeasureContext,
Ref,
Timestamp,
TxCreateDoc,
TxCUD,
TxFactory,
TxOperations,
TxProcessor,
TxUpdateDoc,
WorkspaceUuid
} from '@hcengineering/core'
import gmail, { type Message, type NewMessage } from '@hcengineering/gmail'
import core, { AccountUuid, Client, MeasureContext, SocialId, TxOperations, WorkspaceUuid } from '@hcengineering/core'
import gmail, { type NewMessage } from '@hcengineering/gmail'
import { type StorageAdapter } from '@hcengineering/server-core'
import setting from '@hcengineering/setting'
import type { GaxiosResponse } from 'gaxios'
import type { Credentials, OAuth2Client } from 'google-auth-library'
import { gmail_v1, google } from 'googleapis'
import type { Collection, Db } from 'mongodb'
import { v4 as uuid } from 'uuid'
import { arrayBufferToBase64, decode64, encode64 } from './base64'
import { encode64 } from './base64'
import config from './config'
import { GmailController } from './gmailController'
import { RateLimiter } from './rateLimiter'
import type { AttachedFile, Channel, History, ProjectCredentials, Token, User } from './types'
import { diffAttributes, isToken } from './utils'
import type { ProjectCredentials, Token, User } from './types'
import { addFooter, isToken, serviceToken, getKvsClient } from './utils'
import type { WorkspaceClient } from './workspaceClient'
import { getOrCreateSocialId } from './accounts'
import { createIntegrationIfNotEsixts, disableIntegration, removeIntegration } from './integrations'
import { AttachmentHandler } from './message/attachments'
import { TokenStorage } from './tokens'
import { MessageManager } from './message/message'
import { SyncManager } from './message/sync'
import { getEmail } from './gmail/utils'
import { Integration } from '@hcengineering/account-client'
const SCOPES = ['https://www.googleapis.com/auth/gmail.modify']
const EMAIL_REGEX =
/(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))/
function makeHTMLBody (message: NewMessage, from: string): string {
const str = [
'Content-Type: text/html; charset="UTF-8"\n',
@@ -78,65 +64,6 @@ function makeHTMLBody (message: NewMessage, from: string): string {
return encode64(res).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '')
}
function addFooter (message: string): string {
if (config.FooterMessage === undefined || config.FooterMessage.trim() === '') return message
return message + config.FooterMessage.trim()
}
function convertMessage (
message: GaxiosResponse<gmail_v1.Schema$Message>,
me: string
): AttachedData<Message> & { modifiedOn: Timestamp } {
const date = message.data.internalDate != null ? new Date(Number.parseInt(message.data.internalDate)) : new Date()
const from = getHeaderValue(message.data.payload, 'From') ?? ''
const to = getHeaderValue(message.data.payload, 'To') ?? ''
const copy =
getHeaderValue(message.data.payload, 'Cc')
?.split(',')
.map((p) => p.trim()) ?? undefined
const incoming = !from.includes(me)
return {
modifiedOn: date.getTime(),
messageId: getHeaderValue(message.data.payload, 'Message-ID') ?? '',
replyTo: getHeaderValue(message.data.payload, 'In-Reply-To'),
copy,
content: getPartMessage(message.data.payload, 'text/html'),
textContent: getPartMessage(message.data.payload, 'text/plain').replace(/<[^>]*>/g, ''),
from,
to,
incoming,
subject: getHeaderValue(message.data.payload, 'Subject') ?? '',
sendOn: date.getTime()
}
}
function getHeaderValue (payload: gmail_v1.Schema$MessagePart | undefined, name: string): string | undefined {
if (payload === undefined) return undefined
const headers = payload.headers
return headers?.find((header) => header.name?.toLowerCase() === name.toLowerCase())?.value ?? undefined
}
function getPartsMessage (parts: gmail_v1.Schema$MessagePart[] | undefined, mime: string): string {
let result = ''
if (parts !== undefined) {
const htmlPart = parts.find((part) => part.mimeType === mime)
const filtredParts = htmlPart !== undefined ? parts.filter((part) => part.mimeType === mime) : parts
for (const part of filtredParts ?? []) {
result += getPartMessage(part, mime)
}
}
return result
}
function getPartMessage (part: gmail_v1.Schema$MessagePart | undefined, mime: string): string {
if (part === undefined) return ''
if (part.body?.data != null) {
return decode64(part.body.data)
}
return getPartsMessage(part.parts, mime)
}
async function wait (sec: number): Promise<void> {
await new Promise<void>((resolve) => {
setTimeout(() => {
@@ -146,56 +73,116 @@ async function wait (sec: number): Promise<void> {
}
export class GmailClient {
private readonly oAuth2Client: OAuth2Client
private readonly gmail: gmail_v1.Resource$Users
private me: string | undefined = undefined
private readonly tokens: Collection<Token>
private readonly histories: Collection<History>
private readonly account: AccountUuid
private email: string | undefined = undefined
private readonly tokenStorage: TokenStorage
private readonly client: TxOperations
private watchTimer: NodeJS.Timeout | undefined = undefined
private refreshTimer: NodeJS.Timeout | undefined = undefined
private syncPromise: Promise<void> | undefined = undefined
private readonly rateLimiter = new RateLimiter(1000, 200) // in fact 250, but let's make reserve
private readonly rateLimiter = new RateLimiter(1000, 200)
private readonly attachmentHandler: AttachmentHandler
private readonly messageManager: MessageManager
private readonly syncManager: SyncManager
private readonly integrationToken: string
private integration: Integration | undefined = undefined
private constructor (
private readonly ctx: MeasureContext,
credentials: ProjectCredentials,
private readonly oAuth2Client: OAuth2Client,
private readonly gmail: gmail_v1.Resource$Users,
private readonly user: User,
mongo: Db,
client: Client,
private readonly workspaceId: WorkspaceUuid,
private readonly storageAdapter: StorageAdapter,
private readonly workspace: WorkspaceClient
workspaceId: WorkspaceUuid,
storageAdapter: StorageAdapter,
private readonly workspace: WorkspaceClient,
email: string,
private socialId: SocialId
) {
const { client_secret, client_id, redirect_uris } = credentials.web // eslint-disable-line
this.oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]) // eslint-disable-line
this.gmail = google.gmail({ version: 'v1', auth: this.oAuth2Client }).users
this.tokens = mongo.collection<Token>('tokens')
this.histories = mongo.collection<History>('histories')
this.client = new TxOperations(client, this.user.userId)
this.email = email
this.integrationToken = serviceToken()
this.tokenStorage = new TokenStorage(this.ctx, workspaceId, this.integrationToken)
this.client = new TxOperations(client, this.socialId._id)
this.account = this.user.userId
this.attachmentHandler = new AttachmentHandler(ctx, workspaceId, storageAdapter, this.gmail, this.client)
this.messageManager = new MessageManager(
ctx,
this.client,
this.attachmentHandler,
this.socialId._id,
this.workspace
)
const keyValueClient = getKvsClient(this.integrationToken)
this.syncManager = new SyncManager(
ctx,
this.messageManager,
this.gmail,
this.user.workspace,
keyValueClient,
this.rateLimiter
)
}
static async create (
ctx: MeasureContext,
credentials: ProjectCredentials,
user: User | Token,
mongo: Db,
client: Client,
workspace: WorkspaceClient,
workspaceId: WorkspaceUuid,
storageAdapter: StorageAdapter
storageAdapter: StorageAdapter,
authCode?: string
): Promise<GmailClient> {
const gmailClient = new GmailClient(ctx, credentials, user, mongo, client, workspaceId, storageAdapter, workspace)
if (isToken(user)) {
console.log('Setting token while creating', user.workspace, user.userId, user)
await gmailClient.setToken(user)
// eslint-disable-next-line @typescript-eslint/naming-convention
const { client_secret, client_id, redirect_uris } = credentials.web
const oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0])
const googleClient = google.gmail({ version: 'v1', auth: oAuth2Client }).users
let token: Credentials | undefined = isToken(user) ? user : undefined
if (token === undefined && authCode !== undefined) {
const tokenResponse = await oAuth2Client.getToken(authCode)
token = {
...user,
...tokenResponse.tokens
}
oAuth2Client.setCredentials(token)
} else if (token !== undefined) {
oAuth2Client.setCredentials(token)
}
const email = user.email ?? (await getEmail(googleClient))
if (email === undefined) {
throw new Error('Cannot retrieve user email')
}
const socialId = user.socialId ?? (await getOrCreateSocialId(user.userId, email))
if (socialId?._id == null) {
throw new Error(`Cannot create gmail client without social id: ${user.userId}, ${workspaceId}`)
}
user.socialId = socialId
const gmailClient = new GmailClient(
ctx,
oAuth2Client,
googleClient,
user,
client,
workspaceId,
storageAdapter,
workspace,
email,
socialId
)
await gmailClient.createIntregration()
if (token !== undefined) {
ctx.info('Setting token while creating', { workspaceUuid: user.workspace, userId: user.userId })
await gmailClient.setToken(token)
await gmailClient.refreshToken()
await gmailClient.addClient()
}
ctx.info('Created gmail client', { workspaceUuid: user.workspace, userId: user.userId })
void gmailClient.initIntegration()
return gmailClient
}
getAutUrl (redirectURL: string): string {
getAuthUrl (redirectURL: string): string {
const { workspace, userId } = this.user
const authUrl = this.oAuth2Client.generateAuthUrl({
access_type: 'offline',
@@ -205,42 +192,54 @@ export class GmailClient {
return authUrl
}
async authorize (code: string): Promise<void> {
const token = await this.oAuth2Client.getToken(code)
console.log('Setting token after authorize', this.user.workspace, this.user.userId, token)
await this.setToken(token.tokens)
await this.refreshToken()
await this.addClient()
void this.startSync()
void this.getNewMessagesAfterAuth()
async initIntegration (): Promise<void> {
try {
this.ctx.info('Init integration', { workspaceUuid: this.user.workspace, userId: this.user.userId })
const me = await this.getMe()
const integrations = await this.client.findAll(setting.class.Integration, {
createdBy: this.user.userId,
type: gmail.integrationType.Gmail
})
const updated = integrations.find((p) => p.disabled && p.value === me)
void this.startSync()
void this.getNewMessagesAfterAuth()
for (const integration of integrations.filter((p) => p._id !== updated?._id)) {
await this.client.remove(integration)
const integrations = await this.client.findAll(setting.class.Integration, {
createdBy: this.socialId._id,
type: gmail.integrationType.Gmail
})
const updated = integrations.find((p) => p.disabled && p.value === this.email)
for (const integration of integrations.filter((p) => p._id !== updated?._id)) {
await this.client.remove(integration)
}
if (updated !== undefined) {
await this.client.update(updated, {
disabled: false
})
} else {
await this.client.createDoc(setting.class.Integration, core.space.Workspace, {
type: gmail.integrationType.Gmail,
disabled: false,
value: this.socialId._id
})
}
} catch (err: any) {
this.ctx.info('Failed to start message sync', {
workspaceUuid: this.user.workspace,
userId: this.user.userId,
error: err.message
})
}
if (updated !== undefined) {
await this.client.update(updated, {
disabled: false
})
} else {
await this.client.createDoc(setting.class.Integration, core.space.Workspace, {
type: gmail.integrationType.Gmail,
disabled: false,
value: me
})
}
async createIntregration (): Promise<void> {
try {
this.integration = await createIntegrationIfNotEsixts(this.socialId._id, this.user.workspace)
} catch (err: any) {
this.ctx.error('Failed to create integration', { socialdId: this.socialId, workspace: this.workspace })
}
}
async getNewMessagesAfterAuth (): Promise<void> {
const newMessages = await this.client.findAll(gmail.class.NewMessage, {
status: 'new',
from: this.user.userId
from: this.socialId._id
})
await this.workspace.subscribeMessages()
for (const message of newMessages) {
@@ -251,11 +250,12 @@ export class GmailClient {
async createMessage (message: NewMessage): Promise<void> {
if (message.status === 'sent') return
try {
const me = await this.getMe()
this.ctx.info('Send gmail message', { id: message._id, from: message.from, to: message.to })
const email = await this.getEmail()
const body =
message.attachments != null && message.attachments > 0
? await this.makeAttachmentsBody(message, me)
: makeHTMLBody(message, me)
? await this.makeAttachmentsBody(message, email)
: makeHTMLBody(message, email)
await this.rateLimiter.take(100)
await this.gmail.messages.send({
userId: 'me',
@@ -272,37 +272,31 @@ export class GmailClient {
status: 'error',
error: JSON.stringify(err)
})
console.log('Create message error', this.user.workspace, this.user.userId, err.message)
this.ctx.error('Create message error', {
workspaceUuid: this.user.workspace,
userId: this.user.userId,
message: err.message
})
if (err?.response?.data?.error === 'invalid_grant') {
await this.refreshToken()
}
}
}
private async checkErrorWithSignout (err: any): Promise<boolean> {
if (this.checkError(err)) {
console.log('Invalid grant error', this.user.workspace, this.user.userId)
await this.workspace.signoutByUserId(this.user.userId, true)
return true
}
return false
}
private checkError (err: any): boolean {
return err?.response?.data?.error === 'invalid_grant'
}
async signout (byError: boolean = false): Promise<void> {
this.ctx.info('Deactivate gmail client', { socialId: this.socialId, workspace: this.user.workspace })
try {
await this.close()
await this.oAuth2Client.revokeCredentials()
} catch {}
await this.tokens.deleteOne({
userId: this.user.userId,
workspace: this.user.workspace
})
await this.tokenStorage.deleteToken(this.socialId._id)
await this.disableIntegration(byError)
const integration = await this.client.findOne(setting.class.Integration, {
createdBy: this.user.userId,
createdBy: this.socialId._id,
type: gmail.integrationType.Gmail
})
if (integration !== undefined) {
@@ -314,26 +308,19 @@ export class GmailClient {
}
}
async sync (): Promise<void> {
if (this.syncPromise !== undefined) {
await this.syncPromise
}
const history = await this.getHistory()
try {
if (history != null) {
this.syncPromise = this.partSync(history.historyId)
await this.syncPromise
async disableIntegration (byError: boolean = false): Promise<void> {
if (this.integration !== undefined) {
if (byError) {
await disableIntegration(this.integration)
} else {
this.syncPromise = this.fullSync()
await this.syncPromise
await removeIntegration(this.socialId._id, this.user.workspace)
}
} finally {
this.syncPromise = undefined
}
}
async startSync (): Promise<void> {
await this.sync()
this.ctx.info('Start sync', { workspaceUuid: this.user.workspace, userId: this.user.userId, email: this.email })
await this.syncManager.sync(this.socialId._id, this.email)
await this.watch()
// recall every 24 hours https://developers.google.com/gmail/api/guides/push
this.watchTimer = setInterval(() => {
@@ -341,64 +328,72 @@ export class GmailClient {
}, 86400000)
}
async sync (): Promise<void> {
this.ctx.info('Sync', { workspaceUuid: this.user.workspace, userId: this.user.userId })
await this.syncManager.sync(this.socialId._id, this.email)
}
async newChannel (value: string): Promise<void> {
const query = `{from:${value} to:${value} cc:${value} bcc:${value}}`
await this.fullSync(query)
}
private async getMe (): Promise<string> {
if (this.me !== undefined) {
return this.me
private async getEmail (): Promise<string> {
if (this.email !== undefined) {
return this.email
}
for (let i = 0; i < 3; i++) {
await this.rateLimiter.take(1)
const profile = await this.gmail.getProfile({
userId: 'me'
})
this.me = profile.data.emailAddress ?? undefined
if (this.me !== undefined) return this.me
this.email = profile.data.emailAddress ?? undefined
if (this.email !== undefined) return this.email
await wait(5)
}
throw new Error("Can't get profile email after 3 attempts")
}
private async getCurrentToken (): Promise<Token | null> {
return await this.tokens.findOne({
userId: this.user.userId,
workspace: this.user.workspace
})
private async syncUserInfo (credentials: Credentials): Promise<void> {
if (credentials.access_token == null) return
if (this.email === undefined) {
this.ctx.error('User email is missing', { workspaceUuid: this.user.workspace, userId: this.user.userId })
return
}
this.socialId = await getOrCreateSocialId(this.account, this.email)
}
private async updateCurrentToken (token: Credentials): Promise<void> {
await this.tokens.updateOne(
{
userId: this.user.userId,
workspace: this.user.workspace
},
{
$set: {
...token
}
}
)
private async getCurrentToken (): Promise<Token | null> {
return await this.tokenStorage.getToken(this.socialId._id)
}
private async addClient (): Promise<void> {
try {
const me = await this.getMe()
this.ctx.info('Register client', { socialId: this.socialId._id, email: this.email })
const controller = GmailController.getGmailController()
controller.addClient(me, this)
controller.addClient(this.socialId._id, this)
} catch (err) {
console.log('Add client error', this.user.workspace, this.user.userId, (err as any).message)
this.ctx.error('Add client error', {
workspaceUuid: this.user.workspace,
userId: this.user.userId,
message: (err as any).message
})
}
}
private async setToken (token: Credentials): Promise<void> {
try {
this.oAuth2Client.setCredentials(token)
await this.getMe()
const email = await this.getEmail()
await this.syncUserInfo(token)
await this.updateSocialId(email)
await this.updateToken(token)
} catch (err: any) {
console.log('Set token error', this.user.workspace, this.user.userId, err.message)
this.ctx.error('Set token error', {
workspaceUuid: this.user.workspace,
userId: this.user.userId,
message: err.message
})
if (this.checkError(err)) {
await this.signout(true)
}
@@ -406,338 +401,31 @@ export class GmailClient {
}
}
private async updateSocialId (email: string): Promise<void> {
const existingEmail = this.socialId !== undefined ? this.socialId.value : undefined
if (existingEmail === email) return
this.socialId = await getOrCreateSocialId(this.user.userId, email)
}
private async updateToken (token: Credentials): Promise<void> {
try {
const currentToken = await this.getCurrentToken()
if (currentToken != null) {
await this.updateCurrentToken(token)
} else {
await this.tokens.insertOne({
userId: this.user.userId,
workspace: this.user.workspace,
token: this.user.token,
...token
})
}
await this.tokenStorage.saveToken(this.socialId._id, token as Token)
} catch (err) {
console.log('update token error', this.user.workspace, this.user.userId, (err as any).message)
}
}
private async getHistory (): Promise<History | null> {
return await this.histories.findOne({
userId: this.user.userId,
workspace: this.user.workspace
})
}
private async clearHistory (): Promise<void> {
await this.histories.deleteOne({
userId: this.user.userId,
workspace: this.user.workspace
})
}
private async setHistoryId (historyId: string): Promise<void> {
await this.histories.updateOne(
{
this.ctx.error('update token error', {
workspaceUuid: this.user.workspace,
userId: this.user.userId,
workspace: this.user.workspace
},
{
$set: {
historyId
}
},
{ upsert: true }
)
}
private async partSync (historyId: string): Promise<void> {
let pageToken: string | undefined
let histories: GaxiosResponse<gmail_v1.Schema$ListHistoryResponse>
while (true) {
try {
await this.rateLimiter.take(2)
histories = await this.gmail.history.list({
userId: 'me',
historyTypes: ['messageAdded'],
startHistoryId: historyId,
pageToken
})
} catch (err) {
console.log('Part sync get history error', this.user.workspace, this.user.userId, err)
const signout = await this.checkErrorWithSignout(err)
if (signout) return
await this.clearHistory()
await this.sync()
return
}
const nextPageToken = histories.data.nextPageToken
const array = histories.data.history ?? []
try {
for (const history of array) {
for (const message of history.messagesAdded ?? []) {
if (message.message?.id == null || (message.message.labelIds?.includes('DRAFT') ?? false)) {
continue
}
try {
const res = await this.getMessage(message.message.id)
await this.saveMessage(res)
} catch (err) {
console.log('Part sync message error', this.user.workspace, this.user.userId, message.message.id, err)
const signout = await this.checkErrorWithSignout(err)
if (signout) return
}
}
if (history.id != null) {
await this.setHistoryId(history.id)
}
}
if (nextPageToken == null) {
return
} else {
pageToken = nextPageToken
}
} catch (err) {
console.log('Part sync error', this.user.workspace, this.user.userId, err)
return
}
message: (err as any).message
})
}
}
private async fullSync (q?: string): Promise<void> {
const pageToken: string | undefined = undefined
const query: gmail_v1.Params$Resource$Users$Messages$List = {
userId: 'me',
pageToken
}
if (q !== undefined) {
query.q = q
}
let currentHistoryId: string | undefined
console.log('start full sync', this.user.workspace, this.user.userId, q)
try {
const messagesIds: string[] = []
while (true) {
await this.rateLimiter.take(5)
const messages = await this.gmail.messages.list(query)
if (query.pageToken == null) {
console.log('Total messages', this.user.workspace, this.user.userId, messages.data.resultSizeEstimate)
}
const ids = messages.data.messages?.map((p) => p.id) ?? []
for (let index = 0; index < ids.length; index++) {
const id = ids[index]
if (id == null) continue
messagesIds.push(id)
}
if (messages.data.nextPageToken == null) {
console.log('Break, total new messages: ', messagesIds.length)
break
}
query.pageToken = messages.data.nextPageToken
}
for (let index = messagesIds.length - 1; index >= 0; index--) {
const id = messagesIds[index]
try {
const message = await this.getMessage(id)
const historyId = message.data.historyId
await this.saveMessage(message)
if (historyId != null && q === undefined) {
if (currentHistoryId == null || Number(currentHistoryId) < Number(historyId)) {
await this.setHistoryId(historyId)
currentHistoryId = historyId
}
}
} catch (err: any) {
console.log('Full sync message error', this.user.workspace, this.user.userId, err)
const signout = await this.checkErrorWithSignout(err)
if (signout) return
}
}
} catch (err) {
console.log('Full sync error', this.user.workspace, this.user.userId, err)
await this.checkErrorWithSignout(err)
}
}
private async getMessage (id: string): Promise<GaxiosResponse<gmail_v1.Schema$Message>> {
await this.rateLimiter.take(5)
return await this.gmail.messages.get({
id,
userId: 'me',
format: 'FULL'
this.ctx.info('Full sync client', {
workspaceUuid: this.user.workspace,
userId: this.user.userId,
email: this.email
})
}
private findChannels (res: AttachedData<Message>): Channel[] {
const recievers = res.copy != null ? [res.to, ...res.copy] : [res.to]
const values = res.incoming ? [res.from] : recievers
const result: Channel[] = []
for (const value of values) {
const res = value.match(EMAIL_REGEX)
if (res !== null) {
const email = res[0]
const channel = this.workspace.getChannel(email)
if (channel !== undefined) result.push(channel)
}
}
return result
}
private async saveMessage (message: GaxiosResponse<gmail_v1.Schema$Message>): Promise<void> {
const me = await this.getMe()
const res = convertMessage(message, me)
const channels = this.findChannels(res)
if (channels.length === 0) return
const attachments = await this.getPartFiles(message.data.payload, message.data.id ?? '')
const factory = new TxFactory(this.user.userId)
for (const channel of channels) {
const current = await this.client.findOne(gmail.class.Message, {
messageId: res.messageId,
attachedTo: channel._id
})
const tx = current != null ? this.updateTx(res, current, factory, channel) : this.createTx(res, factory, channel)
if (tx !== undefined) {
const resultMessage =
current != null
? TxProcessor.updateDoc2Doc(current, tx as TxUpdateDoc<Message>)
: TxProcessor.createDoc2Doc(tx as TxCreateDoc<Message>)
await this.client.tx(tx)
if (attachments.length > 0) {
const currentAttachemtns: Attachment[] =
current !== undefined
? await this.client.findAll(attachment.class.Attachment, { attachedTo: current._id })
: []
for (const attachment of attachments) {
await this.addAttachement(attachment, resultMessage, currentAttachemtns)
}
}
}
}
}
private async addAttachement (file: AttachedFile, message: Message, currentAttachemtns: Attachment[]): Promise<void> {
try {
if (currentAttachemtns.findIndex((p) => p.name === file.name && p.lastModified === file.lastModified) !== -1) {
return
}
const id = uuid()
const data: AttachedData<Attachment> = {
name: file.name,
file: id as Ref<Blob>,
type: file.type ?? 'undefined',
size: file.size ?? Buffer.from(file.file, 'base64').length,
lastModified: file.lastModified
}
await this.storageAdapter.put(this.ctx, this.workspaceId as any, id, file.file, data.type, data.size) // TODO: FIXME
await this.client.addCollection(
attachment.class.Attachment,
message.space,
message._id,
message._class,
'attachments',
data
)
} catch (err: any) {
console.log('Add attachment error', this.user.workspace, this.user.userId, err.message)
}
}
private async getPartFiles (
part: gmail_v1.Schema$MessagePart | undefined,
messageId: string
): Promise<AttachedFile[]> {
if (part === undefined) return []
if (part.filename != null && part.filename.length > 0) {
if (part.body?.attachmentId != null) {
const attachment = await this.getAttachmentById(part.body?.attachmentId, messageId)
if (attachment.data == null) return []
return [
{
file: attachment.data,
name: part.filename,
type: part.mimeType ?? undefined,
size: attachment.size ?? undefined,
lastModified: new Date().getTime()
}
]
}
if (part.body?.data == null) return []
return [
{
file: part.body.data,
name: part.filename,
type: part.mimeType ?? undefined,
size: part.body.size ?? undefined,
lastModified: new Date().getTime()
}
]
}
return await this.getPartsFiles(part.parts, messageId)
}
private async getAttachmentById (attachmentId: string, messageId: string): Promise<gmail_v1.Schema$MessagePartBody> {
await this.rateLimiter.take(5)
const res = await this.gmail.messages.attachments.get({ id: attachmentId, userId: 'me', messageId })
return res.data
}
private async getPartsFiles (
parts: gmail_v1.Schema$MessagePart[] | undefined,
messageId: string
): Promise<AttachedFile[]> {
const result: AttachedFile[] = []
if (parts !== undefined) {
const filtredParts = parts.filter((part) => part.filename != null && part.filename.length > 0)
for (const part of filtredParts ?? []) {
const res = await this.getPartFiles(part, messageId)
if (res.length > 0) {
result.push(...res)
}
}
}
return result
}
private createTx (
message: AttachedData<Message> & { modifiedOn: Timestamp },
factory: TxFactory,
channel: Channel
): TxCUD<Message> {
const tx = factory.createTxCollectionCUD<Channel, Message>(
channel._class,
channel._id,
channel.space,
'items',
factory.createTxCreateDoc<Message>(
gmail.class.Message,
core.space.Workspace,
message as unknown as Data<Message>,
undefined,
message.modifiedOn
),
message.modifiedOn
)
return tx
}
private updateTx (
message: AttachedData<Message> & { modifiedOn: Timestamp },
current: Message,
factory: TxFactory,
channel: Channel
): TxCUD<Message> | undefined {
const operations = diffAttributes(current, message)
if (Object.keys(operations).length === 0) return undefined
const tx = factory.createTxCollectionCUD<Channel, Message>(
channel._class,
channel._id,
channel.space,
'items',
factory.createTxUpdateDoc<Message>(current._class, current.space, current._id, operations)
)
return tx
await this.syncManager.fullSync(this.socialId._id, this.email, q)
}
private async refreshToken (): Promise<void> {
@@ -752,9 +440,13 @@ export class GmailClient {
30 * 60 * 1000
)
} catch (err: any) {
console.log("Couldn't refresh token, error:", err.message)
this.ctx.error("Couldn't refresh token", {
workspaceUuid: this.user.workspace,
userId: this.user.userId,
message: err.message
})
if (err?.response?.data?.error === 'invalid_grant') {
await this.workspace.signoutByUserId(this.user.userId, true)
await this.workspace.signoutBySocialId(this.socialId._id, true)
} else {
this.refreshTimer = setTimeout(
() => {
@@ -777,62 +469,21 @@ export class GmailClient {
}
})
} catch (err) {
console.log('Watch error', (err as any).message)
this.ctx.error('Watch error', {
workspaceUuid: this.user.workspace,
userId: this.user.userId,
message: (err as any).message
})
}
}
private async makeAttachmentsBody (message: NewMessage, from: string): Promise<string> {
const str = [
'Content-Type: multipart/mixed; boundary="mail"\n',
'MIME-Version: 1.0\n',
`To: ${message.to} \n`,
`From: ${from} \n`
]
if (message.replyTo != null) {
str.push(`In-Reply-To: ${message.replyTo} \n`)
try {
return await this.attachmentHandler.makeAttachmentsBody(message, from)
} catch (err: any) {
this.ctx.error('Failed to make attachment body', err)
return makeHTMLBody(message, from)
}
if (message.copy != null && message.copy.length > 0) {
str.push(`Cc: ${message.copy.join(', ')} \n`)
}
if (message.subject != null) {
str.push(`Subject: =?UTF-8?B?${encode64(message.subject)}?= \n`)
}
str.push('\n\n')
str.push('--mail\n')
str.push('Content-Type: text/html; charset="UTF-8"\n')
str.push('MIME-Version: 1.0\n')
str.push('Content-Transfer-Encoding: 7bit\n\n')
str.push(addFooter(message.content))
str.push('\n\n')
const attachments = await this.client.findAll(attachment.class.Attachment, { attachedTo: message._id })
for (const attachment of attachments) {
const attachmentStrings = await this.makeAttachmentPart(attachment)
str.push(...attachmentStrings)
}
str.push('--mail--')
return encode64(str.join('')).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '')
}
private async makeAttachmentPart (attachment: Attachment): Promise<string[]> {
const buffer = await this.storageAdapter.read(this.ctx, this.workspaceId as any, attachment.file) // TODO: FIXME
const data = arrayBufferToBase64(Buffer.concat(buffer))
const res: string[] = []
res.push('--mail\n')
res.push(`Content-Type: ${attachment.type}\n`)
res.push('MIME-Version: 1.0\n')
res.push('Content-Transfer-Encoding: base64\n')
res.push(`Content-Disposition: attachment; filename="${attachment.name}"\n\n`)
res.push(data)
res.push('\n\n')
return res
}
async close (): Promise<void> {
@@ -844,7 +495,11 @@ export class GmailClient {
userId: 'me'
})
} catch (err) {
console.log('close error', (err as any).message)
this.ctx.error('close error', {
workspaceUuid: this.user.workspace,
userId: this.user.userId,
message: (err as any).message
})
}
}
}
services/gmail/pod-gmail/src/gmail/auth.ts
+46
View File
@@ -0,0 +1,46 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { google } from 'googleapis'
import { OAuth2Client } from 'google-auth-library'
import { ProjectCredentials } from '../types'
export class AuthProvider {
private oAuth2Client: OAuth2Client | undefined
private readonly credentials: ProjectCredentials
constructor (credentials: ProjectCredentials) {
this.credentials = credentials
this.initClient()
}
initClient (): void {
// eslint-disable-next-line @typescript-eslint/naming-convention
const { client_secret, client_id, redirect_uris } = this.credentials.web
this.oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0])
}
getAuthUrl (redirectURL: string, user: { workspace: string, userId: string }): string {
const { workspace, userId } = user
if (this.oAuth2Client === undefined) {
throw new Error('OAuth client not initialized')
}
const authUrl = this.oAuth2Client.generateAuthUrl({
access_type: 'offline',
scope: ['https://www.googleapis.com/auth/gmail.modify'],
state: Buffer.from(JSON.stringify({ workspace, userId, redirectURL })).toString('base64')
})
return authUrl
}
}
services/gmail/pod-gmail/src/gmail/utils.ts
+28
View File
@@ -0,0 +1,28 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { wait } from '../utils'
import { gmail_v1 } from 'googleapis'
export async function getEmail (gmail: gmail_v1.Resource$Users): Promise<string> {
for (let i = 0; i < 3; i++) {
const profile = await gmail.getProfile({
userId: 'me'
})
const email = profile.data.emailAddress ?? undefined
if (email !== undefined) return email
await wait(5)
}
throw new Error("Can't get profile email after 3 attempts")
}
services/gmail/pod-gmail/src/gmailController.ts
+78 -59
View File
@@ -13,41 +13,52 @@
// limitations under the License.
//
import { isActiveMode, MeasureContext, RateLimiter, systemAccountUuid, WorkspaceUuid } from '@hcengineering/core'
import {
AccountUuid,
isActiveMode,
MeasureContext,
RateLimiter,
WorkspaceUuid,
type PersonId
} from '@hcengineering/core'
import type { StorageAdapter } from '@hcengineering/server-core'
import { type Db } from 'mongodb'
import { decode64 } from './base64'
import config from './config'
import { type GmailClient } from './gmail'
import { type ProjectCredentials, type Token, type User } from './types'
import { WorkspaceClient } from './workspaceClient'
import { generateToken } from '@hcengineering/server-token'
import { getAccountClient } from '@hcengineering/server-client'
import { getIntegrations } from './integrations'
import { serviceToken } from './utils'
import { getWorkspaceTokens } from './tokens'
import { AuthProvider } from './gmail/auth'
export class GmailController {
private readonly workspaces: Map<string, WorkspaceClient> = new Map<string, WorkspaceClient>()
private readonly credentials: ProjectCredentials
private readonly clients: Map<string, GmailClient[]> = new Map<string, GmailClient[]>()
private readonly clients: Map<PersonId, GmailClient[]> = new Map<PersonId, GmailClient[]>()
private readonly initLimitter = new RateLimiter(config.InitLimit)
private readonly authProvider
protected static _instance: GmailController
private constructor (
private readonly ctx: MeasureContext,
private readonly mongo: Db,
private readonly storageAdapter: StorageAdapter
) {
this.credentials = JSON.parse(config.Credentials)
this.authProvider = new AuthProvider(this.credentials)
GmailController._instance = this
}
static create (ctx: MeasureContext, mongo: Db, storageAdapter: StorageAdapter): GmailController {
static create (ctx: MeasureContext, storageAdapter: StorageAdapter): GmailController {
if (GmailController._instance !== undefined) {
throw new Error('GmailController already exists')
}
return new GmailController(ctx, mongo, storageAdapter)
return new GmailController(ctx, storageAdapter)
}
static getGmailController (): GmailController {
@@ -58,45 +69,43 @@ export class GmailController {
}
async startAll (): Promise<void> {
const tokens = await this.mongo.collection<Token>('tokens').find().toArray()
const groups = new Map<string, Token[]>()
console.log('start gmail service', tokens.length)
for (const token of tokens) {
const group = groups.get(token.workspace)
if (group === undefined) {
groups.set(token.workspace, [token])
} else {
group.push(token)
groups.set(token.workspace, group)
}
}
try {
const token = serviceToken()
const integrations = await getIntegrations(token)
this.ctx.info('Start integrations', { count: integrations.length })
const limiter = new RateLimiter(config.InitLimit)
for (const [workspace, tokens] of groups) {
await limiter.add(async () => {
const wstok = generateToken(systemAccountUuid, workspace as any) // TODO: FIXME
const accountClient = getAccountClient(wstok)
const info = await accountClient.getWorkspaceInfo()
const limiter = new RateLimiter(config.InitLimit)
for (const integration of integrations) {
if (integration.workspaceUuid === null) continue
await limiter.add(async () => {
if (integration.workspaceUuid === null) return
const accountClient = getAccountClient(token)
const info = await accountClient.getWorkspaceInfo()
if (info === undefined) {
console.log('workspace not found', workspace)
return
}
if (!isActiveMode(info.mode)) {
console.log('workspace is not active', workspace)
return
}
const startPromise = this.startWorkspace(workspace as any, tokens) // TODO: FIXME
const timeoutPromise = new Promise<void>((resolve) => {
setTimeout(() => {
resolve()
}, 60000)
if (info === undefined) {
this.ctx.info('workspace not found', { workspaceUuid: integration.workspaceUuid })
return
}
if (!isActiveMode(info.mode)) {
this.ctx.info('workspace is not active', { workspaceUuid: integration.workspaceUuid })
return
}
const tokens = await getWorkspaceTokens(accountClient, integration.workspaceUuid)
this.ctx.info('Use stored tokens', { count: tokens.length })
const startPromise = this.startWorkspace(integration.workspaceUuid, tokens)
const timeoutPromise = new Promise<void>((resolve) => {
setTimeout(() => {
resolve()
}, 60000)
})
await Promise.race([startPromise, timeoutPromise])
})
await Promise.race([startPromise, timeoutPromise])
})
}
}
await limiter.waitProcessing()
await limiter.waitProcessing()
} catch (err: any) {
this.ctx.error('Failed to start existing integrations', err)
}
}
async startWorkspace (workspace: WorkspaceUuid, tokens: Token[]): Promise<void> {
@@ -105,15 +114,20 @@ export class GmailController {
for (const token of tokens) {
try {
const timeout = setTimeout(() => {
console.log('init client hang', token.workspace, token.userId)
this.ctx.info('init client hang', { workspaceUuid: token.workspace, userId: token.userId })
}, 60000)
const client = await workspaceClient.createGmailClient(token)
clearTimeout(timeout)
clients.push(client)
} catch (err) {
console.error(`Couldn't create client for ${workspace} ${token.userId}`)
} catch (err: any) {
this.ctx.error("Couldn't create client", {
workspaceUuid: workspace,
userId: token.userId,
message: err.message
})
}
}
this.ctx.info('Start clients sync', { workspaceUuid: workspace, count: clients.length })
for (const client of clients) {
void this.initLimitter.add(async () => {
await client.startSync()
@@ -133,25 +147,30 @@ export class GmailController {
}
}
addClient (email: string, client: GmailClient): void {
const clients = this.clients.get(email)
addClient (socialId: PersonId, client: GmailClient): void {
const clients = this.clients.get(socialId)
if (clients === undefined) {
this.clients.set(email, [client])
this.clients.set(socialId, [client])
} else {
clients.push(client)
this.clients.set(email, clients)
this.clients.set(socialId, clients)
}
}
async getGmailClient (email: string, workspace: WorkspaceUuid, token: string): Promise<GmailClient> {
/*
async getGmailClient (userId: AccountUuid, workspace: WorkspaceUuid, token: string): Promise<GmailClient> {
const workspaceClient = await this.getWorkspaceClient(workspace)
const userId = await workspaceClient.getUserId(email)
return await workspaceClient.createGmailClient({ userId, workspace, token })
}
*/
async signout (workspace: WorkspaceUuid, email: string): Promise<void> {
getAuthProvider (): AuthProvider {
return this.authProvider
}
async signout (workspace: WorkspaceUuid, account: AccountUuid): Promise<void> {
const workspaceClient = await this.getWorkspaceClient(workspace)
const clients = await workspaceClient.signout(email)
const clients = await workspaceClient.signoutByAccountId(account)
if (clients === 0) {
this.workspaces.delete(workspace)
}
@@ -164,9 +183,9 @@ export class GmailController {
this.workspaces.clear()
}
async createClient (user: User | Token): Promise<GmailClient> {
async createClient (user: User | Token, authCode?: string): Promise<GmailClient> {
const workspace = await this.getWorkspaceClient(user.workspace)
const newClient = await workspace.createGmailClient(user)
const newClient = await workspace.createGmailClient(user, authCode)
return newClient
}
@@ -174,12 +193,12 @@ export class GmailController {
let res = this.workspaces.get(workspace)
if (res === undefined) {
try {
console.log('create workspace worker for', workspace)
res = await WorkspaceClient.create(this.ctx, this.credentials, this.mongo, this.storageAdapter, workspace)
this.ctx.info('create workspace worker', { workspaceUuid: workspace })
res = await WorkspaceClient.create(this.ctx, this.credentials, this.storageAdapter, workspace)
this.workspaces.set(workspace, res)
console.log('created workspace worker for', workspace)
this.ctx.info('created workspace worker', { workspaceUuid: workspace })
} catch (err) {
console.error(`Couldn't create workspace worker for ${workspace}, reason: `, err)
this.ctx.error("Couldn't create workspace worker", { workspaceUuid: workspace, reason: err })
throw err
}
}
services/gmail/pod-gmail/src/integrations.ts
+132
View File
@@ -0,0 +1,132 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { AccountUuid, MeasureContext, PersonId, TxOperations, WorkspaceUuid } from '@hcengineering/core'
import { getAccountClient } from '@hcengineering/server-client'
import { Integration } from '@hcengineering/account-client'
import gmail from '@hcengineering/gmail'
import setting from '@hcengineering/setting'
import { serviceToken } from './utils'
import { GMAIL_INTEGRATION } from './types'
import { getAccountSocialIds } from './accounts'
export async function getIntegration (socialId: PersonId, workspaceUuid?: WorkspaceUuid): Promise<Integration | null> {
const client = getAccountClient(serviceToken())
return await client.getIntegration({ kind: GMAIL_INTEGRATION, socialId, workspaceUuid: workspaceUuid ?? null })
}
export async function getIntegrations (token: string): Promise<Integration[]> {
const client = getAccountClient(token)
return (await client.listIntegrations({ kind: GMAIL_INTEGRATION })) ?? []
}
export async function getIntegrationByAccount (
account: AccountUuid,
workspaceUuid?: WorkspaceUuid
): Promise<Integration | null> {
const token = serviceToken()
const client = getAccountClient(token)
const integrations = await client.listIntegrations({ kind: GMAIL_INTEGRATION, workspaceUuid: workspaceUuid ?? null })
if (integrations.length === 0) return null
const socialIds = await getAccountSocialIds(account)
for (const integration of integrations) {
if (integration.workspaceUuid == null) continue
const socialId = socialIds.find((it) => it._id === integration.socialId)
if (socialId !== undefined) {
return integration
}
}
return null
}
export async function removeIntegration (
socialId: PersonId | undefined | null,
workspaceUuid: WorkspaceUuid
): Promise<void> {
if (socialId == null) return
const accountClient = getAccountClient(serviceToken())
const integrations = await accountClient.listIntegrations({ kind: GMAIL_INTEGRATION, socialId, workspaceUuid })
for (const integration of integrations) {
await accountClient.deleteIntegration({
socialId,
kind: GMAIL_INTEGRATION,
workspaceUuid: integration.workspaceUuid
})
}
}
export async function deleteIntegration (integration: Integration): Promise<void> {
const accountClient = getAccountClient(serviceToken())
await accountClient.deleteIntegration(integration)
}
export async function createIntegrationIfNotEsixts (socialId: PersonId, workspace: WorkspaceUuid): Promise<Integration> {
const accountClient = getAccountClient(serviceToken())
const integration = {
socialId,
kind: GMAIL_INTEGRATION,
workspaceUuid: workspace
}
const existingIntegration = await accountClient.getIntegration(integration)
if (existingIntegration != null) return existingIntegration
await accountClient.createIntegration(integration)
return integration
}
export async function disableIntegration (integration: Integration): Promise<void> {
const client = getAccountClient(serviceToken())
await client.updateIntegration({
...integration,
data: {
disabled: true
}
})
}
export async function enableIntegration (integration: Integration): Promise<void> {
const client = getAccountClient(serviceToken())
await client.updateIntegration({
...integration,
data: {
disabled: false
}
})
}
export async function cleanIntegrations (
ctx: MeasureContext,
client: TxOperations,
userId: AccountUuid,
workspace: WorkspaceUuid
): Promise<void> {
const integration = await getIntegrationByAccount(userId, workspace)
if (integration != null) {
await deleteIntegration(integration)
ctx.info('Deleted integration', { userId, workspace })
}
const personIds = ((await getAccountSocialIds(userId)) ?? []).map((id) => id._id)
const integrationSetting = await client.findOne(setting.class.Integration, {
createdBy: { $in: personIds },
type: gmail.integrationType.Gmail
})
if (integrationSetting != null) {
await client.remove(integrationSetting)
ctx.info('Deleted integration setting', { userId, workspace })
}
}
services/gmail/pod-gmail/src/main.ts
+57 -41
View File
@@ -18,17 +18,17 @@
import { SplitLogger } from '@hcengineering/analytics-service'
import { MeasureMetricsContext, newMetrics } from '@hcengineering/core'
import { setMetadata } from '@hcengineering/platform'
import serverClient from '@hcengineering/server-client'
import serverClient, { getAccountClient } from '@hcengineering/server-client'
import { isWorkspaceLoginInfo } from '@hcengineering/account-client'
import { initStatisticsContext, type StorageConfiguration } from '@hcengineering/server-core'
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import serverToken from '@hcengineering/server-token'
import serverToken, { decodeToken } from '@hcengineering/server-token'
import { type IncomingHttpHeaders } from 'http'
import { join } from 'path'
import { decode64 } from './base64'
import config from './config'
import { GmailController } from './gmailController'
import { createServer, listen } from './server'
import { closeDB, getDB } from './storage'
import { type Endpoint, type State } from './types'
const extractToken = (header: IncomingHttpHeaders): any => {
@@ -61,8 +61,7 @@ export const main = async (): Promise<void> => {
const storageConfig: StorageConfiguration = storageConfigFromEnv()
const storageAdapter = buildStorageFromConfig(storageConfig)
const db = await getDB()
const gmailController = GmailController.create(ctx, db, storageAdapter)
const gmailController = GmailController.create(ctx, storageAdapter)
await gmailController.startAll()
const endpoints: Endpoint[] = [
{
@@ -70,22 +69,30 @@ export const main = async (): Promise<void> => {
type: 'get',
handler: async (req, res) => {
try {
// TODO: FIXME
throw new Error('Not implemented')
// const token = extractToken(req.headers)
ctx.info('Signin request received')
const token = extractToken(req.headers)
// if (token === undefined) {
// res.status(401).send()
// return
// }
// const redirectURL = req.query.redirectURL as string
if (token === undefined) {
res.status(401).send()
return
}
const redirectURL = req.query.redirectURL as string
// const { workspace } = decodeToken(token)
// const gmail = await gmailController.getGmailClient(email, workspace, token)
// const url = gmail.getAutUrl(redirectURL)
// res.send(url)
const accountClient = getAccountClient(token)
const wsLoginInfo = await accountClient.getLoginInfoByToken()
if (!isWorkspaceLoginInfo(wsLoginInfo)) {
res.status(400).send({ err: "Couldn't find workspace with the provided token" })
return
}
const authProvider = gmailController.getAuthProvider()
const url = authProvider.getAuthUrl(redirectURL, {
workspace: wsLoginInfo.workspace,
userId: wsLoginInfo.account
})
res.send(url)
} catch (err) {
console.log('signin error', (err as any).message)
ctx.error('signin error', { message: (err as any).message })
res.status(500).send()
}
}
@@ -94,31 +101,36 @@ export const main = async (): Promise<void> => {
endpoint: '/signin/code',
type: 'get',
handler: async (req, res) => {
const code = req.query.code as string
const state = JSON.parse(decode64(req.query.state as string)) as unknown as State
const gmail = await gmailController.createClient(state)
await gmail.authorize(code)
res.redirect(state.redirectURL)
try {
ctx.info('Signin code request received')
const code = req.query.code as string
const state = JSON.parse(decode64(req.query.state as string)) as unknown as State
await gmailController.createClient(state, code)
res.redirect(state.redirectURL)
} catch (err) {
ctx.error('Failed to process signin code', { message: (err as any).message })
res.status(500).send()
}
}
},
{
endpoint: '/signout',
type: 'get',
handler: async (req, res) => {
ctx.info('Signout request received')
try {
// TODO: FIXME
throw new Error('Not implemented')
// const token = extractToken(req.headers)
const token = extractToken(req.headers)
// if (token === undefined) {
// res.status(401).send()
// return
// }
if (token === undefined) {
res.status(401).send()
return
}
// const { email, workspace } = decodeToken(token)
// await gmailController.signout(workspace.name, email)
const { account, workspace } = decodeToken(token)
await gmailController.signout(workspace, account)
} catch (err) {
console.log('signout error', JSON.stringify(err))
ctx.error('signout error', { message: JSON.stringify(err) })
}
res.send()
@@ -128,14 +140,19 @@ export const main = async (): Promise<void> => {
endpoint: '/push',
type: 'post',
handler: async (req, res) => {
const data = req.body?.message?.data
if (data === undefined) {
res.status(400).send({ err: "'data' is missing" })
return
}
gmailController.push(data)
try {
const data = req.body?.message?.data
if (data === undefined) {
res.status(400).send({ err: "'data' is missing" })
return
}
gmailController.push(data)
res.send()
res.send()
} catch (err) {
ctx.error('Push request failed', { message: JSON.stringify(err) })
res.status(500).send()
}
}
}
]
@@ -145,7 +162,6 @@ export const main = async (): Promise<void> => {
const asyncClose = async (): Promise<void> => {
await gmailController.close()
await storageAdapter.close()
await closeDB()
}
const shutdown = (): void => {
services/gmail/pod-gmail/src/message/attachments.ts
+165
View File
@@ -0,0 +1,165 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import attachment, { Attachment } from '@hcengineering/attachment'
import { AttachedData, Blob, MeasureContext, Ref, TxOperations, WorkspaceUuid } from '@hcengineering/core'
import { StorageAdapter } from '@hcengineering/server-core'
import { gmail_v1 } from 'googleapis'
import { v4 as uuid } from 'uuid'
import { encode64 } from '../base64'
import type { AttachedFile } from '../types'
import { addFooter } from '../utils'
export class AttachmentHandler {
constructor (
private readonly ctx: MeasureContext,
private readonly workspaceId: WorkspaceUuid,
private readonly storageAdapter: StorageAdapter,
private readonly gmail: gmail_v1.Resource$Users,
private readonly client: TxOperations
) {}
async addAttachement (file: AttachedFile, message: any, currentAttachemtns: Attachment[]): Promise<void> {
try {
if (currentAttachemtns.findIndex((p) => p.name === file.name && p.lastModified === file.lastModified) !== -1) {
return
}
const id = uuid()
const data: AttachedData<Attachment> = {
name: file.name,
file: id as Ref<Blob>,
type: file.type ?? 'undefined',
size: file.size ?? Buffer.from(file.file, 'base64').length,
lastModified: file.lastModified
}
await this.storageAdapter.put(this.ctx, this.workspaceId as any, id, file.file, data.type, data.size) // TODO: FIXME
await this.client.addCollection(
attachment.class.Attachment,
message.space,
message._id,
message._class,
'attachments',
data
)
} catch (err: any) {
this.ctx.error('Add attachment error', { error: err.message })
}
}
async getPartFiles (part: gmail_v1.Schema$MessagePart | undefined, messageId: string): Promise<AttachedFile[]> {
if (part === undefined) return []
if (part.filename != null && part.filename.length > 0) {
if (part.body?.attachmentId != null) {
const attachment = await this.getAttachmentById(part.body?.attachmentId, messageId)
if (attachment.data == null) return []
return [
{
file: attachment.data,
name: part.filename,
type: part.mimeType ?? undefined,
size: attachment.size ?? undefined,
lastModified: new Date().getTime()
}
]
}
if (part.body?.data == null) return []
return [
{
file: part.body.data,
name: part.filename,
type: part.mimeType ?? undefined,
size: part.body.size ?? undefined,
lastModified: new Date().getTime()
}
]
}
return await this.getPartsFiles(part.parts, messageId)
}
private async getAttachmentById (attachmentId: string, messageId: string): Promise<gmail_v1.Schema$MessagePartBody> {
const res = await this.gmail.messages.attachments.get({ id: attachmentId, userId: 'me', messageId })
return res.data
}
private async getPartsFiles (
parts: gmail_v1.Schema$MessagePart[] | undefined,
messageId: string
): Promise<AttachedFile[]> {
const result: AttachedFile[] = []
if (parts !== undefined) {
const filtredParts = parts.filter((part) => part.filename != null && part.filename.length > 0)
for (const part of filtredParts ?? []) {
const res = await this.getPartFiles(part, messageId)
if (res.length > 0) {
result.push(...res)
}
}
}
return result
}
async makeAttachmentsBody (message: any, from: string): Promise<string> {
const str = [
'Content-Type: multipart/mixed; boundary="mail"\n',
'MIME-Version: 1.0\n',
`To: ${message.to} \n`,
`From: ${from} \n`
]
if (message.replyTo != null) {
str.push(`In-Reply-To: ${message.replyTo} \n`)
}
if (message.copy != null && message.copy.length > 0) {
str.push(`Cc: ${message.copy.join(', ')} \n`)
}
if (message.subject != null) {
str.push(`Subject: =?UTF-8?B?${encode64(message.subject)}?= \n`)
}
str.push('\n\n')
str.push('--mail\n')
str.push('Content-Type: text/html; charset="UTF-8"\n')
str.push('MIME-Version: 1.0\n')
str.push('Content-Transfer-Encoding: 7bit\n\n')
str.push(addFooter(message.content))
str.push('\n\n')
const attachments = await this.client.findAll(attachment.class.Attachment, { attachedTo: message._id })
for (const attachment of attachments) {
const attachmentStrings = await this.makeAttachmentPart(attachment)
str.push(...attachmentStrings)
}
str.push('--mail--')
return encode64(str.join('')).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '')
}
private async makeAttachmentPart (attachment: Attachment): Promise<string[]> {
const buffer = await this.storageAdapter.read(this.ctx, this.workspaceId as any, attachment.file) // TODO: FIXME
const data = Buffer.concat(buffer.map((b) => new Uint8Array(b))).toString('base64')
const res: string[] = []
res.push('--mail\n')
res.push(`Content-Type: ${attachment.type}\n`)
res.push('MIME-Version: 1.0\n')
res.push('Content-Transfer-Encoding: base64\n')
res.push(`Content-Disposition: attachment; filename="${attachment.name}"\n\n`)
res.push(data)
res.push('\n\n')
return res
}
}
services/gmail/pod-gmail/src/message/message.ts
+201
View File
@@ -0,0 +1,201 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import {
type MeasureContext,
PersonId,
Timestamp,
TxCUD,
TxFactory,
TxOperations,
TxProcessor,
TxCreateDoc,
TxUpdateDoc,
AttachedData,
Data
} from '@hcengineering/core'
import gmail, { type Message } from '@hcengineering/gmail'
import { type GaxiosResponse } from 'gaxios'
import { gmail_v1 } from 'googleapis'
import core from '@hcengineering/core'
import attachment, { Attachment } from '@hcengineering/attachment'
import sanitizeHtml from 'sanitize-html'
import { type Channel } from '../types'
import { AttachmentHandler } from './attachments'
import { decode64 } from '../base64'
import { diffAttributes } from '../utils'
const EMAIL_REGEX =
/(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))/
export class MessageManager {
constructor (
private readonly ctx: MeasureContext,
private readonly client: TxOperations,
private readonly attachmentHandler: AttachmentHandler,
private readonly socialId: PersonId,
private readonly workspace: { getChannel: (email: string) => Channel | undefined }
) {}
private findChannels (res: AttachedData<Message>): Channel[] {
const recievers = res.copy != null ? [res.to, ...res.copy] : [res.to]
const values = res.incoming ? [res.from] : recievers
const result: Channel[] = []
for (const value of values) {
const res = value.match(EMAIL_REGEX)
if (res !== null) {
const email = res[0]
const channel = this.workspace.getChannel(email)
if (channel !== undefined) result.push(channel)
}
}
return result
}
async saveMessage (message: GaxiosResponse<gmail_v1.Schema$Message>, me: string): Promise<void> {
const res = convertMessage(message, me)
const channels = this.findChannels(res)
if (channels.length === 0) return
const attachments = await this.attachmentHandler.getPartFiles(message.data.payload, message.data.id ?? '')
const factory = new TxFactory(this.socialId)
for (const channel of channels) {
const current = await this.client.findOne(gmail.class.Message, {
messageId: res.messageId,
attachedTo: channel._id
})
const tx = current != null ? this.updateTx(res, current, factory, channel) : this.createTx(res, factory, channel)
if (tx !== undefined) {
const resultMessage =
current != null
? TxProcessor.updateDoc2Doc(current, tx as TxUpdateDoc<Message>)
: TxProcessor.createDoc2Doc(tx as TxCreateDoc<Message>)
await this.client.tx(tx)
if (attachments.length > 0) {
const currentAttachemtns: Attachment[] =
current !== undefined
? await this.client.findAll(attachment.class.Attachment, { attachedTo: current._id })
: []
for (const attachment of attachments) {
await this.attachmentHandler.addAttachement(attachment, resultMessage, currentAttachemtns)
}
}
}
}
}
private createTx (
message: AttachedData<Message> & { modifiedOn: Timestamp },
factory: TxFactory,
channel: Channel
): TxCUD<Message> {
const tx = factory.createTxCollectionCUD<Channel, Message>(
channel._class,
channel._id,
channel.space,
'items',
factory.createTxCreateDoc<Message>(
gmail.class.Message,
core.space.Workspace,
message as unknown as Data<Message>,
undefined,
message.modifiedOn
),
message.modifiedOn
)
return tx
}
private updateTx (
message: AttachedData<Message> & { modifiedOn: Timestamp },
current: Message,
factory: TxFactory,
channel: Channel
): TxCUD<Message> | undefined {
const operations = diffAttributes(current, message)
if (Object.keys(operations).length === 0) return undefined
const tx = factory.createTxCollectionCUD<Channel, Message>(
channel._class,
channel._id,
channel.space,
'items',
factory.createTxUpdateDoc<Message>(current._class, current.space, current._id, operations)
)
return tx
}
}
function getHeaderValue (payload: gmail_v1.Schema$MessagePart | undefined, name: string): string | undefined {
if (payload === undefined) return undefined
const headers = payload.headers
return headers?.find((header) => header.name?.toLowerCase() === name.toLowerCase())?.value ?? undefined
}
function getPartsMessage (parts: gmail_v1.Schema$MessagePart[] | undefined, mime: string): string {
let result = ''
if (parts !== undefined) {
const htmlPart = parts.find((part) => part.mimeType === mime)
const filtredParts = htmlPart !== undefined ? parts.filter((part) => part.mimeType === mime) : parts
for (const part of filtredParts ?? []) {
result += getPartMessage(part, mime)
}
}
return result
}
const sanitizeOptions: sanitizeHtml.IOptions = {
allowedTags: [],
allowedAttributes: {}
}
export function sanitizeText (input: string): string {
if (input == null) return ''
return sanitizeHtml(input, sanitizeOptions)
}
function getPartMessage (part: gmail_v1.Schema$MessagePart | undefined, mime: string): string {
if (part === undefined) return ''
if (part.body?.data != null) {
return decode64(part.body.data)
}
return getPartsMessage(part.parts, mime)
}
function convertMessage (
message: GaxiosResponse<gmail_v1.Schema$Message>,
me: string
): AttachedData<Message> & { modifiedOn: Timestamp } {
const date = message.data.internalDate != null ? new Date(Number.parseInt(message.data.internalDate)) : new Date()
const from = getHeaderValue(message.data.payload, 'From') ?? ''
const to = getHeaderValue(message.data.payload, 'To') ?? ''
const copy =
getHeaderValue(message.data.payload, 'Cc')
?.split(',')
.map((p) => p.trim()) ?? undefined
const incoming = !from.includes(me)
return {
modifiedOn: date.getTime(),
messageId: getHeaderValue(message.data.payload, 'Message-ID') ?? '',
replyTo: getHeaderValue(message.data.payload, 'In-Reply-To'),
copy,
content: sanitizeHtml(getPartMessage(message.data.payload, 'text/html')),
textContent: sanitizeText(getPartMessage(message.data.payload, 'text/plain')),
from,
to,
incoming,
subject: getHeaderValue(message.data.payload, 'Subject') ?? '',
sendOn: date.getTime()
}
}
services/gmail/pod-gmail/src/message/sync.ts
+246
View File
@@ -0,0 +1,246 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type GaxiosResponse } from 'gaxios'
import { gmail_v1 } from 'googleapis'
import { type MeasureContext, PersonId } from '@hcengineering/core'
import { type KeyValueClient } from '@hcengineering/kvs-client'
import { RateLimiter } from '../rateLimiter'
import { MessageManager } from './message'
interface History {
historyId: string
userId: string
workspace: string
}
export class SyncMutex {
private readonly locks = new Map<string, Promise<void>>()
async lock (key: string): Promise<() => void> {
// Wait for any existing lock to be released
const currentLock = this.locks.get(key)
if (currentLock != null) {
await currentLock
}
// Create a new lock
let releaseFn!: () => void
const newLock = new Promise<void>((resolve) => {
releaseFn = resolve
})
// Store the lock
this.locks.set(key, newLock)
// Return the release function
return () => {
if (this.locks.get(key) === newLock) {
this.locks.delete(key)
}
releaseFn()
}
}
}
export class SyncManager {
private readonly syncMutex = new SyncMutex()
constructor (
private readonly ctx: MeasureContext,
private readonly messageManager: MessageManager,
private readonly gmail: gmail_v1.Resource$Users,
private readonly workspace: string,
private readonly keyValueClient: KeyValueClient,
private readonly rateLimiter: RateLimiter
) {}
private async getHistory (userId: PersonId): Promise<History | null> {
const historyKey = this.getHistoryKey(userId)
return await this.keyValueClient.getValue<History>(historyKey)
}
private async clearHistory (userId: PersonId): Promise<void> {
const historyKey = this.getHistoryKey(userId)
await this.keyValueClient.deleteKey(historyKey)
}
private async setHistoryId (userId: PersonId, historyId: string): Promise<void> {
const historyKey = this.getHistoryKey(userId)
const history: History = {
historyId,
userId,
workspace: this.workspace
}
await this.keyValueClient.setValue(historyKey, history)
}
private async partSync (userId: PersonId, userEmail: string | undefined, historyId: string): Promise<void> {
if (userEmail === undefined) {
throw new Error('Cannot sync without user email')
}
let pageToken: string | undefined
let histories: GaxiosResponse<gmail_v1.Schema$ListHistoryResponse>
while (true) {
try {
await this.rateLimiter.take(2)
histories = await this.gmail.history.list({
userId: 'me',
historyTypes: ['messageAdded'],
startHistoryId: historyId,
pageToken
})
} catch (err: any) {
this.ctx.error('Part sync get history error', { workspaceUuid: this.workspace, userId, error: err.message })
await this.clearHistory(userId)
void this.sync(userId)
return
}
const nextPageToken = histories.data.nextPageToken
const array = histories.data.history ?? []
this.ctx.info('Messages to migrate', { count: array.length })
try {
for (const history of array) {
for (const message of history.messagesAdded ?? []) {
if (message.message?.id == null || (message.message.labelIds?.includes('DRAFT') ?? false)) {
continue
}
try {
const res = await this.getMessage(message.message.id)
await this.messageManager.saveMessage(res, userEmail)
} catch (err) {
this.ctx.error('Part sync message error', {
workspaceUuid: this.workspace,
userId,
messageId: message.message.id,
err
})
}
}
if (history.id != null) {
await this.setHistoryId(userId, history.id)
}
}
if (nextPageToken == null) {
return
} else {
pageToken = nextPageToken
}
} catch (err) {
this.ctx.error('Part sync error', { workspaceUuid: this.workspace, userId, historyId, err })
return
}
}
}
async fullSync (userId: PersonId, userEmail?: string, q?: string): Promise<void> {
this.ctx.info('Start full sync', { workspaceUuid: this.workspace, userId, userEmail })
if (userEmail === undefined) {
throw new Error('Cannot sync without user email')
}
const pageToken: string | undefined = undefined
const query: gmail_v1.Params$Resource$Users$Messages$List = {
userId: 'me',
pageToken
}
if (q !== undefined) {
query.q = q
}
let currentHistoryId: string | undefined
try {
const messagesIds: string[] = []
while (true) {
await this.rateLimiter.take(5)
const messages = await this.gmail.messages.list(query)
if (query.pageToken == null) {
this.ctx.info('Total messages', {
workspace: this.workspace,
userId,
resultSizeEstimate: messages.data.resultSizeEstimate
})
}
const ids = messages.data.messages?.map((p) => p.id) ?? []
for (let index = 0; index < ids.length; index++) {
const id = ids[index]
if (id == null) continue
messagesIds.push(id)
}
if (messages.data.nextPageToken == null) {
this.ctx.info('Break', { totalNewMessages: messagesIds.length })
break
}
query.pageToken = messages.data.nextPageToken
}
for (let index = messagesIds.length - 1; index >= 0; index--) {
const id = messagesIds[index]
try {
const message = await this.getMessage(id)
const historyId = message.data.historyId
await this.messageManager.saveMessage(message, userEmail)
if (historyId != null && q === undefined) {
if (currentHistoryId == null || Number(currentHistoryId) < Number(historyId)) {
await this.setHistoryId(userId, historyId)
currentHistoryId = historyId
}
}
if (index % 500 === 0) {
this.ctx.info('Remaining messages to sync', { workspace: this.workspace, userId, count: index })
}
} catch (err: any) {
this.ctx.error('Full sync message error', { workspace: this.workspace, userId, err })
}
}
this.ctx.info('Full sync finished', { workspaceUuid: this.workspace, userId, userEmail })
} catch (err) {
this.ctx.error('Full sync error', { workspace: this.workspace, userId, err })
}
}
private async getMessage (id: string): Promise<GaxiosResponse<gmail_v1.Schema$Message>> {
await this.rateLimiter.take(5)
return await this.gmail.messages.get({
id,
userId: 'me',
format: 'FULL'
})
}
async sync (userId: PersonId, userEmail?: string): Promise<void> {
const mutexKey = `${this.workspace}:${userId}`
const releaseLock = await this.syncMutex.lock(mutexKey)
try {
this.ctx.info('Sync history', { workspaceUuid: this.workspace, userId, userEmail })
const history = await this.getHistory(userId)
if (history?.historyId != null && history?.historyId !== '') {
this.ctx.info('Start part sync', { workspaceUuid: this.workspace, userId, historyId: history.historyId })
await this.partSync(userId, userEmail, history.historyId)
} else {
this.ctx.info('Start full sync', { workspaceUuid: this.workspace, userId })
await this.fullSync(userId, userEmail)
}
} catch (err) {
this.ctx.error('Sync error', { workspace: this.workspace, userId, err })
} finally {
releaseLock()
}
}
private getHistoryKey (userId: PersonId): string {
return `history:${this.workspace}:${userId}`
}
}
services/gmail/pod-gmail/src/storage.ts
-36
View File
@@ -1,36 +0,0 @@
//
// Copyright © 2020, 2021 Anticrm Platform Contributors.
// Copyright © 2021 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { MongoClientReference, getMongoClient } from '@hcengineering/mongo'
import { MongoClient } from 'mongodb'
import config from './config'
const clientRef: MongoClientReference = getMongoClient(config.MongoURI)
let client: MongoClient | undefined
export const getDB = (() => {
return async () => {
if (client === undefined) {
client = await clientRef.getClient()
}
return client.db(config.MongoDB)
}
})()
export const closeDB: () => Promise<void> = async () => {
clientRef.close()
}
services/gmail/pod-gmail/src/tokens.ts
+86
View File
@@ -0,0 +1,86 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { MeasureContext, PersonId, WorkspaceUuid } from '@hcengineering/core'
import type { AccountClient } from '@hcengineering/account-client'
import { getAccountClient } from '@hcengineering/server-client'
import { GMAIL_INTEGRATION, SecretType, Token } from './types'
export class TokenStorage {
private readonly accountClient: AccountClient
constructor (
private readonly ctx: MeasureContext,
private readonly workspace: WorkspaceUuid,
platformToken: string
) {
this.accountClient = getAccountClient(platformToken)
}
async getToken (socialId: PersonId): Promise<Token | null> {
const secret = await this.accountClient.getIntegrationSecret({
key: SecretType.TOKEN,
kind: GMAIL_INTEGRATION,
socialId,
workspaceUuid: this.workspace
})
return secret?.secret !== undefined ? JSON.parse(secret.secret) : null
}
async saveToken (socialId: PersonId, token: Token): Promise<void> {
const existingToken = await this.getToken(socialId)
if (existingToken !== null) {
const updatedToken = {
...existingToken,
...token
}
await this.accountClient.updateIntegrationSecret({
key: SecretType.TOKEN,
kind: GMAIL_INTEGRATION,
socialId,
secret: JSON.stringify(updatedToken),
workspaceUuid: this.workspace
})
this.ctx.info('Updated integration secret', { socialId, workspaceUuid: this.workspace })
} else {
await this.accountClient.addIntegrationSecret({
key: SecretType.TOKEN,
kind: GMAIL_INTEGRATION,
socialId,
secret: JSON.stringify(token),
workspaceUuid: this.workspace
})
this.ctx.info('Created integration secret', { socialId, workspaceUuid: this.workspace })
}
}
async deleteToken (socialId: PersonId): Promise<void> {
await this.accountClient.deleteIntegrationSecret({
key: SecretType.TOKEN,
kind: GMAIL_INTEGRATION,
socialId,
workspaceUuid: this.workspace
})
}
}
export async function getWorkspaceTokens (accountClient: AccountClient, workspace: WorkspaceUuid): Promise<Token[]> {
const secrets = await accountClient.listIntegrationsSecrets({
kind: GMAIL_INTEGRATION,
workspaceUuid: workspace
})
return secrets.map((secret: { secret: string }) => JSON.parse(secret.secret))
}
services/gmail/pod-gmail/src/types.ts
+9 -2
View File
@@ -14,7 +14,7 @@
// limitations under the License.
//
import type { PersonId, Doc, WorkspaceUuid } from '@hcengineering/core'
import type { AccountUuid, Doc, SocialId, WorkspaceUuid } from '@hcengineering/core'
import type { NextFunction, Request, Response } from 'express'
import type { Credentials } from 'google-auth-library'
import type { Channel as PlatformChannel } from '@hcengineering/contact'
@@ -26,7 +26,9 @@ export type History = User & {
}
export interface User {
userId: PersonId
userId: AccountUuid
email?: string
socialId: SocialId
workspace: WorkspaceUuid
token: string
}
@@ -68,3 +70,8 @@ export interface ProjectCredentialsData {
client_secret: string
redirect_uris: string[]
}
export const GMAIL_INTEGRATION = 'gmail'
export enum SecretType {
TOKEN = 'token'
}
services/gmail/pod-gmail/src/utils.ts
+28 -1
View File
@@ -14,9 +14,12 @@
// limitations under the License.
//
import { type Data, type Doc, type DocumentUpdate } from '@hcengineering/core'
import { type Data, type Doc, type DocumentUpdate, systemAccountUuid } from '@hcengineering/core'
import { generateToken } from '@hcengineering/server-token'
import { deepEqual } from 'fast-equals'
import { type KeyValueClient, getClient as getKeyValueClient } from '@hcengineering/kvs-client'
import { type Token, type User } from './types'
import config from './config'
export class DeferredPromise<T = any> {
public readonly promise: Promise<T>
@@ -63,3 +66,27 @@ function toUndef (value: any): any {
export function isToken (user: User | Token): user is Token {
return (user as Token).access_token !== undefined
}
export function addFooter (message: string): string {
if (config.FooterMessage === undefined || config.FooterMessage.trim() === '') return message
return message + config.FooterMessage.trim()
}
export function serviceToken (): string {
return generateToken(systemAccountUuid, undefined, { service: 'gmail' })
}
export async function wait (sec: number): Promise<void> {
await new Promise<void>((resolve) => {
setTimeout(() => {
resolve()
}, sec * 1000)
})
}
let keyValueClient: KeyValueClient | undefined
export function getKvsClient (token: string): KeyValueClient {
if (keyValueClient !== undefined) return keyValueClient
keyValueClient = getKeyValueClient('gmail', config.KvsUrl, token)
return keyValueClient
}
services/gmail/pod-gmail/src/workspaceClient.ts
+69 -65
View File
@@ -14,10 +14,9 @@
// limitations under the License.
//
import contact, { type Employee, type Channel as PlatformChannel } from '@hcengineering/contact'
import contact, { type Channel as PlatformChannel, type Person, Employee } from '@hcengineering/contact'
import core, {
type WorkspaceUuid,
type PersonId,
type Client,
type Doc,
MeasureContext,
@@ -27,15 +26,19 @@ import core, {
type TxCreateDoc,
TxProcessor,
type TxRemoveDoc,
type TxUpdateDoc
type TxUpdateDoc,
PersonId,
AccountUuid,
TxOperations
} from '@hcengineering/core'
import gmailP, { type NewMessage } from '@hcengineering/gmail'
import type { StorageAdapter } from '@hcengineering/server-core'
import { generateToken } from '@hcengineering/server-token'
import { type Db } from 'mongodb'
import { getClient } from './client'
import { GmailClient } from './gmail'
import { type Channel, type ProjectCredentials, type User } from './types'
import { getAccountSocialIds } from './accounts'
import { cleanIntegrations } from './integrations'
export class WorkspaceClient {
private messageSubscribed: boolean = false
@@ -49,7 +52,6 @@ export class WorkspaceClient {
private constructor (
private readonly ctx: MeasureContext,
private readonly credentials: ProjectCredentials,
private readonly mongo: Db,
private readonly storageAdapter: StorageAdapter,
private readonly workspace: WorkspaceUuid
) {}
@@ -57,29 +59,30 @@ export class WorkspaceClient {
static async create (
ctx: MeasureContext,
credentials: ProjectCredentials,
mongo: Db,
storageAdapter: StorageAdapter,
workspace: WorkspaceUuid
): Promise<WorkspaceClient> {
const instance = new WorkspaceClient(ctx, credentials, mongo, storageAdapter, workspace)
const instance = new WorkspaceClient(ctx, credentials, storageAdapter, workspace)
await instance.initClient(workspace)
return instance
}
async createGmailClient (user: User): Promise<GmailClient> {
const current = this.getGmailClient(user.userId)
async createGmailClient (user: User, authCode?: string): Promise<GmailClient> {
const current = user.socialId?._id !== undefined ? this.getGmailClient(user.socialId?._id) : undefined
if (current !== undefined) return current
this.ctx.info('Creating new gmail client', { workspaceUuid: this.workspace, userId: user.userId })
this.ctx.info('Creating new gmail user', { user })
const newClient = await GmailClient.create(
this.ctx,
this.credentials,
user,
this.mongo,
this.client,
this,
this.workspace,
this.storageAdapter
this.storageAdapter,
authCode
)
this.clients.set(user.userId, newClient)
this.clients.set(user.socialId._id, newClient)
return newClient
}
@@ -91,32 +94,34 @@ export class WorkspaceClient {
await this.client?.close()
}
async getUserId (email: string): Promise<PersonId> {
// TODO: FIXME
throw new Error('Not implemented')
// const user = this.client.getModel().getAccountByEmail(email)
// if (user === undefined) {
// throw new Error('User not found')
// }
// return user._id
}
async signout (email: string, byError: boolean = false): Promise<number> {
const userId = await this.getUserId(email)
const client = this.clients.get(userId)
if (client !== undefined) {
await client.signout(byError)
async signoutByAccountId (userId: AccountUuid, byError: boolean = false): Promise<number> {
const socialIds = await getAccountSocialIds(userId)
this.ctx.info('socialIds', { socialIds })
this.ctx.info('clients', { clients: this.clients })
let deleted = false
for (const socialId of socialIds) {
const client = this.clients.get(socialId._id)
if (client !== undefined) {
await client.signout(byError)
this.clients.delete(socialId._id)
deleted = true
}
}
this.clients.delete(userId)
if (!deleted && socialIds.length > 0) {
this.ctx.info('Clean up integrations without clients')
const tx = new TxOperations(this.client, socialIds[0]._id)
await cleanIntegrations(this.ctx, tx, userId, this.workspace)
}
return this.clients.size
}
async signoutByUserId (userId: PersonId, byError: boolean = false): Promise<number> {
const client = this.clients.get(userId)
async signoutBySocialId (socialId: PersonId, byError: boolean = false): Promise<number> {
const client = this.clients.get(socialId)
if (client !== undefined) {
await client.signout(byError)
this.clients.delete(socialId)
}
this.clients.delete(userId)
return this.clients.size
}
@@ -126,7 +131,7 @@ export class WorkspaceClient {
private async initClient (workspace: WorkspaceUuid): Promise<Client> {
const token = generateToken(systemAccountUuid, workspace, { service: 'gmail' })
console.log('token', token, workspace)
this.ctx.info('Init client', { workspaceUuid: workspace })
const client = await getClient(token)
client.notify = (...tx: Tx[]) => {
void this.txHandler(...tx)
@@ -161,7 +166,7 @@ export class WorkspaceClient {
const newMessages = await this.client.findAll(gmailP.class.NewMessage, {
status: 'new'
})
console.log('get new messages, recieved', this.workspace, newMessages.length)
this.ctx.info('get new messages', { workspaceUuid: this.workspace, count: newMessages.length })
await this.subscribeMessages()
for (const message of newMessages) {
const from = message.from ?? message.createdBy ?? message.modifiedBy
@@ -169,7 +174,11 @@ export class WorkspaceClient {
if (client !== undefined) {
await client.createMessage(message)
} else {
console.log('client not found, skip message', this.workspace, from, message._id)
this.ctx.error('client not found, skip message', {
workspaceUuid: this.workspace,
from,
messageId: message._id
})
}
}
}
@@ -326,41 +335,36 @@ export class WorkspaceClient {
// #region Users
async checkUsers (): Promise<void> {
// TODO: FIXME
throw new Error('Not implemented')
// const removedEmployees = await this.client.findAll(contact.mixin.Employee, {
// active: false
// })
// const accounts = await this.client.findAll(contact.class.PersonAccount, {
// person: { $in: removedEmployees.map((p) => p._id) }
// })
// for (const acc of accounts) {
// await this.deactivateUser(acc)
// }
// this.txHandlers.push(async (...txes: Tx[]) => {
// for (const tx of txes) {
// await this.txEmployeeHandler(tx)
// }
// })
// console.log('deactivate users', this.workspace, accounts.length)
const removedEmployees = await this.client.findAll(contact.mixin.Employee, {
active: false
})
for (const person of removedEmployees) {
await this.deactivateUser(person)
}
this.txHandlers.push(async (...txes: Tx[]) => {
for (const tx of txes) {
await this.txEmployeeHandler(tx)
}
})
this.ctx.info('deactivate users', { workspaceUuid: this.workspace, count: removedEmployees.length })
}
// private async deactivateUser (acc: PersonAccount): Promise<void> {
// await this.signout(acc.email, true)
// }
private async deactivateUser (acc: Person): Promise<void> {
if (acc.personUuid === undefined) return
await this.signoutByAccountId(acc.personUuid as AccountUuid, true)
}
private async txEmployeeHandler (tx: Tx): Promise<void> {
// TODO: FIXME
throw new Error('Not implemented')
// if (tx._class !== core.class.TxUpdateDoc) return
// const ctx = tx as TxUpdateDoc<Employee>
// if (!this.client.getHierarchy().isDerived(ctx.objectClass, contact.mixin.Employee)) return
// if (ctx.operations.active === false) {
// const acc = await this.client.findOne(contact.class.PersonAccount, { person: ctx.objectId })
// if (acc !== undefined) {
// await this.deactivateUser(acc)
// }
// }
if (tx._class !== core.class.TxUpdateDoc) return
const ctx = tx as TxUpdateDoc<Employee>
if (!this.client.getHierarchy().isDerived(ctx.objectClass, contact.mixin.Employee)) return
if (ctx.operations.active === false) {
const acc = await this.client.findOne(contact.class.Person, { _id: ctx.objectId })
if (acc !== undefined) {
await this.deactivateUser(acc)
}
}
}
// #endregion