diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 53417a16ed..ef4441dde9 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -385,6 +385,9 @@ importers: '@rush-temp/kanban': specifier: file:./projects/kanban.tgz version: file:projects/kanban.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(postcss-load-config@4.0.2(postcss@8.5.3)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(postcss@8.5.3)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + '@rush-temp/kvs-client': + specifier: file:./projects/kvs-client.tgz + version: file:projects/kvs-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/lead': specifier: file:./projects/lead.tgz version: file:projects/lead.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) @@ -771,7 +774,7 @@ importers: version: file:projects/pod-github.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)(utf-8-validate@6.0.4)(y-prosemirror@1.2.15(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2)(y-protocols@1.0.6(yjs@13.6.23))(yjs@13.6.23)) '@rush-temp/pod-gmail': specifier: file:./projects/pod-gmail.tgz - version: file:projects/pod-gmail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + version: file:projects/pod-gmail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3) '@rush-temp/pod-inbound-mail': specifier: file:./projects/pod-inbound-mail.tgz version: file:projects/pod-inbound-mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9)) @@ -1870,6 +1873,9 @@ importers: jest-environment-jsdom: specifier: ^29.7.0 version: 29.7.0(bufferutil@4.0.8)(utf-8-validate@6.0.4) + jest-fetch-mock: + specifier: ^3.0.3 + version: 3.0.3(encoding@0.1.13) jimp: specifier: ^0.16.1 version: 0.16.13 @@ -1942,6 +1948,9 @@ importers: mongodb: specifier: ^6.12.0 version: 6.12.0(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3) + mongodb-memory-server: + specifier: ^10.1.4 + version: 10.1.4(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3) morgan: specifier: ^1.10.0 version: 1.10.0 @@ -4313,6 +4322,10 @@ packages: resolution: {integrity: sha512-a1aAx7zBxeZjaejI86Ho69hHcEm7F4dkdD9rVCv2BVHzjG/1C8mq7pv+QWHFGoxkiQwwm9MatVV4IMCSjDjAZw==, tarball: file:projects/kanban.tgz} version: 0.0.0 + '@rush-temp/kvs-client@file:projects/kvs-client.tgz': + resolution: {integrity: sha512-UHZpGqCKncG8BGVHJPeceusj86Knh9SjhNvp972sLvA/FY89ph6Rpz+34X6aI3D5Cu3048ePmwszH81PKphNrQ==, tarball: file:projects/kvs-client.tgz} + version: 0.0.0 + '@rush-temp/lead-assets@file:projects/lead-assets.tgz': resolution: {integrity: sha512-VVUGNaGtZaGVHj7V/o6b0kmXeLwYxQA3LwuWvW0vbv2xdMsfZO/QC6ShBZPUSrzYPjfOWTjvS6lhH2alTanW7A==, tarball: file:projects/lead-assets.tgz} version: 0.0.0 @@ -4826,7 +4839,7 @@ packages: version: 0.0.0 '@rush-temp/pod-gmail@file:projects/pod-gmail.tgz': - resolution: {integrity: sha512-HyyHadfmgT1OP7pdln0WJvK6cGPTYVVvxq21JBHKshu40WojCB6dLdAch/YrTO0aSS7UzluuLmYnnCn5JGvfcg==, tarball: file:projects/pod-gmail.tgz} + resolution: {integrity: sha512-mWpD7TVaoiHBcGFgqt2KoPioV7Uc/9Y7DXu3LKiwZBrUX6X+WR/77RcKq/gNGPbwNxCKOc+dDsscF5NKMhIqSg==, tarball: file:projects/pod-gmail.tgz} version: 0.0.0 '@rush-temp/pod-inbound-mail@file:projects/pod-inbound-mail.tgz': @@ -5482,7 +5495,7 @@ packages: version: 0.0.0 '@rush-temp/tool@file:projects/tool.tgz': - resolution: {integrity: sha512-zqh0++u5ETaUNlwkipP+8CeICqrFYEDBO4sAjFRyLPd1SHZbanUpQRAg58t4C+Z7q4Owqr149ee/hCGWPult3w==, tarball: file:projects/tool.tgz} + resolution: {integrity: sha512-k0KXvAT1ulcbCo/O0bYbPxjZHX2I7BAGMr7Ntdrt1xLcUBsQ3WSOUw9AxCLU4+PiJmIHQVpFEhJPNc06CeFfCA==, tarball: file:projects/tool.tgz} version: 0.0.0 '@rush-temp/tracker-assets@file:projects/tracker-assets.tgz': @@ -5510,7 +5523,7 @@ packages: version: 0.0.0 '@rush-temp/ui-next@file:projects/ui-next.tgz': - resolution: {integrity: sha512-7yi52BTXL9htk9CapPOLg7a26v5GVVYSKPWqEbWrEQgin1JvjL7F7qv3EcIGVtdcCnfkvATeDXT8HLvZNg9DwA==, tarball: file:projects/ui-next.tgz} + resolution: {integrity: sha512-bisoYF6n3HWSicnaOnc1DhInOtgSsIW77QjtWRtuq3lmJMzKQ8u+RF2wqa6gUaw3Qa/qcbE+HyzkJGQyCVB0og==, tarball: file:projects/ui-next.tgz} version: 0.0.0 '@rush-temp/ui@file:projects/ui.tgz': @@ -7137,6 +7150,9 @@ packages: async-mutex@0.3.2: resolution: {integrity: sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==} + async-mutex@0.5.0: + resolution: {integrity: sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==} + async@3.2.5: resolution: {integrity: sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==} @@ -7644,6 +7660,9 @@ packages: resolution: {integrity: sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==} engines: {node: ^12.20.0 || >=14} + commondir@1.0.1: + resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + compare-version@0.1.2: resolution: {integrity: sha512-pJDh5/4wrEnXX/VWRZvruAGHkzKdr46z11OlTPN+VrATlWWhSKewNCJ1futCO5C7eJB3nPMFZA1LeYtcFboZ2A==} engines: {node: '>=0.10.0'} @@ -9064,6 +9083,10 @@ packages: resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==} engines: {node: '>= 0.8'} + find-cache-dir@3.3.2: + resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==} + engines: {node: '>=8'} + find-up@3.0.0: resolution: {integrity: sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==} engines: {node: '>=6'} @@ -9112,6 +9135,15 @@ packages: debug: optional: true + follow-redirects@1.15.9: + resolution: {integrity: sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + for-each@0.3.3: resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==} @@ -10049,6 +10081,9 @@ packages: resolution: {integrity: sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + jest-fetch-mock@3.0.3: + resolution: {integrity: sha512-Ux1nWprtLrdrH4XwE7O7InRY6psIi3GOsqNESJgMJ+M5cv4A8Lh7SN9d2V2kKRZ8ebAfcd1LNyZguAOb6JiDqw==} + jest-get-type@29.6.3: resolution: {integrity: sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -10651,6 +10686,10 @@ packages: resolution: {integrity: sha512-8vBuFF/I/+OSLRmdf2wwFCJCz+nSn0m6DPvGH1fS/KiQoSaR+sETbov0eIk9KhEKy8CYqIkIAnbohxT/4H0kuA==} engines: {node: '>=12'} + make-dir@3.1.0: + resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} + engines: {node: '>=8'} + make-dir@4.0.0: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} @@ -10903,6 +10942,14 @@ packages: mongodb-connection-string-url@3.0.0: resolution: {integrity: sha512-t1Vf+m1I5hC2M5RJx/7AtxgABy1cZmIPQRMXw+gEIPn/cZNF3Oiy+l0UIypUwVB5trcWHq3crg2g3uAR9aAwsQ==} + mongodb-memory-server-core@10.1.4: + resolution: {integrity: sha512-o8fgY7ZalEd8pGps43fFPr/hkQu1L8i6HFEGbsTfA2zDOW0TopgpswaBCqDr0qD7ptibyPfB5DmC+UlIxbThzA==} + engines: {node: '>=16.20.1'} + + mongodb-memory-server@10.1.4: + resolution: {integrity: sha512-+oKQ/kc3CX+816oPFRtaF0CN4vNcGKNjpOQe4bHo/21A3pMD+lC7Xz1EX5HP7siCX4iCpVchDMmCOFXVQSGkUg==} + engines: {node: '>=16.20.1'} + mongodb@6.12.0: resolution: {integrity: sha512-RM7AHlvYfS7jv7+BXund/kR64DryVI+cHbVAy9P61fnb1RcWZqOW1/Wj2YhqMCx+MuYhqTRGv7AwHBzmsCKBfA==} engines: {node: '>=16.20.1'} @@ -10997,6 +11044,10 @@ packages: resolution: {integrity: sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==} engines: {node: '>= 0.4.0'} + new-find-package-json@2.0.0: + resolution: {integrity: sha512-lDcBsjBSMlj3LXH2v/FW3txlh2pYTjmbOXPYJD93HI5EwuLzI11tdHSIpUMmfq/IOsldj4Ps8M8flhm+pCK4Ew==} + engines: {node: '>=12.22.0'} + next-tick@1.1.0: resolution: {integrity: sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==} @@ -11801,6 +11852,9 @@ packages: promise-inflight@1.0.1: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} + promise-polyfill@8.3.0: + resolution: {integrity: sha512-H5oELycFml5yto/atYqmjyigJoAo3+OXwolYiH7OfQuYlAqhxNvTfiNMbV9hsC6Yp83yE5r2KTVmtrG6R9i6Pg==} + promise-retry@2.0.1: resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} engines: {node: '>=10'} @@ -12214,6 +12268,9 @@ packages: sanitize-html@2.15.0: resolution: {integrity: sha512-wIjst57vJGpLyBP8ioUbg6ThwJie5SuSIjHxJg53v5Fg+kUK+AXlb7bK3RNXpp315MvwM+0OBGCV6h5pPHsVhA==} + sanitize-html@2.16.0: + resolution: {integrity: sha512-0s4caLuHHaZFVxFTG74oW91+j6vW7gKbGD6CD2+miP73CE6z6YtOBN0ArtLd2UGyi4IC7K47v3ENUbQX4jV3Mg==} + sass-loader@13.3.3: resolution: {integrity: sha512-mt5YN2F1MOZr3d/wBRcZxeFgwgkH44wVc2zohO2YF6JiOMkiXe4BYRZpSu2sO1g71mo/j16txzUhsKZlqjVGzA==} engines: {node: '>= 14.15.0'} @@ -13713,6 +13770,10 @@ packages: yauzl@2.10.0: resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + yauzl@3.2.0: + resolution: {integrity: sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==} + engines: {node: '>=12'} + yjs@13.6.23: resolution: {integrity: sha512-ExtnT5WIOVpkL56bhLeisG/N5c4fmzKn4k0ROVfJa5TY2QHbH7F0Wu2T5ZhR7ErsFWQEFafyrnSI8TPKVF9Few==} engines: {node: '>=16.0.0', npm: '>=8.0.0'} @@ -19372,6 +19433,34 @@ snapshots: - supports-color - ts-node + '@rush-temp/kvs-client@file:projects/kvs-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': + dependencies: + '@types/jest': 29.5.12 + '@types/node': 20.11.19 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + cross-env: 7.0.3 + esbuild: 0.24.2 + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + jest-fetch-mock: 3.0.3(encoding@0.1.13) + prettier: 3.2.5 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) + typescript: 5.7.3 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - babel-jest + - babel-plugin-macros + - encoding + - node-notifier + - supports-color + - ts-node + '@rush-temp/lead-assets@file:projects/lead-assets.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: '@types/jest': 29.5.12 @@ -22053,17 +22142,19 @@ snapshots: - utf-8-validate - y-prosemirror - '@rush-temp/pod-gmail@file:projects/pod-gmail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': + '@rush-temp/pod-gmail@file:projects/pod-gmail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)': dependencies: '@tsconfig/node16': 1.0.4 '@types/cors': 2.8.17 '@types/express': 4.17.21 '@types/jest': 29.5.12 '@types/node': 20.11.19 + '@types/sanitize-html': 2.15.0 '@types/uuid': 8.3.4 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) cors: 2.8.5 + cross-env: 7.0.3 dotenv: 16.0.3 esbuild: 0.24.2 eslint: 8.56.0 @@ -22082,7 +22173,9 @@ snapshots: jwt-simple: 0.5.6 mongodb: 6.12.0(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3) prettier: 3.2.5 + sanitize-html: 2.16.0 ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.3.3) + ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) ts-node-dev: 2.0.0(@types/node@20.11.19)(typescript@5.3.3) typescript: 5.3.3 uuid: 8.3.2 @@ -22103,7 +22196,6 @@ snapshots: - snappy - socks - supports-color - - ts-node '@rush-temp/pod-inbound-mail@file:projects/pod-inbound-mail.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))': dependencies: @@ -26977,6 +27069,7 @@ snapshots: libphonenumber-js: 1.10.56 mime-types: 2.1.35 mongodb: 6.12.0(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3) + mongodb-memory-server: 10.1.4(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3) msgpackr: 1.11.2 msgpackr-extract: 3.0.3 postgres: 3.4.5 @@ -29730,6 +29823,10 @@ snapshots: dependencies: tslib: 2.7.0 + async-mutex@0.5.0: + dependencies: + tslib: 2.8.1 + async@3.2.5: {} asynckit@0.4.0: {} @@ -30326,6 +30423,8 @@ snapshots: commander@9.5.0: {} + commondir@1.0.1: {} + compare-version@0.1.2: {} component-emitter@1.3.1: {} @@ -32105,6 +32204,12 @@ snapshots: statuses: 2.0.1 unpipe: 1.0.0 + find-cache-dir@3.3.2: + dependencies: + commondir: 1.0.1 + make-dir: 3.1.0 + pkg-dir: 4.2.0 + find-up@3.0.0: dependencies: locate-path: 3.0.0 @@ -32140,6 +32245,10 @@ snapshots: follow-redirects@1.15.6: {} + follow-redirects@1.15.9(debug@4.4.0): + optionalDependencies: + debug: 4.4.0 + for-each@0.3.3: dependencies: is-callable: 1.2.7 @@ -33252,6 +33361,13 @@ snapshots: jest-mock: 29.7.0 jest-util: 29.7.0 + jest-fetch-mock@3.0.3(encoding@0.1.13): + dependencies: + cross-fetch: 3.1.8(encoding@0.1.13) + promise-polyfill: 8.3.0 + transitivePeerDependencies: + - encoding + jest-get-type@29.6.3: {} jest-haste-map@29.7.0: @@ -34052,6 +34168,10 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.4.15 + make-dir@3.1.0: + dependencies: + semver: 6.3.1 + make-dir@4.0.0: dependencies: semver: 7.6.3 @@ -34339,6 +34459,44 @@ snapshots: '@types/whatwg-url': 11.0.4 whatwg-url: 13.0.0 + mongodb-memory-server-core@10.1.4(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3): + dependencies: + async-mutex: 0.5.0 + camelcase: 6.3.0 + debug: 4.4.0 + find-cache-dir: 3.3.2 + follow-redirects: 1.15.9(debug@4.4.0) + https-proxy-agent: 7.0.5 + mongodb: 6.12.0(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3) + new-find-package-json: 2.0.0 + semver: 7.6.3 + tar-stream: 3.1.7 + tslib: 2.8.1 + yauzl: 3.2.0 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - gcp-metadata + - kerberos + - mongodb-client-encryption + - snappy + - socks + - supports-color + + mongodb-memory-server@10.1.4(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3): + dependencies: + mongodb-memory-server-core: 10.1.4(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3) + tslib: 2.8.1 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - gcp-metadata + - kerberos + - mongodb-client-encryption + - snappy + - socks + - supports-color + mongodb@6.12.0(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3): dependencies: '@mongodb-js/saslprep': 1.1.9 @@ -34416,6 +34574,12 @@ snapshots: netmask@2.0.2: {} + new-find-package-json@2.0.0: + dependencies: + debug: 4.4.0 + transitivePeerDependencies: + - supports-color + next-tick@1.1.0: {} nice-try@1.0.5: {} @@ -35201,6 +35365,8 @@ snapshots: promise-inflight@1.0.1: {} + promise-polyfill@8.3.0: {} + promise-retry@2.0.1: dependencies: err-code: 2.0.3 @@ -35714,6 +35880,15 @@ snapshots: parse-srcset: 1.0.2 postcss: 8.5.3 + sanitize-html@2.16.0: + dependencies: + deepmerge: 4.3.1 + escape-string-regexp: 4.0.0 + htmlparser2: 8.0.2 + is-plain-object: 5.0.0 + parse-srcset: 1.0.2 + postcss: 8.5.3 + sass-loader@13.3.3(sass@1.71.1)(webpack@5.97.1): dependencies: neo-async: 2.6.2 @@ -37616,6 +37791,11 @@ snapshots: buffer-crc32: 0.2.13 fd-slicer: 1.1.0 + yauzl@3.2.0: + dependencies: + buffer-crc32: 0.2.13 + pend: 1.2.0 + yjs@13.6.23: dependencies: lib0: 0.2.99 diff --git a/dev/tool/package.json b/dev/tool/package.json index 20c9dc357f..5e54f2a454 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -55,7 +55,8 @@ "@types/request": "~2.48.8", "jest": "^29.7.0", "ts-jest": "^29.1.1", - "@types/jest": "^29.5.5" + "@types/jest": "^29.5.5", + "mongodb-memory-server": "^10.1.4" }, "dependencies": { "@elastic/elasticsearch": "^7.17.14", @@ -160,6 +161,7 @@ "@hcengineering/collaboration": "^0.6.0", "@hcengineering/datalake": "^0.6.0", "@hcengineering/s3": "^0.6.0", + "@hcengineering/kvs-client": "^0.6.0", "commander": "^8.1.0", "csv-parse": "~5.1.0", "email-addresses": "^5.0.0", diff --git a/dev/tool/src/__start.ts b/dev/tool/src/__start.ts index 031a099d3c..b1d6e027b4 100644 --- a/dev/tool/src/__start.ts +++ b/dev/tool/src/__start.ts @@ -103,4 +103,13 @@ export function getAccountDBUrl (): string { return url } +export function getKvsUrl (): string { + const url = process.env.KVS_URL + if (url === undefined) { + console.error('please provide KVS_URL') + process.exit(1) + } + return url +} + devTool(prepareTools) diff --git a/dev/tool/src/__tests__/gmail.test.ts b/dev/tool/src/__tests__/gmail.test.ts new file mode 100644 index 0000000000..79425ac035 --- /dev/null +++ b/dev/tool/src/__tests__/gmail.test.ts @@ -0,0 +1,311 @@ +import { getAccountClient } from '@hcengineering/server-client' +import { getClient as getKvsClient } from '@hcengineering/kvs-client' +import { generateToken } from '@hcengineering/server-token' +import { MongoMemoryServer } from 'mongodb-memory-server' +import { MongoClient, type Db } from 'mongodb' +import { performGmailAccountMigrations } from '../gmail' + +// Mock dependencies +jest.mock('@hcengineering/server-client') +jest.mock('@hcengineering/server-token') +jest.mock('@hcengineering/kvs-client') + +describe('Gmail Migrations', () => { + // Setup MongoDB in-memory server + let mongoServer: MongoMemoryServer + let mongoClient: MongoClient + let db: Db + + // Mock implementations + const mockAccountClient = { + listWorkspaces: jest.fn(), + findFullSocialIdBySocialKey: jest.fn(), + getIntegration: jest.fn(), + createIntegration: jest.fn(), + getIntegrationSecret: jest.fn(), + addIntegrationSecret: jest.fn(), + updateIntegrationSecret: jest.fn() + } + + const mockKvsClient = { + getValue: jest.fn(), + setValue: jest.fn(), + deleteKey: jest.fn(), + listKeys: jest.fn() + } + + beforeAll(async () => { + mongoServer = await MongoMemoryServer.create() + const uri = mongoServer.getUri() + mongoClient = await MongoClient.connect(uri) + db = mongoClient.db('test-db') + + // Setup mocks + ;(getAccountClient as jest.Mock).mockReturnValue(mockAccountClient) + ;(getKvsClient as jest.Mock).mockReturnValue(mockKvsClient) + ;(generateToken as jest.Mock).mockReturnValue('mock-token') + }) + + afterAll(async () => { + await mongoClient.close() + await mongoServer.stop() + }) + + beforeEach(async () => { + // Reset mocks + jest.clearAllMocks() + + // Setup collections + await db.collection('tokens').deleteMany({}) + await db.collection('histories').deleteMany({}) + + // Reset mock implementations + mockAccountClient.listWorkspaces.mockReset() + mockAccountClient.findFullSocialIdBySocialKey.mockReset() + mockAccountClient.getIntegration.mockReset() + mockAccountClient.createIntegration.mockReset() + mockAccountClient.getIntegrationSecret.mockReset() + mockAccountClient.addIntegrationSecret.mockReset() + mockAccountClient.updateIntegrationSecret.mockReset() + + mockKvsClient.getValue.mockReset() + mockKvsClient.setValue.mockReset() + }, 10000) + + it('should migrate tokens to new integration format', async () => { + // Setup test data + const workspace1 = { uuid: 'ws1', name: 'Workspace 1' } + const workspace2 = { uuid: 'ws2', dataId: 'oldWs2', name: 'Workspace 2' } + + // Mock workspace list + mockAccountClient.listWorkspaces.mockResolvedValue([workspace1, workspace2]) + + // Setup tokens in DB + await db.collection('tokens').insertMany([ + { + userId: 'user1@example.com', + workspace: 'ws1', + token: 'token1', + refresh_token: 'refresh1', + access_token: 'access1' + }, + { + userId: 'user2@example.com', + workspace: 'oldWs2', + token: 'token2', + refresh_token: 'refresh2', + access_token: 'access2' + } + ]) + + // Mock social ID lookup + mockAccountClient.findFullSocialIdBySocialKey.mockImplementation((key) => { + if (key === 'email:user1@example.com') { + return Promise.resolve({ _id: 'social1', personUuid: 'person1' }) + } else if (key === 'email:user2@example.com') { + return Promise.resolve({ _id: 'social2', personUuid: 'person2' }) + } + return Promise.resolve(undefined) + }) + + // Mock integration checks + mockAccountClient.getIntegration.mockResolvedValue(null) + mockAccountClient.getIntegrationSecret.mockResolvedValue(null) + + // Run migration + await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url') + + // Verify tokens were migrated + expect(mockAccountClient.createIntegration).toHaveBeenCalledTimes(2) + expect(mockAccountClient.addIntegrationSecret).toHaveBeenCalledTimes(2) + + // Check that tokens were migrated with correct data + const calls = mockAccountClient.addIntegrationSecret.mock.calls + expect(calls).toContainEqual([ + expect.objectContaining({ + socialId: 'social1', + workspaceUuid: 'ws1', + secret: expect.stringContaining('user1@example.com') + }) + ]) + expect(calls).toContainEqual([ + expect.objectContaining({ + socialId: 'social2', + workspaceUuid: 'ws2', + secret: expect.stringContaining('user2@example.com') + }) + ]) + }) + + it('should migrate with oids and github ids', async () => { + // Setup test data + const workspace1 = { uuid: 'ws1', name: 'Workspace 1' } + const workspace2 = { uuid: 'ws2', dataId: 'oldWs2', name: 'Workspace 2' } + + // Mock workspace list + mockAccountClient.listWorkspaces.mockResolvedValue([workspace1, workspace2]) + + // Setup tokens in DB + await db.collection('tokens').insertMany([ + { + userId: 'github:user1', + workspace: 'ws1', + token: 'token1', + refresh_token: 'refresh1', + access_token: 'access1' + }, + { + userId: 'openid:user2', + workspace: 'oldWs2', + token: 'token2', + refresh_token: 'refresh2', + access_token: 'access2' + } + ]) + + // Mock social ID lookup + mockAccountClient.findFullSocialIdBySocialKey.mockImplementation((key) => { + if (key === 'github:user1') { + return Promise.resolve({ _id: 'social1', personUuid: 'person1' }) + } else if (key === 'oidc:user2') { + return Promise.resolve({ _id: 'social2', personUuid: 'person2' }) + } + return Promise.resolve(undefined) + }) + + // Mock integration checks + mockAccountClient.getIntegration.mockResolvedValue(null) + mockAccountClient.getIntegrationSecret.mockResolvedValue(null) + + // Run migration + await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url') + + // Verify tokens were migrated + expect(mockAccountClient.createIntegration).toHaveBeenCalledTimes(2) + expect(mockAccountClient.addIntegrationSecret).toHaveBeenCalledTimes(2) + + // Check that tokens were migrated with correct data + const calls = mockAccountClient.addIntegrationSecret.mock.calls + expect(calls).toContainEqual([ + expect.objectContaining({ + socialId: 'social1', + workspaceUuid: 'ws1', + secret: expect.stringContaining('github:user1') + }) + ]) + expect(calls).toContainEqual([ + expect.objectContaining({ + socialId: 'social2', + workspaceUuid: 'ws2', + secret: expect.stringContaining('openid:user2') + }) + ]) + }) + + it('should migrate history records to KVS', async () => { + // Setup test data + const workspace1 = { uuid: 'ws1', name: 'Workspace 1' } + + // Mock workspace list + mockAccountClient.listWorkspaces.mockResolvedValue([workspace1]) + + // Setup histories in DB + await db.collection('histories').insertMany([ + { + userId: 'user1@example.com', + workspace: 'ws1', + token: 'token1', + historyId: 'history1' + } + ]) + + // Mock social ID lookup + mockAccountClient.findFullSocialIdBySocialKey.mockImplementation((key) => { + if (key === 'email:user1@example.com') { + return Promise.resolve({ _id: 'social1', personUuid: 'person1' }) + } + return Promise.resolve(undefined) + }) + + // Mock KVS client responses + mockKvsClient.getValue.mockResolvedValue(null) + + // Run migration + await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url') + + // Verify KVS calls + expect(mockKvsClient.setValue).toHaveBeenCalledWith( + 'history:ws1:social1', + expect.objectContaining({ + historyId: 'history1', + email: 'user1@example.com', + userId: 'person1', + workspace: 'ws1' + }) + ) + }) + + it('should handle errors gracefully', async () => { + // Mock console.error to capture errors + const originalConsoleError = console.error + console.error = jest.fn() + + // Setup test data that will cause errors + mockAccountClient.listWorkspaces.mockResolvedValue([]) + mockAccountClient.findFullSocialIdBySocialKey.mockRejectedValue(new Error('Network error')) + + // Insert some data + await db.collection('tokens').insertOne({ + userId: 'error@example.com', + workspace: 'non-existent', + token: 'token-error' + }) + + // Run migration + await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url') + + // Should not throw but log errors + expect(console.error).toHaveBeenCalled() + + // Restore console.error + console.error = originalConsoleError + }) + + it('should update existing integration secrets', async () => { + // Setup test data + const workspace = { uuid: 'ws1', name: 'Workspace 1' } + + // Mock workspace list + mockAccountClient.listWorkspaces.mockResolvedValue([workspace]) + + // Setup token in DB + await db.collection('tokens').insertOne({ + userId: 'user1@example.com', + workspace: 'ws1', + token: 'token1', + refresh_token: 'refresh1' + }) + + // Mock social ID lookup + mockAccountClient.findFullSocialIdBySocialKey.mockResolvedValue({ _id: 'social1', personUuid: 'person1' }) + + // Mock existing integration and token + mockAccountClient.getIntegration.mockResolvedValue({ _id: 'integration1' }) + mockAccountClient.getIntegrationSecret.mockResolvedValue({ + scope: 'old-scope', + token_type: 'Bearer' + }) + + // Run migration + await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url') + + // Verify update was called instead of add + expect(mockAccountClient.createIntegration).not.toHaveBeenCalled() + expect(mockAccountClient.addIntegrationSecret).not.toHaveBeenCalled() + expect(mockAccountClient.updateIntegrationSecret).toHaveBeenCalledWith( + expect.objectContaining({ + secret: expect.stringContaining('refresh1') + }) + ) + }) +}) diff --git a/dev/tool/src/gmail.ts b/dev/tool/src/gmail.ts new file mode 100644 index 0000000000..b544017920 --- /dev/null +++ b/dev/tool/src/gmail.ts @@ -0,0 +1,225 @@ +import { + type AccountUuid, + type PersonId, + type SocialId, + type WorkspaceInfoWithStatus, + type WorkspaceUuid, + buildSocialIdString, + systemAccountUuid +} from '@hcengineering/core' +import { getAccountClient } from '@hcengineering/server-client' +import { getClient as getKvsClient } from '@hcengineering/kvs-client' +import { generateToken } from '@hcengineering/server-token' +import { getSocialKeyByOldEmail } from '@hcengineering/model-core' +import type { Db } from 'mongodb' + +// Old token and history types +interface Credentials { + refresh_token?: string | null + expiry_date?: number | null + access_token?: string | null + token_type?: string | null + id_token?: string | null + scope?: string +} + +interface User { + userId: string + workspace: WorkspaceUuid + token: string +} + +type History = User & { + historyId: string +} + +// Updated token and history types +interface UserV2 { + userId: AccountUuid + email?: string + socialId: SocialId + workspace: WorkspaceUuid + token: string +} + +export type TokenV2 = UserV2 & Credentials + +export type HistoryV2 = UserV2 & { + historyId: string +} + +type Token = User & Credentials + +interface WorkspaceInfoProvider { + getWorkspaceInfo: (workspaceUuid: WorkspaceUuid) => Promise +} + +const GMAIL_INTEGRATION = 'gmail' +const TOKEN_TYPE = 'token' + +export async function performGmailAccountMigrations (db: Db, region: string | null, kvsUrl: string): Promise { + console.log('Start Gmail migrations') + const token = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'admin', admin: 'true' }) + const accountClient = getAccountClient(token) + + const allWorkpaces = await accountClient.listWorkspaces(region) + const byId = new Map(allWorkpaces.map((it) => [it.uuid, it])) + const oldNewIds = new Map(allWorkpaces.map((it) => [it.dataId ?? it.uuid, it])) + const workspaceProvider: WorkspaceInfoProvider = { + getWorkspaceInfo: async (workspaceUuid: WorkspaceUuid) => { + const ws = oldNewIds.get(workspaceUuid as any) ?? byId.get(workspaceUuid as any) + if (ws == null) { + console.error('No workspace found for token', workspaceUuid) + return undefined + } + return ws + } + } + + await migrateGmailIntegrations(db, token, workspaceProvider) + + await migrateGmailHistory(db, token, kvsUrl, workspaceProvider) + console.log('Finished Gmail migrations') +} + +async function migrateGmailIntegrations ( + db: Db, + token: string, + workspaceProvider: WorkspaceInfoProvider +): Promise { + try { + console.log('Start Gmail account migrations') + const gmailToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'gmail' }) + const accountClient = getAccountClient(token) + + const gmailAccountClient = getAccountClient(gmailToken) + + const tokens = db.collection('tokens') + + const allTokens = await tokens.find({}).toArray() + + for (const token of allTokens) { + try { + const ws = await workspaceProvider.getWorkspaceInfo(token.workspace as any) + if (ws == null) { + continue + } + token.workspace = ws.uuid + + const socialKey = buildSocialIdString(getSocialKeyByOldEmail(token.userId)) + const socialId = + socialKey !== undefined ? await accountClient.findFullSocialIdBySocialKey(socialKey) : undefined + if (socialId == null) { + console.error('No socialId found for token', token) + continue + } + // Check/create integration in account + const existing = await gmailAccountClient.getIntegration({ + kind: GMAIL_INTEGRATION, + workspaceUuid: ws?.uuid, + socialId: socialId._id + }) + + if (existing == null) { + await gmailAccountClient.createIntegration({ + kind: GMAIL_INTEGRATION, + workspaceUuid: ws?.uuid, + socialId: socialId._id + }) + } + + const existingToken = await gmailAccountClient.getIntegrationSecret({ + key: TOKEN_TYPE, + kind: GMAIL_INTEGRATION, + socialId: socialId._id, + workspaceUuid: ws?.uuid + }) + const newToken: TokenV2 = { + ...token, + workspace: ws?.uuid, + email: token.userId, + userId: socialId.personUuid as AccountUuid, + socialId + } + if (existingToken == null) { + await gmailAccountClient.addIntegrationSecret({ + key: TOKEN_TYPE, + kind: GMAIL_INTEGRATION, + socialId: socialId._id, + secret: JSON.stringify(newToken), + workspaceUuid: token.workspace + }) + } else { + const updatedToken = { + ...existingToken, + ...newToken + } + await gmailAccountClient.updateIntegrationSecret({ + key: TOKEN_TYPE, + kind: GMAIL_INTEGRATION, + socialId: socialId._id, + secret: JSON.stringify(updatedToken), + workspaceUuid: token.workspace + }) + } + } catch (e) { + console.error('Error migrating token', token, e) + } + } + console.log('Gmail integrations migrations done, integration count:', allTokens.length) + } catch (e) { + console.error('Error migrating tokens', e) + } +} + +async function migrateGmailHistory ( + db: Db, + token: string, + kvsUrl: string, + workspaceProvider: WorkspaceInfoProvider +): Promise { + try { + console.log('Start Gmail history migrations') + const accountClient = getAccountClient(token) + const history = db.collection('histories') + const allHistories = await history.find({}).toArray() + + const kvsClient = getKvsClient(kvsUrl, token) + + for (const history of allHistories) { + try { + const socialKey = buildSocialIdString(getSocialKeyByOldEmail(history.userId)) + const socialId = + socialKey !== undefined ? await accountClient.findFullSocialIdBySocialKey(socialKey) : undefined + if (socialId == null) { + console.error('No socialId found for history', history) + continue + } + // Update/create history in KVS + const ws = await workspaceProvider.getWorkspaceInfo(history.workspace as any) + if (ws == null) { + continue + } + const historyKey = getHistoryKey(ws.uuid, socialId._id) + const existingHistory = await kvsClient.getValue(historyKey) + const updatedHistory: HistoryV2 = { + ...(existingHistory ?? history), + email: history.userId, + workspace: ws.uuid, + userId: socialId.personUuid as AccountUuid, + socialId + } + await kvsClient.setValue(historyKey, updatedHistory) + } catch (e) { + console.error('Error migrating history', history, e) + } + } + console.log('Finished migrating gmail history, count:', allHistories.length) + } catch (e) { + console.error('Error migrating gmail history', e) + } +} + +function getHistoryKey (workspace: WorkspaceUuid, userId: PersonId): string { + return `history:${workspace}:${userId}` +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 20a1577a80..93cffba3d6 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -89,13 +89,14 @@ import { type QueueWorkspaceMessage, type StorageAdapter } from '@hcengineering/server-core' -import { getAccountDBUrl, getMongoDBUrl } from './__start' +import { getAccountDBUrl, getKvsUrl, getMongoDBUrl } from './__start' // import { fillGithubUsers, fixAccountEmails, renameAccount } from './account' import { changeConfiguration } from './configuration' import { performGithubAccountMigrations } from './github' import { migrateCreatedModifiedBy, ensureGlobalPersonsForLocalAccounts, moveAccountDbFromMongoToPG } from './db' import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils' +import { performGmailAccountMigrations } from './gmail' const colorConstants = { colorRed: '\u001b[31m', @@ -2429,6 +2430,21 @@ export function devTool ( await queue.createTopics(parseInt(cmd.tx ?? '1')) }) + program + .command('migrate-gmail-account') + .option('--db ', 'DB name', 'gmail-service') + .option('--region ', 'DB region') + .action(async (cmd: { db: string, region?: string }) => { + const mongodbUri = getMongoDBUrl() + const client = getMongoClient(mongodbUri) + const _client = await client.getClient() + + const kvsUrl = getKvsUrl() + await performGmailAccountMigrations(_client.db(cmd.db), cmd.region ?? null, kvsUrl) + await _client.close() + client.close() + }) + extendProgram?.(program) program.parse(process.argv) diff --git a/packages/kvs-client/.eslintrc.js b/packages/kvs-client/.eslintrc.js new file mode 100644 index 0000000000..72235dc283 --- /dev/null +++ b/packages/kvs-client/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/packages/kvs-client/.npmignore b/packages/kvs-client/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/packages/kvs-client/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/packages/kvs-client/config/rig.json b/packages/kvs-client/config/rig.json new file mode 100644 index 0000000000..0110930f55 --- /dev/null +++ b/packages/kvs-client/config/rig.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig" +} diff --git a/packages/kvs-client/jest.config.js b/packages/kvs-client/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/packages/kvs-client/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/packages/kvs-client/jest.setup.js b/packages/kvs-client/jest.setup.js new file mode 100644 index 0000000000..78d57e3d23 --- /dev/null +++ b/packages/kvs-client/jest.setup.js @@ -0,0 +1,2 @@ +// Set up fetch mock +require('jest-fetch-mock').enableMocks() diff --git a/packages/kvs-client/package.json b/packages/kvs-client/package.json new file mode 100644 index 0000000000..dea934495a --- /dev/null +++ b/packages/kvs-client/package.json @@ -0,0 +1,58 @@ +{ + "name": "@hcengineering/kvs-client", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "files": [ + "lib/**/*", + "types/**/*", + "tsconfig.json" + ], + "author": "Hardcore Engineering Inc.", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "format": "format src", + "test": "jest --passWithNoTests --silent", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "cross-env": "~7.0.3", + "@hcengineering/platform-rig": "^0.6.0", + "@types/node": "~20.11.16", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "esbuild": "^0.24.2", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "typescript": "^5.3.3", + "jest": "^29.7.0", + "jest-fetch-mock": "^3.0.3", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5" + }, + "dependencies": { + "@hcengineering/core": "^0.6.32", + "@hcengineering/platform": "^0.6.11" + }, + "repository": "https://github.com/hcengineering/platform", + "publishConfig": { + "registry": "https://npm.pkg.github.com" + }, + "exports": { + ".": { + "types": "./types/index.d.ts", + "require": "./lib/index.js", + "import": "./lib/index.js" + } + } +} diff --git a/packages/kvs-client/src/__tests__/client.test.ts b/packages/kvs-client/src/__tests__/client.test.ts new file mode 100644 index 0000000000..f7820d6493 --- /dev/null +++ b/packages/kvs-client/src/__tests__/client.test.ts @@ -0,0 +1,201 @@ +import { getClient } from '../client' +import { KeyValueClient } from '../types' +import fetchMock from 'jest-fetch-mock' + +// Enable fetch mocks +fetchMock.enableMocks() + +describe('KeyValueClient', () => { + let client: KeyValueClient + const namespace = 'test-ns' + + beforeEach(() => { + // Reset fetch mocks for each test + fetchMock.resetMocks() + + // Default successful response for all fetch calls + fetchMock.mockResponse(JSON.stringify({ key: 'value' }), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }) + + client = getClient(namespace, 'https://api.example.com', 'test-token') + }) + + afterEach(() => { + fetchMock.resetMocks() + }) + + it('should throw error when baseUrl is empty', () => { + expect(() => getClient('test-ns', '')).toThrow('Key-value API URL not specified') + }) + + it('should throw error when namespace is empty', () => { + expect(() => getClient('', 'https://api.example.com')).toThrow('Namespace not specified') + }) + + describe('setValue', () => { + it('should send POST request with correct headers and body', async () => { + // Mock 204 response for setValue + fetchMock.mockResponseOnce('', { status: 204 }) + + const key = 'test-key' + const value = { data: 'test-value' } + + await client.setValue(key, value) + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.example.com/api/test-ns/test-key', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ + 'Content-Type': 'application/json', + Authorization: 'Bearer test-token', + Connection: 'keep-alive' + }), + body: JSON.stringify(value) + }) + ) + }) + + it('should throw error when response status is not 204', async () => { + fetchMock.mockResponseOnce(JSON.stringify({ error: 'Server error' }), { status: 500 }) + + await expect(client.setValue('key', 'value')).rejects.toThrow('Failed to store value') + }) + }) + + describe('getValue', () => { + it('should send GET request with correct headers', async () => { + const testData = { message: 'test data' } + fetchMock.mockResponseOnce(JSON.stringify(testData), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }) + + const result = await client.getValue('key') + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.example.com/api/test-ns/key', + expect.objectContaining({ + method: 'GET', + headers: expect.objectContaining({ + Authorization: 'Bearer test-token', + Connection: 'keep-alive' + }) + }) + ) + expect(result).toEqual(testData) + }) + + it('should return null when key does not exist (404)', async () => { + fetchMock.mockResponseOnce('', { status: 404 }) + + const result = await client.getValue('key') + + expect(result).toBeNull() + }) + + it('should throw error for non-404 error status', async () => { + fetchMock.mockResponseOnce(JSON.stringify({ error: 'Server error' }), { status: 500 }) + + await expect(client.getValue('key')).rejects.toThrow('Request failed') + }) + }) + + describe('deleteKey', () => { + it('should send DELETE request with correct headers', async () => { + fetchMock.mockResponseOnce('', { status: 204 }) + + await client.deleteKey('key') + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.example.com/api/test-ns/key', + expect.objectContaining({ + method: 'DELETE', + headers: expect.objectContaining({ + Authorization: 'Bearer test-token', + Connection: 'keep-alive' + }) + }) + ) + }) + + it('should not throw error when key does not exist (404)', async () => { + fetchMock.mockResponseOnce('', { status: 404 }) + + await expect(client.deleteKey('key')).resolves.toBeUndefined() + }) + + it('should throw error for status other than 204 or 404', async () => { + fetchMock.mockResponseOnce(JSON.stringify({ error: 'Server error' }), { status: 500 }) + + await expect(client.deleteKey('key')).rejects.toThrow('Failed to delete key') + }) + }) + + describe('listKeys', () => { + it('should send GET request to namespace endpoint', async () => { + const testData = { key1: 'value1', key2: 'value2' } + fetchMock.mockResponseOnce(JSON.stringify(testData), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }) + + const result = await client.listKeys() + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.example.com/api/test-ns', + expect.objectContaining({ method: 'GET' }) + ) + expect(result).toEqual(testData) + }) + + it('should include prefix query parameter when provided', async () => { + fetchMock.mockResponseOnce(JSON.stringify({}), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }) + + await client.listKeys('prefix') + + expect(fetchMock).toHaveBeenCalledWith('https://api.example.com/api/test-ns?prefix=prefix', expect.anything()) + }) + + it('should encode namespace and prefix properly', async () => { + // Create client with a namespace that needs encoding + const specialClient = getClient('name/space', 'https://api.example.com', 'test-token') + + fetchMock.mockResponseOnce(JSON.stringify({}), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }) + + await specialClient.listKeys('pre/fix') + + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.example.com/api/name%2Fspace?prefix=pre%2Ffix', + expect.anything() + ) + }) + }) + + describe('retry logic', () => { + it('should retry on network errors', async () => { + // First call throws network error, second succeeds + fetchMock.mockRejectOnce(new Error('Network error')).mockResponseOnce(JSON.stringify({ key: 'value' }), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }) + + const result = await client.getValue('key') + + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(result).toEqual({ key: 'value' }) + }) + }) +}) diff --git a/packages/kvs-client/src/client.ts b/packages/kvs-client/src/client.ts new file mode 100644 index 0000000000..07f8a352f1 --- /dev/null +++ b/packages/kvs-client/src/client.ts @@ -0,0 +1,200 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { concatLink } from '@hcengineering/core' +import { PlatformError } from '@hcengineering/platform' +import { KeyValueClient } from './types' + +/** + * Get a KeyValueClient instance + * @param namespace - Namespace for the key-value operations + * @param baseUrl - URL of the key-value API server + * @param token - Optional authorization token + * @param retryTimeoutMs - Optional timeout for retrying failed requests + * @returns KeyValueClient instance + * @public + */ +export function getClient (namespace: string, baseUrl: string, token?: string, retryTimeoutMs?: number): KeyValueClient { + if (baseUrl === undefined) { + throw new Error('Key-value API URL not specified') + } + + return new KeyValueClientImpl(namespace, baseUrl, token, retryTimeoutMs) +} + +class KeyValueClientImpl implements KeyValueClient { + private readonly requestInit: RequestInit + + constructor ( + private readonly namespace: string, + private readonly baseUrl: string, + private readonly token?: string, + private readonly retryTimeoutMs: number = 5000 + ) { + if (baseUrl === '') { + throw new Error('Key-value API URL not specified') + } + + if (namespace === '' || namespace === undefined) { + throw new Error('Namespace not specified') + } + + const isBrowser = typeof window !== 'undefined' + + this.requestInit = { + keepalive: true, + headers: { + ...(this.token === undefined + ? {} + : { + Authorization: 'Bearer ' + this.token + }) + }, + ...(isBrowser ? { credentials: 'include' } : {}) + } + } + + async setValue(key: string, value: T): Promise { + const url = this.buildUrl(key) + await this.sendRequest(url, { + method: 'POST', + body: JSON.stringify(value), + contentType: 'application/json', + errorMessage: 'Failed to store value' + }) + } + + async getValue(key: string): Promise { + const url = this.buildUrl(key) + return await this.sendRequest(url, { + method: 'GET', + returnNullOn404: true + }) + } + + async deleteKey (key: string): Promise { + const url = this.buildUrl(key) + await this.sendRequest(url, { + method: 'DELETE', + acceptNotFound: true, + errorMessage: 'Failed to delete key' + }) + } + + async listKeys(prefix?: string): Promise | null> { + let url = this.buildUrl() + if (prefix !== undefined) { + url += `?prefix=${encodeURIComponent(prefix)}` + } + return await this.sendRequest>(url, { + method: 'GET', + errorMessage: 'Failed to list keys' + }) + } + + private buildUrl (key?: string): string { + const baseApiUrl = concatLink(this.baseUrl, `/api/${encodeURIComponent(this.namespace)}`) + return key !== undefined ? concatLink(baseApiUrl, encodeURIComponent(key)) : baseApiUrl + } + + private async sendRequest( + url: string, + options: { + method: string + body?: string + contentType?: string + returnNullOn404?: boolean + acceptNotFound?: boolean + errorMessage?: string + } + ): Promise { + const headers = { + ...this.requestInit.headers, + ...(options.contentType !== undefined && options.contentType !== '' + ? { 'Content-Type': options.contentType } + : {}) + } + + const response = await this.fetchWithRetry(url, { + ...this.requestInit, + method: options.method, + headers, + body: options.body + }) + + if (options.returnNullOn404 === true && response.status === 404) { + return null + } + + if (options.acceptNotFound === true && response.status === 404) { + return null + } + + if (!response.ok) { + try { + const errorBody = await response.json() + if (errorBody?.error != null) { + throw new PlatformError(errorBody?.error) + } + } catch (e) { + // Ignore JSON parsing errors + } + throw new Error(options.errorMessage ?? 'Request failed') + } + + // Parse JSON response when needed + const contentType = response.headers.get('content-type') + if ( + response.status !== 204 && + (options.method === 'GET' || (contentType != null && contentType.includes('application/json'))) + ) { + return await response.json() + } + + return null + } + + private async fetchWithRetry (url: string, init: RequestInit): Promise { + const timeout = Date.now() + this.retryTimeoutMs + const connectionErrorCodes = ['ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND'] + let intervalMs = 25 + + while (true) { + try { + const headers = { + ...init.headers, + Connection: 'keep-alive' + } + + return await fetch(url, { ...init, headers }) + } catch (err: any) { + // Check if this is a network error we should retry + const errMessage: string = err?.message ?? '' + const isNetworkError = + connectionErrorCodes.includes(err?.cause?.code) || + errMessage.includes('Network error') || + errMessage.includes('network') + + if (!isNetworkError || timeout < Date.now()) { + throw err + } + + await new Promise((resolve) => setTimeout(resolve, intervalMs)) + if (intervalMs < 1000) { + intervalMs += 100 + } + } + } + } +} diff --git a/packages/kvs-client/src/index.ts b/packages/kvs-client/src/index.ts new file mode 100644 index 0000000000..0f078ded71 --- /dev/null +++ b/packages/kvs-client/src/index.ts @@ -0,0 +1,17 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export * from './client' +export * from './types' diff --git a/packages/kvs-client/src/types.ts b/packages/kvs-client/src/types.ts new file mode 100644 index 0000000000..5f2d1d413a --- /dev/null +++ b/packages/kvs-client/src/types.ts @@ -0,0 +1,49 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +/** + * Client for interacting with the key-value store API + * @public + */ +export interface KeyValueClient { + /** + * Store a value for a key in a namespace + * @param key - The key to store the value under + * @param value - The value to store + * @returns Promise that resolves when the value is stored + */ + setValue: (key: string, value: T) => Promise + + /** + * Retrieve a value for a key in a namespace + * @param key - The key to retrieve the value for + * @returns Promise that resolves to the value, or null if not found + */ + getValue: (key: string) => Promise + + /** + * Delete a key-value pair in a namespace + * @param key - The key to delete + * @returns Promise that resolves when the key is deleted + */ + deleteKey: (key: string) => Promise + + /** + * List all key-value pairs in a namespace, optionally filtered by prefix + * @param prefix - Optional prefix to filter keys by + * @returns Promise that resolves to an object with keys and their values + */ + listKeys: (prefix?: string) => Promise | null> +} diff --git a/packages/kvs-client/tsconfig.json b/packages/kvs-client/tsconfig.json new file mode 100644 index 0000000000..59e4fd4297 --- /dev/null +++ b/packages/kvs-client/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/plugins/gmail-assets/lang/cs.json b/plugins/gmail-assets/lang/cs.json index 15bca6da7d..1e626bb02a 100644 --- a/plugins/gmail-assets/lang/cs.json +++ b/plugins/gmail-assets/lang/cs.json @@ -38,6 +38,7 @@ "ConfigLabel": "E-mail", "ConfigDescription": "Rozšíření pro integraci e-mailů Gmail", "GooglePrivacy": "Použití a přenos informací získaných z Google API do jakékoli jiné aplikace bude v souladu s zásadami Google API Services User Data Policy, včetně požadavků na omezené použití.", - "EmailNotificationsDescription": "Dostávejte osobní oznámení na e-mail." + "EmailNotificationsDescription": "Dostávejte osobní oznámení na e-mail.", + "FailedToConnect": "Nepodařilo se připojit k Gmail" } } \ No newline at end of file diff --git a/plugins/gmail-assets/lang/de.json b/plugins/gmail-assets/lang/de.json index 8f0c1a991b..7689d8f192 100644 --- a/plugins/gmail-assets/lang/de.json +++ b/plugins/gmail-assets/lang/de.json @@ -38,6 +38,7 @@ "ConfigLabel": "E-Mail", "ConfigDescription": "Erweiterung für Gmail-Integration", "GooglePrivacy": "Die Nutzung und Übertragung von Informationen, die Huly von Google-APIs erhält, an andere Apps erfolgt gemäß der Google API Services User Data Policy, einschließlich der Anforderungen zur eingeschränkten Nutzung.", - "EmailNotificationsDescription": "Erhalten Sie persönliche Benachrichtigungen per E-Mail." + "EmailNotificationsDescription": "Erhalten Sie persönliche Benachrichtigungen per E-Mail.", + "FailedToConnect": "Fehler beim Verbinden mit Gmail" } } \ No newline at end of file diff --git a/plugins/gmail-assets/lang/en.json b/plugins/gmail-assets/lang/en.json index f18b7665c4..a7480da6eb 100644 --- a/plugins/gmail-assets/lang/en.json +++ b/plugins/gmail-assets/lang/en.json @@ -38,6 +38,7 @@ "ConfigLabel": "Email", "ConfigDescription": "Extension for Gmail email integration", "GooglePrivacy": "Huly’s use and transfer of information received from Google APIs to any other app will adhere to Google API Services User Data Policy, including the Limited Use requirements.", - "EmailNotificationsDescription": "Receive personal notifications on email." + "EmailNotificationsDescription": "Receive personal notifications on email.", + "FailedToConnect": "Failed to connect to Gmail" } } \ No newline at end of file diff --git a/plugins/gmail-assets/lang/es.json b/plugins/gmail-assets/lang/es.json index 3a38407cb9..e5c3ecd199 100644 --- a/plugins/gmail-assets/lang/es.json +++ b/plugins/gmail-assets/lang/es.json @@ -38,6 +38,7 @@ "ConfigLabel": "Correo Electrónico", "ConfigDescription": "Extensión para la integración de correo electrónico de Gmail", "GooglePrivacy": "Huly’s use and transfer of information received from Google APIs to any other app will adhere to Google API Services User Data Policy, including the Limited Use requirements.", - "EmailNotificationsDescription": "Reciba notificaciones personales por correo electrónico." + "EmailNotificationsDescription": "Reciba notificaciones personales por correo electrónico.", + "FailedToConnect": "Error al conectar con Gmail" } } \ No newline at end of file diff --git a/plugins/gmail-assets/lang/fr.json b/plugins/gmail-assets/lang/fr.json index 173e4c6065..c05a212fd0 100644 --- a/plugins/gmail-assets/lang/fr.json +++ b/plugins/gmail-assets/lang/fr.json @@ -38,6 +38,7 @@ "ConfigLabel": "Email", "ConfigDescription": "Extension pour l'intégration des emails Gmail", "GooglePrivacy": "L'utilisation et le transfert des informations reçues des API Google par Huly à toute autre application respecteront les règles d'utilisation des données utilisateur des services API Google, y compris les exigences d'utilisation limitée.", - "EmailNotificationsDescription": "Recevez des notifications personnelles par e-mail." + "EmailNotificationsDescription": "Recevez des notifications personnelles par e-mail.", + "FailedToConnect": "Impossible de se connecter au compte Gmail" } } \ No newline at end of file diff --git a/plugins/gmail-assets/lang/it.json b/plugins/gmail-assets/lang/it.json index 68fc2a19c2..72c775d58f 100644 --- a/plugins/gmail-assets/lang/it.json +++ b/plugins/gmail-assets/lang/it.json @@ -38,6 +38,7 @@ "ConfigLabel": "Email", "ConfigDescription": "Estensione per l'integrazione delle email di Gmail", "GooglePrivacy": "L'uso e il trasferimento delle informazioni ricevute dalle API di Google a qualsiasi altra app di Huly adempirà alla Politica sui dati degli utenti dei servizi API di Google, inclusi i requisiti di utilizzo limitato.", - "EmailNotificationsDescription": "Ricevi notifiche personali sulle email." + "EmailNotificationsDescription": "Ricevi notifiche personali sulle email.", + "FailedToConnect": "Impossibile connettere l'account Gmail" } } diff --git a/plugins/gmail-assets/lang/pt.json b/plugins/gmail-assets/lang/pt.json index 35a2ad469c..1eea278f1c 100644 --- a/plugins/gmail-assets/lang/pt.json +++ b/plugins/gmail-assets/lang/pt.json @@ -38,6 +38,7 @@ "ConfigLabel": "Email", "ConfigDescription": "Extensão para a integração de email do Gmail", "GooglePrivacy": "Huly’s use and transfer of information received from Google APIs to any other app will adhere to Google API Services User Data Policy, including the Limited Use requirements.", - "EmailNotificationsDescription": "Receba notificações pessoais por e-mail." + "EmailNotificationsDescription": "Receba notificações pessoais por e-mail.", + "FailedToConnect": "Não foi possível conectar a conta do Gmail" } } \ No newline at end of file diff --git a/plugins/gmail-assets/lang/ru.json b/plugins/gmail-assets/lang/ru.json index d395af3142..f1ad4201ba 100644 --- a/plugins/gmail-assets/lang/ru.json +++ b/plugins/gmail-assets/lang/ru.json @@ -38,6 +38,7 @@ "ConfigLabel": "Электронная почта", "ConfigDescription": "Расширение по работе с Gmail электронной почтой", "GooglePrivacy": "Использование и передача информации, полученной Huly от Google API, будет соответствовать Политике использования данных пользователей Google API, включая требования ограниченного использования.", - "EmailNotificationsDescription": "Получайте персональные уведомления на электронную почту." + "EmailNotificationsDescription": "Получайте персональные уведомления на электронную почту.", + "FailedToConnect": "Не удалось подключиться к Gmail" } } \ No newline at end of file diff --git a/plugins/gmail-assets/lang/zh.json b/plugins/gmail-assets/lang/zh.json index fdbb854092..32988675f8 100644 --- a/plugins/gmail-assets/lang/zh.json +++ b/plugins/gmail-assets/lang/zh.json @@ -38,6 +38,7 @@ "ConfigLabel": "电子邮件", "ConfigDescription": "Gmail 邮件集成扩展", "GooglePrivacy": "Huly 从 Google API 接收的信息的使用和传输将遵守 Google API 服务用户数据政策，包括有限使用要求。", - "EmailNotificationsDescription": "透過電子郵件接收個人通知。" + "EmailNotificationsDescription": "透過電子郵件接收個人通知。", + "FailedToConnect": "無法連接 Gmail 帳戶。請檢查您的憑證並重試。" } } diff --git a/plugins/gmail-resources/src/components/Chats.svelte b/plugins/gmail-resources/src/components/Chats.svelte index e293687b0c..2579596183 100644 --- a/plugins/gmail-resources/src/components/Chats.svelte +++ b/plugins/gmail-resources/src/components/Chats.svelte @@ -16,14 +16,12 @@ {#if channel && object} @@ -154,14 +163,7 @@ {:else if currentMessage} {:else} - 0} - on:select={selectHandler} - /> + 0} on:select={selectHandler} /> {/if} {/if} diff --git a/plugins/gmail-resources/src/plugin.ts b/plugins/gmail-resources/src/plugin.ts index eb0cdf3cb8..25c151b383 100644 --- a/plugins/gmail-resources/src/plugin.ts +++ b/plugins/gmail-resources/src/plugin.ts @@ -43,6 +43,7 @@ export default mergeIds(gmailId, gmail, { Email: '' as IntlString, HaveWrittenEmail: '' as IntlString, NewIncomingMessage: '' as IntlString, - GooglePrivacy: '' as IntlString + GooglePrivacy: '' as IntlString, + FailedToConnect: '' as IntlString } }) diff --git a/plugins/gmail-resources/src/utils.ts b/plugins/gmail-resources/src/utils.ts index a2cd0c1f45..0601ccff39 100644 --- a/plugins/gmail-resources/src/utils.ts +++ b/plugins/gmail-resources/src/utils.ts @@ -1,9 +1,10 @@ -// import { getName as getContactName } from '@hcengineering/contact' -import contact, { type Channel, type Contact, type Employee } from '@hcengineering/contact' -import { type Client, type Doc, type IdMap, type Ref } from '@hcengineering/core' +import { get } from 'svelte/store' +import { getName as getContactName } from '@hcengineering/contact' +import contact, { type Channel, type Contact } from '@hcengineering/contact' +import { personByPersonIdStore, employeeBySocialKeyStore } from '@hcengineering/contact-resources' +import { buildSocialIdString, type PersonId, SocialIdType, type Client, type Doc, type Ref } from '@hcengineering/core' import { type Message, type SharedMessage } from '@hcengineering/gmail' import { getClient } from '@hcengineering/presentation' -import { type Integration } from '@hcengineering/setting' import gmail from './plugin' export function getTime (time: number): string { @@ -61,78 +62,53 @@ export async function checkHasEmail (doc: Doc | Doc[] | undefined): Promise()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))/ +const EMAIL_REGEX = + /(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))/ -export function convertMessages ( - object: Contact, - channel: Channel, - messages: Message[], - integrations: Integration[], - accounts: any, - employees: IdMap -): SharedMessage[] { - // TODO: FIXME - throw new Error('Not implemented') - // return messages.map((m) => { - // return { - // ...m, - // _id: m._id as string as Ref, - // sender: getName(object, channel, m, integrations, accounts, employees, true), - // receiver: getName(object, channel, m, integrations, accounts, employees, false) - // } - // }) +export function convertMessages (object: Contact, channel: Channel, messages: Message[]): SharedMessage[] { + return messages.map((m) => { + return { + ...m, + _id: m._id as string as Ref, + sender: getName(object, channel, m, true), + receiver: getName(object, channel, m, false) + } + }) } -export async function convertMessage ( - object: Contact, - channel: Channel, - message: Message, - integrations: Integration[], - accounts: any, - employees: IdMap -): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // return { - // ...message, - // _id: message._id as string as Ref, - // sender: getName(object, channel, message, integrations, accounts, employees, true), - // receiver: getName(object, channel, message, integrations, accounts, employees, false) - // } +export async function convertMessage (object: Contact, channel: Channel, message: Message): Promise { + return { + ...message, + _id: message._id as string as Ref, + sender: getName(object, channel, message, true), + receiver: getName(object, channel, message, false) + } } -export function getName ( - object: Contact, - channel: Channel, - message: Message, - integrations: Integration[], - accounts: any, - employees: IdMap, - sender: boolean -): string { - // TODO: FIXME - throw new Error('Not implemented') - // const h = getClient().getHierarchy() - // if (message._class === gmail.class.NewMessage) { - // if (!sender) return `${getContactName(h, object)} (${channel.value})` - // const from = message.from ?? message.createdBy ?? message.modifiedBy - // const account = accounts.get(from) - // const emp = account != null ? employees.get(account?.person as Ref) : undefined - // const integration = integrations.find((p) => p.createdBy === from) - // const email = integration?.value ?? integrations[0]?.value - // return emp != null ? `${getContactName(h, emp)} (${email})` : email - // } - // if (message.incoming === sender) { - // return `${getContactName(h, object)} (${channel.value})` - // } else { - // const account = accounts.get(message.modifiedBy) - // const emp = account != null ? employees.get(account?.person as Ref) : undefined - // const value = message.incoming ? message.to : message.from - // const email = value.match(EMAIL_REGEX) - // const emailVal = email?.[0] ?? value - // return emp != null ? `${getContactName(h, emp)} (${emailVal})` : emailVal - // } +export function getName (object: Contact, channel: Channel, message: Message, sender: boolean): string { + const h = getClient().getHierarchy() + if (message._class === gmail.class.NewMessage) { + if (!sender) return `${getContactName(h, object)} (${channel.value})` + return getPersonName(message.from ?? message.createdBy ?? message.modifiedBy) + } + if (message.incoming === sender) { + return `${getContactName(h, object)} (${channel.value})` + } else { + return getPersonName(message.modifiedBy) + } +} + +export function getPersonName (emailOrId: string): string { + const personName = get(personByPersonIdStore).get(emailOrId as PersonId)?.name + if (personName != null) return personName + const emailSearch = emailOrId.match(EMAIL_REGEX) + const email = emailSearch?.[0] + if (email != null && email !== '') { + const socialId = buildSocialIdString({ type: SocialIdType.EMAIL, value: email }) + const name = get(employeeBySocialKeyStore).get(socialId)?.name + if (name != null) return name + } + return emailOrId } export async function MessageTitleProvider (client: Client, ref: Ref, doc?: Message): Promise { diff --git a/rush.json b/rush.json index fe20065deb..1b1b984c36 100644 --- a/rush.json +++ b/rush.json @@ -2472,6 +2472,11 @@ "packageName": "@hcengineering/model-achievement", "projectFolder": "models/achievement", "shouldPublish": false + }, + { + "packageName": "@hcengineering/kvs-client", + "projectFolder": "packages/kvs-client", + "shouldPublish": false } ] } diff --git a/server/account/src/serviceOperations.ts b/server/account/src/serviceOperations.ts index ad7ad308af..1db6e1b53d 100644 --- a/server/account/src/serviceOperations.ts +++ b/server/account/src/serviceOperations.ts @@ -549,7 +549,7 @@ export async function addSocialIdToPerson ( const { person, type, value, confirmed, displayValue } = params const { extra } = decodeTokenVerbose(ctx, token) - verifyAllowedServices(['github', 'telegram-bot'], extra) + verifyAllowedServices(['github', 'telegram-bot', 'gmail'], extra) return await addSocialId(db, person, type, value, confirmed, displayValue) } @@ -564,7 +564,7 @@ export async function updateSocialId ( const { personId, displayValue } = params const { extra } = decodeTokenVerbose(ctx, token) - verifyAllowedServices(['telegram-bot'], extra) + verifyAllowedServices(['telegram-bot', 'gmail'], extra) const socialId = await db.socialId.findOne({ _id: personId }) if (socialId != null) { @@ -815,7 +815,7 @@ export async function findFullSocialIdBySocialKey ( params: { socialKey: string } ): Promise { const { extra } = decodeTokenVerbose(ctx, token) - verifyAllowedServices(['telegram-bot'], extra) + verifyAllowedServices(['telegram-bot', 'gmail'], extra) const { socialKey } = params diff --git a/server/account/src/utils.ts b/server/account/src/utils.ts index af9174315a..9022f6ac35 100644 --- a/server/account/src/utils.ts +++ b/server/account/src/utils.ts @@ -583,7 +583,7 @@ export async function selectWorkspace ( throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspaceUrl })) } - if (accountUuid !== systemAccountUuid) { + if (accountUuid !== systemAccountUuid && meta !== undefined) { void setTimezoneIfNotDefined(ctx, db, accountUuid, account, meta) } @@ -1475,7 +1475,7 @@ export async function setTimezoneIfNotDefined ( } // Move to config? -export const integrationServices = ['github', 'telegram-bot', 'telegram', 'mailbox', 'caldav'] +export const integrationServices = ['github', 'telegram-bot', 'telegram', 'mailbox', 'caldav', 'gmail'] export async function findExistingIntegration ( account: AccountUuid, diff --git a/services/gmail/pod-gmail/.gitignore b/services/gmail/pod-gmail/.gitignore new file mode 100644 index 0000000000..2eea525d88 --- /dev/null +++ b/services/gmail/pod-gmail/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/services/gmail/pod-gmail/README.md b/services/gmail/pod-gmail/README.md new file mode 100644 index 0000000000..1a21d80016 --- /dev/null +++ b/services/gmail/pod-gmail/README.md @@ -0,0 +1,116 @@ +# Pod Gmail Service + +A service that provides Gmail integration capabilities for the Hardcore Engineering Platform. This service handles Gmail authentication, message synchronization, and workspace management. + +## Overview + +The pod-gmail service is responsible for: +- Managing Gmail client connections +- Synchronizing emails across workspaces +- Handling user authentication and token management +- Providing Gmail integration features for the platform + +## Prerequisites + +- Node.js 16 or higher +- Gmail API credentials +- Hardcore Engineering Platform dependencies + +## Installation + +1. Install dependencies: +```bash +npm install +``` + +2. Configure environment variables: +- Create a `.env` file with necessary credentials +- Configure Gmail API credentials + +## Example .env Configuration + +Below is an example `.env` file for running the service locally: + +```env +PORT=8093 +ACCOUNTS_URL="http://127.0.0.1:3000" +SECRET="secret" +WATCH_TOPIC_NAME="test" +Credentials={"web":{"client_id":"get_from_your_gmail_account","client_secret":"get_from_your_gmail_account","redirect_uris":["http://localhost:8088/signin/code"]}} +MINIO_ENDPOINT="localhost" +MINIO_ACCESS_KEY="minioadmin" +MINIO_SECRET_KEY="minioadmin" +KVS_URL="http://localhost:8098" +``` + +Make sure to replace the values with your actual credentials and configuration. + +## Development + +### Running Locally + +```bash +rushx run-local +``` + +### Building + +```bash +rushx build +``` + +### Testing + +```bash +rushx test +``` + +## Docker Support + +The service can be built using Docker: + +```bash +# Build Docker image +rushx docker:build +``` + +## Configuration + +The service requires the following configurations: +- Gmail API credentials +- Workspace-specific settings +- Rate limiting configurations + +## API Integration + +The service integrates with: +- Gmail API for email operations +- Hardcore Engineering Platform core services +- Server core components + +## Dependencies + +### Core Dependencies +- @hcengineering/core +- @hcengineering/server-core +- @hcengineering/server-client +- @hcengineering/server-token +- @hcengineering/gmail + +### External Dependencies +- googleapis +- google-auth-library +- express +- cors + +## License + +This project is licensed under the Eclipse Public License, Version 2.0. See the LICENSE file for details. + +## Contributing + +Contributions are welcome! Please read our contributing guidelines before submitting pull requests. + +## Support + +For support, please contact the Hardcore Engineering team or open an issue in the repository. \ No newline at end of file diff --git a/services/gmail/pod-gmail/package.json b/services/gmail/pod-gmail/package.json index ac7f8a8b6c..41cc62a0bc 100644 --- a/services/gmail/pod-gmail/package.json +++ b/services/gmail/pod-gmail/package.json @@ -35,6 +35,7 @@ "@types/cors": "^2.8.12", "@types/express": "^4.17.13", "@types/node": "~20.11.16", + "@types/sanitize-html": "^2.15.0", "@typescript-eslint/eslint-plugin": "^6.11.0", "@typescript-eslint/parser": "^6.11.0", "esbuild": "^0.24.2", @@ -50,10 +51,13 @@ "prettier": "^3.1.0", "ts-node-dev": "^2.0.0", "typescript": "^5.3.3", - "@types/uuid": "^8.3.1" + "@types/uuid": "^8.3.1", + "ts-node": "^10.8.0", + "cross-env": "~7.0.3" }, "dependencies": { "@hcengineering/attachment": "^0.6.14", + "@hcengineering/account-client": "^0.6.0", "@hcengineering/client": "^0.6.18", "@hcengineering/client-resources": "^0.6.27", "@hcengineering/contact": "^0.6.24", @@ -66,6 +70,7 @@ "@hcengineering/server-client": "^0.6.0", "@hcengineering/server-storage": "^0.6.0", "@hcengineering/server-token": "^0.6.11", + "@hcengineering/kvs-client": "^0.6.0", "cors": "^2.8.5", "dotenv": "~16.0.0", "express": "^4.21.2", @@ -77,6 +82,7 @@ "jwt-simple": "^0.5.6", "mongodb": "^6.12.0", "uuid": "^8.3.2", - "@hcengineering/analytics-service": "^0.6.0" + "@hcengineering/analytics-service": "^0.6.0", + "sanitize-html": "^2.15.0" } } diff --git a/services/gmail/pod-gmail/src/__mocks__/config.ts b/services/gmail/pod-gmail/src/__mocks__/config.ts new file mode 100644 index 0000000000..fbc6ce1c62 --- /dev/null +++ b/services/gmail/pod-gmail/src/__mocks__/config.ts @@ -0,0 +1,12 @@ +const config = { + Port: 8087, + AccountsURL: 'http://localhost:8080', + ServiceID: 'gmail-service-test', + Secret: 'test-secret', + Credentials: 'test-credentials', + WATCH_TOPIC_NAME: 'test-watch-topic', + FooterMessage: '

Sent via Huly

', + InitLimit: 50 +} + +export default config diff --git a/services/gmail/pod-gmail/src/__tests__/attachments.test.ts b/services/gmail/pod-gmail/src/__tests__/attachments.test.ts new file mode 100644 index 0000000000..def6a84b28 --- /dev/null +++ b/services/gmail/pod-gmail/src/__tests__/attachments.test.ts @@ -0,0 +1,223 @@ +import { + MeasureContext, + Ref, + Blob, + TxOperations, + WorkspaceUuid, + Doc, + Space, + Class, + PersonId +} from '@hcengineering/core' +import { StorageAdapter } from '@hcengineering/server-core' +import { gmail_v1 } from 'googleapis' +import { AttachmentHandler } from '../message/attachments' +import type { AttachedFile } from '../types' +import attachment, { Attachment } from '@hcengineering/attachment' +import { decode64, encode64 } from '../base64' + +jest.mock('../config') + +/* eslint-disable @typescript-eslint/unbound-method */ + +describe('AttachmentHandler', () => { + const mockCtx: MeasureContext = { + contextData: {}, + newChild: jest.fn(), + with: jest.fn(), + withSync: jest.fn(), + withLog: jest.fn(), + logger: { + info: jest.fn(), + error: jest.fn(), + warn: jest.fn(), + logOperation: jest.fn(), + childLogger: jest.fn(), + close: jest.fn() + }, + measure: jest.fn(), + error: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + end: jest.fn() + } + const mockWorkspaceId = 'test-workspace' as WorkspaceUuid + const mockStorageAdapter = { + put: jest.fn(), + read: jest.fn() + } as unknown as StorageAdapter + const mockGmail = { + messages: { + attachments: { + get: jest.fn() + } + } + } as unknown as gmail_v1.Resource$Users + const mockClient = { + addCollection: jest.fn(), + findAll: jest.fn() + } as unknown as TxOperations + + const getBaseAttachment = (): Attachment => ({ + _id: 'test-attachment' as Ref, + name: 'test.txt', + type: 'text/plain', + file: 'test-file' as Ref, + size: 0, + lastModified: 0, + attachedTo: '' as Ref, + attachedToClass: '' as Ref>, + collection: '', + space: '' as Ref, + modifiedOn: 0, + modifiedBy: '' as PersonId, + _class: '' as Ref> + }) + + let attachmentHandler: AttachmentHandler + + beforeEach(() => { + attachmentHandler = new AttachmentHandler(mockCtx, mockWorkspaceId, mockStorageAdapter, mockGmail, mockClient) + }) + + describe('addAttachement', () => { + it('should not add attachment if it already exists', async () => { + const file: AttachedFile = { + file: 'test-file', + name: 'test.txt', + type: 'text/plain', + size: 100, + lastModified: Date.now() + } + const message = { _id: 'test-message', _class: 'test-class', space: 'test-space' } + const currentAttachments: Attachment[] = [ + { + ...getBaseAttachment(), + name: 'test.txt', + lastModified: file.lastModified + } + ] + + await attachmentHandler.addAttachement(file, message, currentAttachments) + + expect(mockStorageAdapter.put).not.toHaveBeenCalled() + expect(mockClient.addCollection).not.toHaveBeenCalled() + }) + + it('should add new attachment', async () => { + const file: AttachedFile = { + file: 'test-file', + name: 'test.txt', + type: 'text/plain', + size: 100, + lastModified: Date.now() + } + const message = { _id: 'test-message', _class: 'test-class', space: 'test-space' } + const currentAttachments: Attachment[] = [] + + await attachmentHandler.addAttachement(file, message, currentAttachments) + + expect(mockStorageAdapter.put).toHaveBeenCalled() + expect(mockClient.addCollection).toHaveBeenCalledWith( + attachment.class.Attachment, + message.space, + message._id, + message._class, + 'attachments', + expect.any(Object) + ) + }) + }) + + describe('getPartFiles', () => { + it('should return empty array for undefined part', async () => { + const result = await attachmentHandler.getPartFiles(undefined, 'test-message') + expect(result).toEqual([]) + }) + + it('should get files from attachment', async () => { + const part: gmail_v1.Schema$MessagePart = { + filename: 'test.txt', + mimeType: 'text/plain', + body: { + attachmentId: 'test-attachment', + size: 100 + } + } + const mockAttachment = { + data: 'test-data', + size: 100 + } + ;(mockGmail.messages.attachments.get as jest.Mock).mockResolvedValue({ data: mockAttachment }) + + const result = await attachmentHandler.getPartFiles(part, 'test-message') + + expect(result).toEqual([ + { + file: 'test-data', + name: 'test.txt', + type: 'text/plain', + size: 100, + lastModified: expect.any(Number) + } + ]) + }) + + it('should get files from body data', async () => { + const part: gmail_v1.Schema$MessagePart = { + filename: 'test.txt', + mimeType: 'text/plain', + body: { + data: 'test-data', + size: 100 + } + } + + const result = await attachmentHandler.getPartFiles(part, 'test-message') + + expect(result).toEqual([ + { + file: 'test-data', + name: 'test.txt', + type: 'text/plain', + size: 100, + lastModified: expect.any(Number) + } + ]) + }) + }) + + describe('makeAttachmentsBody', () => { + it('should create message body with attachments', async () => { + const message = { + to: 'test@example.com', + subject: 'Test Subject', + content: 'Test Content', + _id: 'test-message' + } + const from = 'from@example.com' + const attachment: Attachment = { + ...getBaseAttachment(), + _id: 'test-attachment' as Ref, + name: 'test.txt', + type: 'text/plain', + file: 'test-file' as Ref + } + const mockAttachments: Attachment[] = [attachment] + ;(mockClient.findAll as jest.Mock).mockResolvedValue(mockAttachments) + ;(mockStorageAdapter.read as jest.Mock).mockResolvedValue([Buffer.from('test-data', 'utf8')]) + + const result = await attachmentHandler.makeAttachmentsBody(message, from) + const decodedResult = decode64(result) + + expect(decodedResult).toContain('Content-Type: multipart/mixed') + expect(decodedResult).toContain('To: test@example.com') + expect(decodedResult).toContain('From: from@example.com') + expect(decodedResult).toContain(`Subject: =?UTF-8?B?${encode64('Test Subject')}?= \n`) + expect(decodedResult).toContain('Test Content') + expect(decodedResult).toContain('test.txt') + expect(decodedResult).toContain('Content-Disposition: attachment; filename="test.txt"') + expect(decodedResult).toContain('

Sent via Huly

') + }) + }) +}) diff --git a/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts b/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts new file mode 100644 index 0000000000..a54f0fcca5 --- /dev/null +++ b/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts @@ -0,0 +1,345 @@ +import { + AccountUuid, + PersonId, + WorkspaceUuid, + type SocialId, + type SocialIdType, + type MeasureContext +} from '@hcengineering/core' + +import { GmailClient } from '../gmail' +import { ProjectCredentials, ProjectCredentialsData, Token, User } from '../types' +import { TokenStorage } from '../tokens' +import { SyncManager } from '../message/sync' + +// Mock dependencies +const mockedGoogleAuth = { + generateAuthUrl: jest.fn().mockReturnValue('https://auth-url.com'), + getToken: jest.fn().mockResolvedValue({ tokens: { access_token: 'new-token' } }), + setCredentials: jest.fn(), + refreshAccessToken: jest.fn().mockResolvedValue({ credentials: { access_token: 'refreshed-token' } }), + revokeCredentials: jest.fn().mockResolvedValue({}) +} + +const mockedGmailUsers = { + getProfile: jest.fn().mockResolvedValue({ data: { emailAddress: 'test@example.com' } }), + messages: { + send: jest.fn().mockResolvedValue({}), + list: jest.fn().mockResolvedValue({ data: { messages: [] } }) + }, + watch: jest.fn().mockResolvedValue({}), + stop: jest.fn().mockResolvedValue({}) +} + +// Mock all imports before they're used +jest.mock('@hcengineering/core', () => { + return { + AccountUuid: String, + PersonId: String, + WorkspaceUuid: String, + // Provide missing properties + configUserAccountUuid: 'test-user-id', + core: { + space: { + Workspace: 'workspace' + } + }, + TxOperations: jest.fn().mockImplementation(() => ({ + findAll: jest.fn().mockResolvedValue([]), + findOne: jest.fn().mockResolvedValue(undefined), + createDoc: jest.fn().mockResolvedValue({}), + update: jest.fn().mockResolvedValue({}), + remove: jest.fn().mockResolvedValue({}), + updateDoc: jest.fn().mockResolvedValue({}), + tx: jest.fn().mockResolvedValue({}) + })) + } +}) + +jest.mock('googleapis', () => ({ + gmail_v1: {}, + google: { + auth: { + OAuth2: jest.fn().mockImplementation(() => mockedGoogleAuth) + }, + gmail: jest.fn().mockImplementation(() => ({ + users: mockedGmailUsers + })) + } +})) + +jest.mock('../tokens') +jest.mock('../message/message') +jest.mock('../message/sync') +jest.mock('../message/attachments') +jest.mock('@hcengineering/server-client', () => ({ + getAccountClient: jest.fn() +})) +jest.mock('../accounts', () => ({ + getOrCreateSocialId: jest.fn().mockResolvedValue({ _id: 'test-social-id' }) +})) +jest.mock('../gmail/utils', () => ({ + getEmail: jest.fn().mockResolvedValue('test@example.com') +})) +jest.mock('../integrations', () => ({ + createIntegrationIfNotEsixts: jest.fn().mockResolvedValue({ _id: 'test-integration-id' }), + disableIntegration: jest.fn(), + removeIntegration: jest.fn() +})) +jest.mock('../utils', () => ({ + addFooter: jest.fn().mockImplementation((content: string) => content), + isToken: jest.fn().mockImplementation((obj: any) => 'token' in obj), + serviceToken: jest.fn().mockReturnValue('service-token'), + getKvsClient: jest.fn().mockImplementation(() => ({ + getValue: jest.fn(), + setValue: jest.fn().mockImplementation(() => Promise.resolve(undefined)), + deleteKey: jest.fn().mockImplementation(() => Promise.resolve(undefined)), + listKeys: jest.fn() + })) +})) + +// Mock gmail module +jest.mock('@hcengineering/gmail', () => ({ + integrationType: { + Gmail: 'gmail' + }, + class: { + NewMessage: 'class.NewMessage' + } +})) + +// Mock setting module +jest.mock('@hcengineering/setting', () => ({ + class: { + Integration: 'class.Integration' + } +})) + +// Mock config +jest.mock('../config', () => ({ + WATCH_TOPIC_NAME: 'test-topic' +})) + +describe('GmailClient', () => { + const mockContext: MeasureContext = { + info: jest.fn(), + error: jest.fn(), + measure: jest.fn().mockImplementation((name: string, fn: () => any) => fn()), + with: jest.fn().mockImplementation((data: any, fn: () => any) => fn()) + } as any + + const mockWorkspaceId = 'test-workspace' as WorkspaceUuid + const mockUserId = 'test-user-id' as AccountUuid + + // Fix the SocialId type issue + const mockSocialId: SocialId = { + _id: 'test-social-id' as PersonId, + value: 'test@example.com', + type: 'email' as SocialIdType, + key: 'email' + } + + const credentialsData: ProjectCredentialsData = { + client_id: 'client-id', + client_secret: 'client-secret', + redirect_uris: ['https://redirect-uri.com'], + project_id: 'test-project', + auth_uri: 'https://auth-uri.com', + token_uri: 'https://token-uri.com', + auth_provider_x509_cert_url: 'https://cert-url.com' + } + const mockCredentials: ProjectCredentials = { + web: credentialsData + } + + const mockUser: User = { + userId: mockUserId, + workspace: mockWorkspaceId, + socialId: mockSocialId, + email: 'test@example.com', + token: 'user-token' + } + + const mockToken: Token = { + userId: mockUserId, + workspace: mockWorkspaceId, + socialId: mockSocialId, + token: 'test-token', + access_token: 'test-access-token', + refresh_token: 'test-refresh-token', + scope: 'test-scope', + token_type: 'Bearer', + expiry_date: 123456789 + } + + const mockClient = { + findAll: jest.fn().mockResolvedValue([]), + findOne: jest.fn().mockResolvedValue(undefined), + createDoc: jest.fn().mockResolvedValue({}), + update: jest.fn().mockResolvedValue({}), + remove: jest.fn().mockResolvedValue({}), + updateDoc: jest.fn().mockResolvedValue({}), + tx: jest.fn().mockResolvedValue({}) + } + + const mockWorkspaceClient = { + subscribeMessages: jest.fn().mockResolvedValue({}), + signoutBySocialId: jest.fn().mockResolvedValue({}) + } + + const mockStorageAdapter = {} + + beforeEach(() => { + jest.clearAllMocks() + // Fix unbound method errors by using the proper mock assignment approach + jest.spyOn(TokenStorage.prototype, 'getToken').mockResolvedValue(mockToken) + jest.spyOn(TokenStorage.prototype, 'saveToken').mockResolvedValue({} as any) + jest.spyOn(TokenStorage.prototype, 'deleteToken').mockResolvedValue() + jest.spyOn(SyncManager.prototype, 'sync').mockResolvedValue() + }) + + it('should create a GmailClient instance', async () => { + const client = await GmailClient.create( + mockContext as any, + mockCredentials, + mockUser, + mockClient as any, + mockWorkspaceClient as any, + mockWorkspaceId, + mockStorageAdapter as any + ) + + expect(client).toBeInstanceOf(GmailClient) + expect(TokenStorage).toHaveBeenCalled() + expect(mockContext.info).toHaveBeenCalledWith('Created gmail client', expect.any(Object)) + }) + + it('should create a GmailClient instance with auth code', async () => { + const client = await GmailClient.create( + mockContext as any, + mockCredentials, + mockUser, + mockClient as any, + mockWorkspaceClient as any, + mockWorkspaceId, + mockStorageAdapter as any, + 'auth-code' + ) + + expect(client).toBeInstanceOf(GmailClient) + expect(mockedGoogleAuth.setCredentials).toHaveBeenCalled() + expect(mockContext.info).toHaveBeenCalledWith('Created gmail client', expect.any(Object)) + }) + + it('should generate auth URL', async () => { + const client = await GmailClient.create( + mockContext as any, + mockCredentials, + mockUser, + mockClient as any, + mockWorkspaceClient as any, + mockWorkspaceId, + mockStorageAdapter as any + ) + + const authUrl = client.getAuthUrl('https://callback-url.com') + expect(authUrl).toBe('https://auth-url.com') + }) + + it('should sign out and clean up resources', async () => { + const client = await GmailClient.create( + mockContext as any, + mockCredentials, + mockUser, + mockClient as any, + mockWorkspaceClient as any, + mockWorkspaceId, + mockStorageAdapter as any + ) + + await client.signout() + + // Fix unbound method reference by using the spy + expect(jest.spyOn(TokenStorage.prototype, 'deleteToken')).toHaveBeenCalledWith(mockSocialId._id) + expect(mockContext.info).toHaveBeenCalledWith('Deactivate gmail client', expect.any(Object)) + }) + + it('should create a message', async () => { + const client = await GmailClient.create( + mockContext as any, + mockCredentials, + mockUser, + mockClient as any, + mockWorkspaceClient as any, + mockWorkspaceId, + mockStorageAdapter as any + ) + + const newMessage = { + _id: 'test-message-id', + _class: 'class.NewMessage', + space: 'space', + status: 'new', + from: mockSocialId._id, + to: 'recipient@example.com', + subject: 'Test Subject', + content: '

Test content

' + } + + // Mock internal TxOperations instance + const mockTxOperations = { + findAll: jest.fn().mockResolvedValue([]), + findOne: jest.fn().mockResolvedValue(undefined), + createDoc: jest.fn().mockResolvedValue({}), + update: jest.fn().mockResolvedValue({}), + remove: jest.fn().mockResolvedValue({}), + updateDoc: jest.fn().mockResolvedValue({}), + tx: jest.fn().mockResolvedValue({}) + } + + // Access and replace the private TxOperations instance + Object.defineProperty(client, 'client', { + value: mockTxOperations, + writable: true + }) + + await client.createMessage(newMessage as any) + + expect(mockTxOperations.updateDoc).toHaveBeenCalledWith('class.NewMessage', 'space', 'test-message-id', { + status: 'sent' + }) + }) + + it('should sync messages', async () => { + const client = await GmailClient.create( + mockContext as any, + mockCredentials, + mockUser, + mockClient as any, + mockWorkspaceClient as any, + mockWorkspaceId, + mockStorageAdapter as any + ) + + await client.sync() + + // Fix unbound method reference by using the spy + expect(jest.spyOn(SyncManager.prototype, 'sync')).toHaveBeenCalledWith(mockSocialId._id, 'test@example.com') + }) + + it('should initialize integration', async () => { + const client = await GmailClient.create( + mockContext as any, + mockCredentials, + mockUser, + mockClient as any, + mockWorkspaceClient as any, + mockWorkspaceId, + mockStorageAdapter as any + ) + + await client.initIntegration() + + expect(mockContext.info).toHaveBeenCalledWith('Init integration', expect.any(Object)) + }) +}) diff --git a/services/gmail/pod-gmail/src/__tests__/message.test.ts b/services/gmail/pod-gmail/src/__tests__/message.test.ts new file mode 100644 index 0000000000..62f7e3b830 --- /dev/null +++ b/services/gmail/pod-gmail/src/__tests__/message.test.ts @@ -0,0 +1,227 @@ +import { type MeasureContext, PersonId, TxOperations, AttachedData } from '@hcengineering/core' +import { type GaxiosResponse } from 'gaxios' +import { gmail_v1 } from 'googleapis' +import { type Message } from '@hcengineering/gmail' + +import { type Channel } from '../types' +import { AttachmentHandler } from '../message/attachments' +import { MessageManager, sanitizeText } from '../message/message' + +/* eslint-disable @typescript-eslint/consistent-type-assertions */ +/* eslint-disable @typescript-eslint/unbound-method */ + +jest.mock('../config') + +describe('MessageManager', () => { + let messageManager: MessageManager + let mockCtx: MeasureContext + let mockClient: TxOperations + let mockAttachmentHandler: AttachmentHandler + let mockSocialId: PersonId + let mockWorkspace: { getChannel: (email: string) => Channel | undefined } + + beforeEach(() => { + mockCtx = { + measure: jest.fn(), + with: jest.fn() + } as unknown as MeasureContext + + mockClient = { + findOne: jest.fn(), + findAll: jest.fn(), + tx: jest.fn() + } as unknown as TxOperations + + mockAttachmentHandler = { + getPartFiles: jest.fn(), + addAttachement: jest.fn() + } as unknown as AttachmentHandler + + mockSocialId = 'test-social-id' as PersonId + + mockWorkspace = { + getChannel: jest.fn() + } + + messageManager = new MessageManager(mockCtx, mockClient, mockAttachmentHandler, mockSocialId, mockWorkspace) + }) + + describe('saveMessage', () => { + const createMockGmailResponse = (): GaxiosResponse => ({ + config: {}, + data: { + id: 'test-message-id', + internalDate: '1234567890', + payload: { + headers: [ + { name: 'From', value: 'sender@example.com' }, + { name: 'To', value: 'recipient@example.com' }, + { name: 'Message-ID', value: 'test-message-id' }, + { name: 'Subject', value: 'Test Subject' } + ], + parts: [ + { + mimeType: 'text/plain', + body: { data: 'dGVzdCBjb250ZW50' } + }, + { + mimeType: 'text/html', + body: { data: 'PHA+dGVzdCBjb250ZW50PC9wPg==' } + } + ] + } + }, + status: 200, + statusText: 'OK', + headers: {}, + request: { + responseURL: 'https://example.com' + } + }) + + const createMockMessage = (): AttachedData => ({ + messageId: 'test-message-id', + textContent: 'test content', + subject: 'Test Subject', + content: 'test content', + sendOn: 1234567890, + from: 'sender@example.com', + to: 'recipient@example.com', + copy: [], + incoming: true + }) + + it('should save message with attachments', async () => { + const mockMessage = createMockGmailResponse() + const mockChannel = { _id: 'test-channel-id' } as Channel + mockWorkspace.getChannel = jest.fn().mockReturnValue(mockChannel) + mockClient.findOne = jest.fn().mockResolvedValue(undefined) + mockAttachmentHandler.getPartFiles = jest.fn().mockResolvedValue([]) + + await messageManager.saveMessage(mockMessage, 'test@example.com') + + expect(mockClient.tx).toHaveBeenCalled() + expect(mockAttachmentHandler.getPartFiles).toHaveBeenCalled() + }) + + it('should update existing message', async () => { + const mockMessage = createMockGmailResponse() + const mockChannel = { _id: 'test-channel-id' } as Channel + const existingMessage = createMockMessage() + mockWorkspace.getChannel = jest.fn().mockReturnValue(mockChannel) + mockClient.findOne = jest.fn().mockResolvedValue(existingMessage) + mockAttachmentHandler.getPartFiles = jest.fn().mockResolvedValue([]) + + await messageManager.saveMessage(mockMessage, 'test@example.com') + + expect(mockClient.tx).toHaveBeenCalled() + expect(mockAttachmentHandler.getPartFiles).toHaveBeenCalled() + }) + + it('should not save message when no channels found', async () => { + const mockMessage = createMockGmailResponse() + mockWorkspace.getChannel = jest.fn().mockReturnValue(undefined) + + await messageManager.saveMessage(mockMessage, 'test@example.com') + + expect(mockClient.tx).not.toHaveBeenCalled() + expect(mockAttachmentHandler.getPartFiles).not.toHaveBeenCalled() + }) + + it('should handle incoming messages correctly', async () => { + const mockMessage = createMockGmailResponse() + const mockChannel = { _id: 'test-channel-id' } as Channel + mockWorkspace.getChannel = jest.fn().mockReturnValue(mockChannel) + mockClient.findOne = jest.fn().mockResolvedValue(undefined) + mockAttachmentHandler.getPartFiles = jest.fn().mockResolvedValue([]) + + await messageManager.saveMessage(mockMessage, 'recipient@example.com') + + expect(mockWorkspace.getChannel).toHaveBeenCalledWith('sender@example.com') + }) + + it('should handle outgoing messages correctly', async () => { + const mockMessage = createMockGmailResponse() + const mockChannel = { _id: 'test-channel-id' } as Channel + mockWorkspace.getChannel = jest.fn().mockReturnValue(mockChannel) + mockClient.findOne = jest.fn().mockResolvedValue(undefined) + mockAttachmentHandler.getPartFiles = jest.fn().mockResolvedValue([]) + + await messageManager.saveMessage(mockMessage, 'sender@example.com') + + expect(mockWorkspace.getChannel).toHaveBeenCalledWith('recipient@example.com') + }) + }) +}) + +describe('sanitizeHtml', () => { + it('should remove all HTML tags', () => { + const html = '

This is bold and italic text

' + const expected = 'This is bold and italic text' + expect(sanitizeText(html)).toBe(expected) + }) + + it('should handle nested tags properly', () => { + const html = '

Nested tags should be removed

' + const expected = 'Nested tags should be removed' + expect(sanitizeText(html)).toBe(expected) + }) + + it('should handle malformed HTML', () => { + const html = 'Text with paragraph

' + const expected = 'Text with paragraph' + expect(sanitizeText(html)).toBe(expected) + }) + + it('should handle scripts and potential XSS attacks', () => { + const html = 'Hello world' + const expected = 'Hello world' + expect(sanitizeText(html)).toBe(expected) + }) + + it('should remove style tags and attributes', () => { + const html = '

Styled text

' + const expected = 'Styled text' + expect(sanitizeText(html)).toBe(expected) + }) + + it('should handle empty input', () => { + expect(sanitizeText('')).toBe('') + }) +}) + +describe('sanitizeText', () => { + it('should remove all HTML tags', () => { + const html = '

This is bold and italic text

' + const expected = 'This is bold and italic text' + expect(sanitizeText(html)).toBe(expected) + }) + + it('should handle nested tags properly', () => { + const html = '

Nested tags should be removed

' + const expected = 'Nested tags should be removed' + expect(sanitizeText(html)).toBe(expected) + }) + + it('should handle malformed HTML', () => { + const html = 'Text with paragraph

' + const expected = 'Text with paragraph' + expect(sanitizeText(html)).toBe(expected) + }) + + it('should handle scripts and potential XSS attacks', () => { + const html = 'Hello world' + const expected = 'Hello world' + expect(sanitizeText(html)).toBe(expected) + }) + + it('should remove style tags and attributes', () => { + const html = '

Styled text

' + const expected = 'Styled text' + expect(sanitizeText(html)).toBe(expected) + }) + + it('should handle empty input', () => { + expect(sanitizeText('')).toBe('') + }) +}) diff --git a/services/gmail/pod-gmail/src/__tests__/sync.test.ts b/services/gmail/pod-gmail/src/__tests__/sync.test.ts new file mode 100644 index 0000000000..81e47709ca --- /dev/null +++ b/services/gmail/pod-gmail/src/__tests__/sync.test.ts @@ -0,0 +1,337 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { MeasureContext, PersonId } from '@hcengineering/core' +import { SyncManager, SyncMutex } from '../message/sync' +import { MessageManager } from '../message/message' +import { RateLimiter } from '../rateLimiter' + +jest.mock('../config') +jest.mock('../message/message') +jest.mock('../utils', () => { + const originalModule = jest.requireActual('../utils') + return { + ...originalModule, + getKvsClient: jest.fn().mockImplementation(() => mockKeyValueClientInstance) + } +}) + +// Create mock types +interface MockKeyValueClient { + getValue: jest.Mock + setValue: jest.Mock + deleteKey: jest.Mock + listKeys: jest.Mock +} + +// Create a shared mock instance that can be referenced in the mock above +const mockKeyValueClientInstance = { + getValue: jest.fn(), + setValue: jest.fn().mockImplementation(() => Promise.resolve(undefined)), + deleteKey: jest.fn().mockImplementation(() => Promise.resolve(undefined)), + listKeys: jest.fn() +} + +describe('SyncManager', () => { + // Mocked dependencies + let mockCtx: MeasureContext + let mockMessageManager: jest.Mocked + let mockGmail: any // Using any for easier mocking + let mockKeyValueClient: MockKeyValueClient + + // Test constants + const workspace = 'test-workspace' + const userId = 'user1' as PersonId + const userEmail = 'user1@example.com' + const historyId = '12345' + + // Initialize SyncManager instance for each test + let syncManager: SyncManager + + beforeEach(() => { + // Setup mocks + mockCtx = { + info: jest.fn(), + error: jest.fn() + } as unknown as jest.Mocked + + mockMessageManager = { + saveMessage: jest.fn().mockResolvedValue(undefined) + } as unknown as jest.Mocked + + mockGmail = { + history: { + list: jest.fn().mockImplementation(() => + Promise.resolve({ + data: { + history: [], + nextPageToken: undefined + } + }) + ) + }, + messages: { + list: jest.fn().mockImplementation(() => + Promise.resolve({ + data: { + messages: [], + nextPageToken: undefined + } + }) + ), + get: jest.fn().mockImplementation(() => + Promise.resolve({ + data: { + id: 'test-id', + historyId + } + }) + ) + } + } + + mockKeyValueClient = { + getValue: jest.fn(), + setValue: jest.fn().mockImplementation(() => Promise.resolve(undefined)), + deleteKey: jest.fn().mockImplementation(() => Promise.resolve(undefined)), + listKeys: jest.fn() + } + + // Create SyncManager instance + syncManager = new SyncManager( + mockCtx, + mockMessageManager, + mockGmail, + workspace, + mockKeyValueClient as any, + new RateLimiter(100, 100) + ) + }) + + afterEach(() => { + jest.clearAllMocks() + }) + + describe('getHistory', () => { + it('should retrieve history from key-value store', async () => { + // Arrange + const mockHistory = { + historyId, + userId, + workspace + } + mockKeyValueClient.getValue.mockResolvedValue(mockHistory) + + // Act + const result = await (syncManager as any).getHistory(userId) + + // Assert + expect(mockKeyValueClient.getValue).toHaveBeenCalledWith(`history:${workspace}:${userId}`) + expect(result).toEqual(mockHistory) + }) + + it('should return null when history not found', async () => { + // Arrange + mockKeyValueClient.getValue.mockResolvedValue(null) + + // Act + const result = await (syncManager as any).getHistory(userId) + + // Assert + expect(result).toBeNull() + }) + }) + + describe('setHistoryId', () => { + it('should store history in key-value store', async () => { + // Act + await (syncManager as any).setHistoryId(userId, historyId) + + // Assert + expect(mockKeyValueClient.setValue).toHaveBeenCalledWith(`history:${workspace}:${userId}`, { + historyId, + userId, + workspace + }) + }) + }) + + describe('clearHistory', () => { + it('should delete history from key-value store', async () => { + // Act + await (syncManager as any).clearHistory(userId) + + // Assert + expect(mockKeyValueClient.deleteKey).toHaveBeenCalledWith(`history:${workspace}:${userId}`) + }) + }) + + describe('sync', () => { + it('should perform full sync when no history exists', async () => { + // Arrange + mockKeyValueClient.getValue.mockResolvedValue(null) + const spyFullSync = jest.spyOn(syncManager, 'fullSync').mockResolvedValue(undefined) + + // Act + await syncManager.sync(userId, userEmail) + + // Assert + expect(spyFullSync).toHaveBeenCalledWith(userId, userEmail) + }) + + it('should perform part sync when history exists', async () => { + // Arrange + mockKeyValueClient.getValue.mockResolvedValue({ + historyId, + userId, + workspace + }) + + // Act + await syncManager.sync(userId, userEmail) + + // Assert + expect(mockGmail.history.list).toHaveBeenCalledWith({ + userId: 'me', + historyTypes: ['messageAdded'], + startHistoryId: historyId, + pageToken: undefined + }) + }) + }) + + describe('getMessage', () => { + it('should retrieve a message by ID', async () => { + // Arrange + const messageId = 'msg123' + const mockResponse = { + data: { + id: messageId, + historyId + } + } + + mockGmail.messages.get.mockImplementation(() => Promise.resolve(mockResponse)) + + // Act + const result = await (syncManager as any).getMessage(messageId) + + // Assert + expect(mockGmail.messages.get).toHaveBeenCalledWith({ + id: messageId, + userId: 'me', + format: 'FULL' + }) + expect(result).toEqual(mockResponse) + }) + }) +}) + +describe('SyncMutex', () => { + let syncMutex: SyncMutex + + beforeEach(() => { + syncMutex = new SyncMutex() + }) + + it('should allow sequential locking and unlocking', async () => { + // Lock first time + const release1 = await syncMutex.lock('test-key') + release1() + + // Lock second time + const release2 = await syncMutex.lock('test-key') + release2() + + // If we got here without hanging, the test passes + expect(true).toBe(true) + }) + + it('should queue up multiple requests for the same key', async () => { + const results: number[] = [] + + // Start 3 concurrent lock operations + const promise1 = (async () => { + const release = await syncMutex.lock('test-key') + results.push(1) + await new Promise((resolve) => setTimeout(resolve, 10)) + release() + })() + + const promise2 = (async () => { + const release = await syncMutex.lock('test-key') + results.push(2) + await new Promise((resolve) => setTimeout(resolve, 5)) + release() + })() + + const promise3 = (async () => { + const release = await syncMutex.lock('test-key') + results.push(3) + release() + })() + + // Wait for all promises to resolve + await Promise.all([promise1, promise2, promise3]) + + // The operations should have happened in order + expect(results).toEqual([1, 2, 3]) + }) + + it('should allow concurrent operations on different keys', async () => { + const results: string[] = [] + + // Lock two different keys concurrently + const promise1 = (async () => { + const release = await syncMutex.lock('key1') + results.push('key1-locked') + await new Promise((resolve) => setTimeout(resolve, 20)) + results.push('key1-unlocked') + release() + })() + + const promise2 = (async () => { + const release = await syncMutex.lock('key2') + results.push('key2-locked') + await new Promise((resolve) => setTimeout(resolve, 10)) + results.push('key2-unlocked') + release() + })() + + // Wait for both promises to resolve + await Promise.all([promise1, promise2]) + + // key2 operations should complete before key1 due to shorter timeout + expect(results.indexOf('key2-locked')).toBeLessThan(results.indexOf('key1-unlocked')) + }) + + it('should release the lock properly even if an error occurs', async () => { + try { + const release = await syncMutex.lock('test-key') + try { + throw new Error('Test error') + } finally { + release() + } + } catch (error) { + // Ignore the error + } + + // Should be able to acquire the lock again + const release = await syncMutex.lock('test-key') + release() + + expect(true).toBe(true) + }) +}) diff --git a/services/gmail/pod-gmail/src/__tests__/tokens.test.ts b/services/gmail/pod-gmail/src/__tests__/tokens.test.ts new file mode 100644 index 0000000000..cd2357af96 --- /dev/null +++ b/services/gmail/pod-gmail/src/__tests__/tokens.test.ts @@ -0,0 +1,139 @@ +import { AccountUuid, PersonId, WorkspaceUuid } from '@hcengineering/core' +import { AccountClient, IntegrationSecret } from '@hcengineering/account-client' +import { getAccountClient } from '@hcengineering/server-client' +import { TokenStorage } from '../tokens' +import { GMAIL_INTEGRATION, SecretType, Token } from '../types' +import type { MeasureContext } from '@hcengineering/core' + +// Mock the getAccountClient function +jest.mock('@hcengineering/server-client', () => ({ + getAccountClient: jest.fn() +})) + +describe('TokenStorage', () => { + const mockSocialId = 'test-social-id' as PersonId + const mockWorkspace = 'test-workspace' as WorkspaceUuid + const mockPlatformToken = 'test-platform-token' + const mockUserId = 'test-user' as AccountUuid + const mockToken: Token = { + userId: mockUserId, + socialId: { _id: '123' } as any, + workspace: mockWorkspace, + token: 'test-token', + access_token: 'test-access-token', + refresh_token: 'test-refresh-token', + scope: 'test-scope', + token_type: 'Bearer', + expiry_date: 123456789 + } + + let tokenStorage: TokenStorage + let mockAccountClient: jest.Mocked + // Add mock MeasureContext + const mockMeasureContext: MeasureContext = { + info: jest.fn(), + error: jest.fn() + } as any + + beforeEach(() => { + // Create mock AccountClient + mockAccountClient = { + getIntegrationSecret: jest.fn(), + addIntegrationSecret: jest.fn(), + updateIntegrationSecret: jest.fn(), + deleteIntegrationSecret: jest.fn() + } as any + + // Set up getAccountClient mock + ;(getAccountClient as jest.Mock).mockReturnValue(mockAccountClient) + + // Create TokenStorage instance with MeasureContext + tokenStorage = new TokenStorage(mockMeasureContext, mockWorkspace, mockPlatformToken) + }) + + afterEach(() => { + jest.clearAllMocks() + }) + + describe('getToken', () => { + it('should return null when no token exists', async () => { + mockAccountClient.getIntegrationSecret.mockResolvedValue(null) + const result = await tokenStorage.getToken(mockSocialId) + expect(result).toBeNull() + expect(mockAccountClient.getIntegrationSecret).toHaveBeenCalledWith({ + key: SecretType.TOKEN, + kind: GMAIL_INTEGRATION, + socialId: mockSocialId, + workspaceUuid: mockWorkspace + }) + }) + + it('should return parsed token when token exists', async () => { + const mockSecret: IntegrationSecret = { + socialId: mockSocialId, + kind: GMAIL_INTEGRATION, + workspaceUuid: mockWorkspace, + key: SecretType.TOKEN, + secret: JSON.stringify(mockToken) + } + mockAccountClient.getIntegrationSecret.mockResolvedValue(mockSecret) + const result = await tokenStorage.getToken(mockSocialId) + expect(result).toEqual(mockToken) + }) + }) + + describe('saveToken', () => { + it('should update token when it exists', async () => { + const existingToken = { existing: 'token' } + const mockSecret: IntegrationSecret = { + socialId: mockSocialId, + kind: GMAIL_INTEGRATION, + workspaceUuid: mockWorkspace, + key: SecretType.TOKEN, + secret: JSON.stringify(existingToken) + } + mockAccountClient.getIntegrationSecret.mockResolvedValue(mockSecret) + await tokenStorage.saveToken(mockSocialId, mockToken) + + // Expect merged token (existing token properties + new token properties) + const expectedMergedToken = { + ...existingToken, + ...mockToken + } + + expect(mockAccountClient.updateIntegrationSecret).toHaveBeenCalledWith({ + key: SecretType.TOKEN, + kind: GMAIL_INTEGRATION, + socialId: mockSocialId, + secret: JSON.stringify(expectedMergedToken), + workspaceUuid: mockWorkspace + }) + expect(mockAccountClient.addIntegrationSecret).not.toHaveBeenCalled() + }) + + it('should add new token when it does not exist', async () => { + mockAccountClient.getIntegrationSecret.mockResolvedValue(null) + await tokenStorage.saveToken(mockSocialId, mockToken) + expect(mockAccountClient.addIntegrationSecret).toHaveBeenCalledWith({ + key: SecretType.TOKEN, + kind: GMAIL_INTEGRATION, + socialId: mockSocialId, + secret: JSON.stringify(mockToken), + workspaceUuid: mockWorkspace + }) + expect(mockAccountClient.updateIntegrationSecret).not.toHaveBeenCalled() + }) + }) + + describe('deleteToken', () => { + it('should delete the token', async () => { + await tokenStorage.deleteToken(mockSocialId) + expect(mockAccountClient.deleteIntegrationSecret).toHaveBeenCalledWith({ + key: SecretType.TOKEN, + kind: GMAIL_INTEGRATION, + socialId: mockSocialId, + workspaceUuid: mockWorkspace + }) + }) + }) +}) diff --git a/services/gmail/pod-gmail/src/accounts.ts b/services/gmail/pod-gmail/src/accounts.ts new file mode 100644 index 0000000000..93a8f99ada --- /dev/null +++ b/services/gmail/pod-gmail/src/accounts.ts @@ -0,0 +1,74 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { AccountUuid, Person, PersonId, SocialId, SocialIdType, buildSocialIdString } from '@hcengineering/core' +import { getAccountClient } from '@hcengineering/server-client' +import { generateToken } from '@hcengineering/server-token' + +import { serviceToken } from './utils' + +export async function getAccountPerson (account: AccountUuid): Promise { + try { + const accountClient = getAccountClient(generateToken(account)) + return await accountClient.getPerson() + } catch (e) { + console.error(e) + } + return undefined +} + +export async function getAccountSocialIds (account: AccountUuid): Promise { + try { + const accountClient = getAccountClient(generateToken(account)) + return await accountClient.getSocialIds() + } catch (e) { + console.error(e) + } + return [] +} + +export async function getOrCreateSocialIdByEmail ( + token: string, + account: AccountUuid, + email: string +): Promise { + const client = getAccountClient(token) + const socialId = await client.findFullSocialIdBySocialKey( + buildSocialIdString({ type: SocialIdType.EMAIL, value: email }) + ) + if (socialId != null) return socialId._id + return await client.addSocialIdToPerson(account, SocialIdType.EMAIL, email, true) +} + +export async function getOrCreateSocialId (account: AccountUuid, email: string): Promise { + const accountClient = getAccountClient(serviceToken()) + let socialId = await accountClient.findFullSocialIdBySocialKey( + buildSocialIdString({ type: SocialIdType.EMAIL, value: email }) + ) + if (socialId == null) { + await accountClient.addSocialIdToPerson(account, SocialIdType.EMAIL, email, true) + socialId = await accountClient.findFullSocialIdBySocialKey( + buildSocialIdString({ type: SocialIdType.EMAIL, value: email }) + ) + if (socialId == null) { + throw new Error('Cannot create social id') + } + } + + if (socialId.personUuid !== account) { + throw new Error('Social id connected to another account') + } + + return socialId +} diff --git a/services/gmail/pod-gmail/src/config.ts b/services/gmail/pod-gmail/src/config.ts index 870f660a19..df14ab4134 100644 --- a/services/gmail/pod-gmail/src/config.ts +++ b/services/gmail/pod-gmail/src/config.ts @@ -13,12 +13,12 @@ // See the License for the specific language governing permissions and // limitations under the License. // +import { config as dotenvConfig } from 'dotenv' + +dotenvConfig() interface Config { Port: number - - MongoURI: string - MongoDB: string AccountsURL: string ServiceID: string Secret: string @@ -26,21 +26,19 @@ interface Config { WATCH_TOPIC_NAME: string FooterMessage: string InitLimit: number + KvsUrl: string } const envMap: { [key in keyof Config]: string } = { Port: 'PORT', - - MongoURI: 'MONGO_URI', - MongoDB: 'MONGO_DB', - AccountsURL: 'ACCOUNTS_URL', ServiceID: 'SERVICE_ID', Secret: 'SECRET', Credentials: 'Credentials', WATCH_TOPIC_NAME: 'WATCH_TOPIC_NAME', FooterMessage: 'FOOTER_MESSAGE', - InitLimit: 'INIT_LIMIT' + InitLimit: 'INIT_LIMIT', + KvsUrl: 'KVS_URL' } const parseNumber = (str: string | undefined): number | undefined => (str !== undefined ? Number(str) : undefined) @@ -48,15 +46,14 @@ const parseNumber = (str: string | undefined): number | undefined => (str !== un const config: Config = (() => { const params: Partial = { Port: parseNumber(process.env[envMap.Port]) ?? 8087, - MongoDB: process.env[envMap.MongoDB] ?? 'gmail-service', - MongoURI: process.env[envMap.MongoURI], AccountsURL: process.env[envMap.AccountsURL], ServiceID: process.env[envMap.ServiceID] ?? 'gmail-service', Secret: process.env[envMap.Secret], Credentials: process.env[envMap.Credentials], WATCH_TOPIC_NAME: process.env[envMap.WATCH_TOPIC_NAME], InitLimit: parseNumber(process.env[envMap.InitLimit]) ?? 50, - FooterMessage: process.env[envMap.FooterMessage] ?? '

Sent via Huly

' + FooterMessage: process.env[envMap.FooterMessage] ?? '

Sent via Huly

', + KvsUrl: process.env[envMap.KvsUrl] } const missingEnv = (Object.keys(params) as Array) diff --git a/services/gmail/pod-gmail/src/gmail.ts b/services/gmail/pod-gmail/src/gmail.ts index d0b425fa99..0db65ef703 100644 --- a/services/gmail/pod-gmail/src/gmail.ts +++ b/services/gmail/pod-gmail/src/gmail.ts @@ -13,44 +13,30 @@ // limitations under the License. // -import attachment, { Attachment } from '@hcengineering/attachment' -import core, { - AttachedData, - Blob, - Client, - Data, - MeasureContext, - Ref, - Timestamp, - TxCreateDoc, - TxCUD, - TxFactory, - TxOperations, - TxProcessor, - TxUpdateDoc, - WorkspaceUuid -} from '@hcengineering/core' -import gmail, { type Message, type NewMessage } from '@hcengineering/gmail' +import core, { AccountUuid, Client, MeasureContext, SocialId, TxOperations, WorkspaceUuid } from '@hcengineering/core' +import gmail, { type NewMessage } from '@hcengineering/gmail' import { type StorageAdapter } from '@hcengineering/server-core' import setting from '@hcengineering/setting' -import type { GaxiosResponse } from 'gaxios' import type { Credentials, OAuth2Client } from 'google-auth-library' import { gmail_v1, google } from 'googleapis' -import type { Collection, Db } from 'mongodb' -import { v4 as uuid } from 'uuid' -import { arrayBufferToBase64, decode64, encode64 } from './base64' +import { encode64 } from './base64' import config from './config' import { GmailController } from './gmailController' import { RateLimiter } from './rateLimiter' -import type { AttachedFile, Channel, History, ProjectCredentials, Token, User } from './types' -import { diffAttributes, isToken } from './utils' +import type { ProjectCredentials, Token, User } from './types' +import { addFooter, isToken, serviceToken, getKvsClient } from './utils' import type { WorkspaceClient } from './workspaceClient' +import { getOrCreateSocialId } from './accounts' +import { createIntegrationIfNotEsixts, disableIntegration, removeIntegration } from './integrations' +import { AttachmentHandler } from './message/attachments' +import { TokenStorage } from './tokens' +import { MessageManager } from './message/message' +import { SyncManager } from './message/sync' +import { getEmail } from './gmail/utils' +import { Integration } from '@hcengineering/account-client' const SCOPES = ['https://www.googleapis.com/auth/gmail.modify'] -const EMAIL_REGEX = - /(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))/ - function makeHTMLBody (message: NewMessage, from: string): string { const str = [ 'Content-Type: text/html; charset="UTF-8"\n', @@ -78,65 +64,6 @@ function makeHTMLBody (message: NewMessage, from: string): string { return encode64(res).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '') } -function addFooter (message: string): string { - if (config.FooterMessage === undefined || config.FooterMessage.trim() === '') return message - return message + config.FooterMessage.trim() -} - -function convertMessage ( - message: GaxiosResponse, - me: string -): AttachedData & { modifiedOn: Timestamp } { - const date = message.data.internalDate != null ? new Date(Number.parseInt(message.data.internalDate)) : new Date() - const from = getHeaderValue(message.data.payload, 'From') ?? '' - const to = getHeaderValue(message.data.payload, 'To') ?? '' - const copy = - getHeaderValue(message.data.payload, 'Cc') - ?.split(',') - .map((p) => p.trim()) ?? undefined - const incoming = !from.includes(me) - return { - modifiedOn: date.getTime(), - messageId: getHeaderValue(message.data.payload, 'Message-ID') ?? '', - replyTo: getHeaderValue(message.data.payload, 'In-Reply-To'), - copy, - content: getPartMessage(message.data.payload, 'text/html'), - textContent: getPartMessage(message.data.payload, 'text/plain').replace(/<[^>]*>/g, ''), - from, - to, - incoming, - subject: getHeaderValue(message.data.payload, 'Subject') ?? '', - sendOn: date.getTime() - } -} - -function getHeaderValue (payload: gmail_v1.Schema$MessagePart | undefined, name: string): string | undefined { - if (payload === undefined) return undefined - const headers = payload.headers - - return headers?.find((header) => header.name?.toLowerCase() === name.toLowerCase())?.value ?? undefined -} - -function getPartsMessage (parts: gmail_v1.Schema$MessagePart[] | undefined, mime: string): string { - let result = '' - if (parts !== undefined) { - const htmlPart = parts.find((part) => part.mimeType === mime) - const filtredParts = htmlPart !== undefined ? parts.filter((part) => part.mimeType === mime) : parts - for (const part of filtredParts ?? []) { - result += getPartMessage(part, mime) - } - } - return result -} - -function getPartMessage (part: gmail_v1.Schema$MessagePart | undefined, mime: string): string { - if (part === undefined) return '' - if (part.body?.data != null) { - return decode64(part.body.data) - } - return getPartsMessage(part.parts, mime) -} - async function wait (sec: number): Promise { await new Promise((resolve) => { setTimeout(() => { @@ -146,56 +73,116 @@ async function wait (sec: number): Promise { } export class GmailClient { - private readonly oAuth2Client: OAuth2Client - private readonly gmail: gmail_v1.Resource$Users - private me: string | undefined = undefined - private readonly tokens: Collection - private readonly histories: Collection + private readonly account: AccountUuid + private email: string | undefined = undefined + private readonly tokenStorage: TokenStorage private readonly client: TxOperations private watchTimer: NodeJS.Timeout | undefined = undefined private refreshTimer: NodeJS.Timeout | undefined = undefined - private syncPromise: Promise | undefined = undefined - private readonly rateLimiter = new RateLimiter(1000, 200) // in fact 250, but let's make reserve + private readonly rateLimiter = new RateLimiter(1000, 200) + private readonly attachmentHandler: AttachmentHandler + private readonly messageManager: MessageManager + private readonly syncManager: SyncManager + private readonly integrationToken: string + private integration: Integration | undefined = undefined private constructor ( private readonly ctx: MeasureContext, - credentials: ProjectCredentials, + private readonly oAuth2Client: OAuth2Client, + private readonly gmail: gmail_v1.Resource$Users, private readonly user: User, - mongo: Db, client: Client, - private readonly workspaceId: WorkspaceUuid, - private readonly storageAdapter: StorageAdapter, - private readonly workspace: WorkspaceClient + workspaceId: WorkspaceUuid, + storageAdapter: StorageAdapter, + private readonly workspace: WorkspaceClient, + email: string, + private socialId: SocialId ) { - const { client_secret, client_id, redirect_uris } = credentials.web // eslint-disable-line - this.oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]) // eslint-disable-line - this.gmail = google.gmail({ version: 'v1', auth: this.oAuth2Client }).users - this.tokens = mongo.collection('tokens') - this.histories = mongo.collection('histories') - this.client = new TxOperations(client, this.user.userId) + this.email = email + this.integrationToken = serviceToken() + this.tokenStorage = new TokenStorage(this.ctx, workspaceId, this.integrationToken) + this.client = new TxOperations(client, this.socialId._id) + this.account = this.user.userId + this.attachmentHandler = new AttachmentHandler(ctx, workspaceId, storageAdapter, this.gmail, this.client) + this.messageManager = new MessageManager( + ctx, + this.client, + this.attachmentHandler, + this.socialId._id, + this.workspace + ) + const keyValueClient = getKvsClient(this.integrationToken) + this.syncManager = new SyncManager( + ctx, + this.messageManager, + this.gmail, + this.user.workspace, + keyValueClient, + this.rateLimiter + ) } static async create ( ctx: MeasureContext, credentials: ProjectCredentials, user: User | Token, - mongo: Db, client: Client, workspace: WorkspaceClient, workspaceId: WorkspaceUuid, - storageAdapter: StorageAdapter + storageAdapter: StorageAdapter, + authCode?: string ): Promise { - const gmailClient = new GmailClient(ctx, credentials, user, mongo, client, workspaceId, storageAdapter, workspace) - if (isToken(user)) { - console.log('Setting token while creating', user.workspace, user.userId, user) - await gmailClient.setToken(user) + // eslint-disable-next-line @typescript-eslint/naming-convention + const { client_secret, client_id, redirect_uris } = credentials.web + const oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]) + const googleClient = google.gmail({ version: 'v1', auth: oAuth2Client }).users + + let token: Credentials | undefined = isToken(user) ? user : undefined + if (token === undefined && authCode !== undefined) { + const tokenResponse = await oAuth2Client.getToken(authCode) + token = { + ...user, + ...tokenResponse.tokens + } + oAuth2Client.setCredentials(token) + } else if (token !== undefined) { + oAuth2Client.setCredentials(token) + } + const email = user.email ?? (await getEmail(googleClient)) + if (email === undefined) { + throw new Error('Cannot retrieve user email') + } + const socialId = user.socialId ?? (await getOrCreateSocialId(user.userId, email)) + if (socialId?._id == null) { + throw new Error(`Cannot create gmail client without social id: ${user.userId}, ${workspaceId}`) + } + user.socialId = socialId + + const gmailClient = new GmailClient( + ctx, + oAuth2Client, + googleClient, + user, + client, + workspaceId, + storageAdapter, + workspace, + email, + socialId + ) + await gmailClient.createIntregration() + if (token !== undefined) { + ctx.info('Setting token while creating', { workspaceUuid: user.workspace, userId: user.userId }) + await gmailClient.setToken(token) await gmailClient.refreshToken() await gmailClient.addClient() } + ctx.info('Created gmail client', { workspaceUuid: user.workspace, userId: user.userId }) + void gmailClient.initIntegration() return gmailClient } - getAutUrl (redirectURL: string): string { + getAuthUrl (redirectURL: string): string { const { workspace, userId } = this.user const authUrl = this.oAuth2Client.generateAuthUrl({ access_type: 'offline', @@ -205,42 +192,54 @@ export class GmailClient { return authUrl } - async authorize (code: string): Promise { - const token = await this.oAuth2Client.getToken(code) - console.log('Setting token after authorize', this.user.workspace, this.user.userId, token) - await this.setToken(token.tokens) - await this.refreshToken() - await this.addClient() - void this.startSync() - void this.getNewMessagesAfterAuth() + async initIntegration (): Promise { + try { + this.ctx.info('Init integration', { workspaceUuid: this.user.workspace, userId: this.user.userId }) - const me = await this.getMe() - const integrations = await this.client.findAll(setting.class.Integration, { - createdBy: this.user.userId, - type: gmail.integrationType.Gmail - }) - const updated = integrations.find((p) => p.disabled && p.value === me) + void this.startSync() + void this.getNewMessagesAfterAuth() - for (const integration of integrations.filter((p) => p._id !== updated?._id)) { - await this.client.remove(integration) + const integrations = await this.client.findAll(setting.class.Integration, { + createdBy: this.socialId._id, + type: gmail.integrationType.Gmail + }) + const updated = integrations.find((p) => p.disabled && p.value === this.email) + + for (const integration of integrations.filter((p) => p._id !== updated?._id)) { + await this.client.remove(integration) + } + if (updated !== undefined) { + await this.client.update(updated, { + disabled: false + }) + } else { + await this.client.createDoc(setting.class.Integration, core.space.Workspace, { + type: gmail.integrationType.Gmail, + disabled: false, + value: this.socialId._id + }) + } + } catch (err: any) { + this.ctx.info('Failed to start message sync', { + workspaceUuid: this.user.workspace, + userId: this.user.userId, + error: err.message + }) } - if (updated !== undefined) { - await this.client.update(updated, { - disabled: false - }) - } else { - await this.client.createDoc(setting.class.Integration, core.space.Workspace, { - type: gmail.integrationType.Gmail, - disabled: false, - value: me - }) + } + + async createIntregration (): Promise { + try { + this.integration = await createIntegrationIfNotEsixts(this.socialId._id, this.user.workspace) + } catch (err: any) { + this.ctx.error('Failed to create integration', { socialdId: this.socialId, workspace: this.workspace }) } } async getNewMessagesAfterAuth (): Promise { const newMessages = await this.client.findAll(gmail.class.NewMessage, { status: 'new', - from: this.user.userId + from: this.socialId._id }) await this.workspace.subscribeMessages() for (const message of newMessages) { @@ -251,11 +250,12 @@ export class GmailClient { async createMessage (message: NewMessage): Promise { if (message.status === 'sent') return try { - const me = await this.getMe() + this.ctx.info('Send gmail message', { id: message._id, from: message.from, to: message.to }) + const email = await this.getEmail() const body = message.attachments != null && message.attachments > 0 - ? await this.makeAttachmentsBody(message, me) - : makeHTMLBody(message, me) + ? await this.makeAttachmentsBody(message, email) + : makeHTMLBody(message, email) await this.rateLimiter.take(100) await this.gmail.messages.send({ userId: 'me', @@ -272,37 +272,31 @@ export class GmailClient { status: 'error', error: JSON.stringify(err) }) - console.log('Create message error', this.user.workspace, this.user.userId, err.message) + this.ctx.error('Create message error', { + workspaceUuid: this.user.workspace, + userId: this.user.userId, + message: err.message + }) if (err?.response?.data?.error === 'invalid_grant') { await this.refreshToken() } } } - private async checkErrorWithSignout (err: any): Promise { - if (this.checkError(err)) { - console.log('Invalid grant error', this.user.workspace, this.user.userId) - await this.workspace.signoutByUserId(this.user.userId, true) - return true - } - return false - } - private checkError (err: any): boolean { return err?.response?.data?.error === 'invalid_grant' } async signout (byError: boolean = false): Promise { + this.ctx.info('Deactivate gmail client', { socialId: this.socialId, workspace: this.user.workspace }) try { await this.close() await this.oAuth2Client.revokeCredentials() } catch {} - await this.tokens.deleteOne({ - userId: this.user.userId, - workspace: this.user.workspace - }) + await this.tokenStorage.deleteToken(this.socialId._id) + await this.disableIntegration(byError) const integration = await this.client.findOne(setting.class.Integration, { - createdBy: this.user.userId, + createdBy: this.socialId._id, type: gmail.integrationType.Gmail }) if (integration !== undefined) { @@ -314,26 +308,19 @@ export class GmailClient { } } - async sync (): Promise { - if (this.syncPromise !== undefined) { - await this.syncPromise - } - const history = await this.getHistory() - try { - if (history != null) { - this.syncPromise = this.partSync(history.historyId) - await this.syncPromise + async disableIntegration (byError: boolean = false): Promise { + if (this.integration !== undefined) { + if (byError) { + await disableIntegration(this.integration) } else { - this.syncPromise = this.fullSync() - await this.syncPromise + await removeIntegration(this.socialId._id, this.user.workspace) } - } finally { - this.syncPromise = undefined } } async startSync (): Promise { - await this.sync() + this.ctx.info('Start sync', { workspaceUuid: this.user.workspace, userId: this.user.userId, email: this.email }) + await this.syncManager.sync(this.socialId._id, this.email) await this.watch() // recall every 24 hours https://developers.google.com/gmail/api/guides/push this.watchTimer = setInterval(() => { @@ -341,64 +328,72 @@ export class GmailClient { }, 86400000) } + async sync (): Promise { + this.ctx.info('Sync', { workspaceUuid: this.user.workspace, userId: this.user.userId }) + await this.syncManager.sync(this.socialId._id, this.email) + } + async newChannel (value: string): Promise { const query = `{from:${value} to:${value} cc:${value} bcc:${value}}` await this.fullSync(query) } - private async getMe (): Promise { - if (this.me !== undefined) { - return this.me + private async getEmail (): Promise { + if (this.email !== undefined) { + return this.email } for (let i = 0; i < 3; i++) { await this.rateLimiter.take(1) const profile = await this.gmail.getProfile({ userId: 'me' }) - this.me = profile.data.emailAddress ?? undefined - if (this.me !== undefined) return this.me + this.email = profile.data.emailAddress ?? undefined + if (this.email !== undefined) return this.email await wait(5) } throw new Error("Can't get profile email after 3 attempts") } - private async getCurrentToken (): Promise { - return await this.tokens.findOne({ - userId: this.user.userId, - workspace: this.user.workspace - }) + private async syncUserInfo (credentials: Credentials): Promise { + if (credentials.access_token == null) return + if (this.email === undefined) { + this.ctx.error('User email is missing', { workspaceUuid: this.user.workspace, userId: this.user.userId }) + return + } + this.socialId = await getOrCreateSocialId(this.account, this.email) } - private async updateCurrentToken (token: Credentials): Promise { - await this.tokens.updateOne( - { - userId: this.user.userId, - workspace: this.user.workspace - }, - { - $set: { - ...token - } - } - ) + private async getCurrentToken (): Promise { + return await this.tokenStorage.getToken(this.socialId._id) } private async addClient (): Promise { try { - const me = await this.getMe() + this.ctx.info('Register client', { socialId: this.socialId._id, email: this.email }) const controller = GmailController.getGmailController() - controller.addClient(me, this) + controller.addClient(this.socialId._id, this) } catch (err) { - console.log('Add client error', this.user.workspace, this.user.userId, (err as any).message) + this.ctx.error('Add client error', { + workspaceUuid: this.user.workspace, + userId: this.user.userId, + message: (err as any).message + }) } } private async setToken (token: Credentials): Promise { try { this.oAuth2Client.setCredentials(token) - await this.getMe() + const email = await this.getEmail() + await this.syncUserInfo(token) + await this.updateSocialId(email) + await this.updateToken(token) } catch (err: any) { - console.log('Set token error', this.user.workspace, this.user.userId, err.message) + this.ctx.error('Set token error', { + workspaceUuid: this.user.workspace, + userId: this.user.userId, + message: err.message + }) if (this.checkError(err)) { await this.signout(true) } @@ -406,338 +401,31 @@ export class GmailClient { } } + private async updateSocialId (email: string): Promise { + const existingEmail = this.socialId !== undefined ? this.socialId.value : undefined + if (existingEmail === email) return + this.socialId = await getOrCreateSocialId(this.user.userId, email) + } + private async updateToken (token: Credentials): Promise { try { - const currentToken = await this.getCurrentToken() - if (currentToken != null) { - await this.updateCurrentToken(token) - } else { - await this.tokens.insertOne({ - userId: this.user.userId, - workspace: this.user.workspace, - token: this.user.token, - ...token - }) - } + await this.tokenStorage.saveToken(this.socialId._id, token as Token) } catch (err) { - console.log('update token error', this.user.workspace, this.user.userId, (err as any).message) - } - } - - private async getHistory (): Promise { - return await this.histories.findOne({ - userId: this.user.userId, - workspace: this.user.workspace - }) - } - - private async clearHistory (): Promise { - await this.histories.deleteOne({ - userId: this.user.userId, - workspace: this.user.workspace - }) - } - - private async setHistoryId (historyId: string): Promise { - await this.histories.updateOne( - { + this.ctx.error('update token error', { + workspaceUuid: this.user.workspace, userId: this.user.userId, - workspace: this.user.workspace - }, - { - $set: { - historyId - } - }, - { upsert: true } - ) - } - - private async partSync (historyId: string): Promise { - let pageToken: string | undefined - let histories: GaxiosResponse - while (true) { - try { - await this.rateLimiter.take(2) - histories = await this.gmail.history.list({ - userId: 'me', - historyTypes: ['messageAdded'], - startHistoryId: historyId, - pageToken - }) - } catch (err) { - console.log('Part sync get history error', this.user.workspace, this.user.userId, err) - const signout = await this.checkErrorWithSignout(err) - if (signout) return - await this.clearHistory() - await this.sync() - return - } - const nextPageToken = histories.data.nextPageToken - const array = histories.data.history ?? [] - try { - for (const history of array) { - for (const message of history.messagesAdded ?? []) { - if (message.message?.id == null || (message.message.labelIds?.includes('DRAFT') ?? false)) { - continue - } - try { - const res = await this.getMessage(message.message.id) - await this.saveMessage(res) - } catch (err) { - console.log('Part sync message error', this.user.workspace, this.user.userId, message.message.id, err) - const signout = await this.checkErrorWithSignout(err) - if (signout) return - } - } - if (history.id != null) { - await this.setHistoryId(history.id) - } - } - if (nextPageToken == null) { - return - } else { - pageToken = nextPageToken - } - } catch (err) { - console.log('Part sync error', this.user.workspace, this.user.userId, err) - return - } + message: (err as any).message + }) } } private async fullSync (q?: string): Promise { - const pageToken: string | undefined = undefined - const query: gmail_v1.Params$Resource$Users$Messages$List = { - userId: 'me', - pageToken - } - if (q !== undefined) { - query.q = q - } - let currentHistoryId: string | undefined - console.log('start full sync', this.user.workspace, this.user.userId, q) - try { - const messagesIds: string[] = [] - while (true) { - await this.rateLimiter.take(5) - const messages = await this.gmail.messages.list(query) - if (query.pageToken == null) { - console.log('Total messages', this.user.workspace, this.user.userId, messages.data.resultSizeEstimate) - } - const ids = messages.data.messages?.map((p) => p.id) ?? [] - for (let index = 0; index < ids.length; index++) { - const id = ids[index] - if (id == null) continue - messagesIds.push(id) - } - if (messages.data.nextPageToken == null) { - console.log('Break, total new messages: ', messagesIds.length) - break - } - query.pageToken = messages.data.nextPageToken - } - for (let index = messagesIds.length - 1; index >= 0; index--) { - const id = messagesIds[index] - try { - const message = await this.getMessage(id) - const historyId = message.data.historyId - await this.saveMessage(message) - if (historyId != null && q === undefined) { - if (currentHistoryId == null || Number(currentHistoryId) < Number(historyId)) { - await this.setHistoryId(historyId) - currentHistoryId = historyId - } - } - } catch (err: any) { - console.log('Full sync message error', this.user.workspace, this.user.userId, err) - const signout = await this.checkErrorWithSignout(err) - if (signout) return - } - } - } catch (err) { - console.log('Full sync error', this.user.workspace, this.user.userId, err) - await this.checkErrorWithSignout(err) - } - } - - private async getMessage (id: string): Promise> { - await this.rateLimiter.take(5) - return await this.gmail.messages.get({ - id, - userId: 'me', - format: 'FULL' + this.ctx.info('Full sync client', { + workspaceUuid: this.user.workspace, + userId: this.user.userId, + email: this.email }) - } - - private findChannels (res: AttachedData): Channel[] { - const recievers = res.copy != null ? [res.to, ...res.copy] : [res.to] - const values = res.incoming ? [res.from] : recievers - const result: Channel[] = [] - for (const value of values) { - const res = value.match(EMAIL_REGEX) - if (res !== null) { - const email = res[0] - const channel = this.workspace.getChannel(email) - if (channel !== undefined) result.push(channel) - } - } - return result - } - - private async saveMessage (message: GaxiosResponse): Promise { - const me = await this.getMe() - const res = convertMessage(message, me) - const channels = this.findChannels(res) - if (channels.length === 0) return - const attachments = await this.getPartFiles(message.data.payload, message.data.id ?? '') - const factory = new TxFactory(this.user.userId) - for (const channel of channels) { - const current = await this.client.findOne(gmail.class.Message, { - messageId: res.messageId, - attachedTo: channel._id - }) - const tx = current != null ? this.updateTx(res, current, factory, channel) : this.createTx(res, factory, channel) - if (tx !== undefined) { - const resultMessage = - current != null - ? TxProcessor.updateDoc2Doc(current, tx as TxUpdateDoc) - : TxProcessor.createDoc2Doc(tx as TxCreateDoc) - await this.client.tx(tx) - if (attachments.length > 0) { - const currentAttachemtns: Attachment[] = - current !== undefined - ? await this.client.findAll(attachment.class.Attachment, { attachedTo: current._id }) - : [] - for (const attachment of attachments) { - await this.addAttachement(attachment, resultMessage, currentAttachemtns) - } - } - } - } - } - - private async addAttachement (file: AttachedFile, message: Message, currentAttachemtns: Attachment[]): Promise { - try { - if (currentAttachemtns.findIndex((p) => p.name === file.name && p.lastModified === file.lastModified) !== -1) { - return - } - const id = uuid() - const data: AttachedData = { - name: file.name, - file: id as Ref, - type: file.type ?? 'undefined', - size: file.size ?? Buffer.from(file.file, 'base64').length, - lastModified: file.lastModified - } - await this.storageAdapter.put(this.ctx, this.workspaceId as any, id, file.file, data.type, data.size) // TODO: FIXME - await this.client.addCollection( - attachment.class.Attachment, - message.space, - message._id, - message._class, - 'attachments', - data - ) - } catch (err: any) { - console.log('Add attachment error', this.user.workspace, this.user.userId, err.message) - } - } - - private async getPartFiles ( - part: gmail_v1.Schema$MessagePart | undefined, - messageId: string - ): Promise { - if (part === undefined) return [] - if (part.filename != null && part.filename.length > 0) { - if (part.body?.attachmentId != null) { - const attachment = await this.getAttachmentById(part.body?.attachmentId, messageId) - if (attachment.data == null) return [] - return [ - { - file: attachment.data, - name: part.filename, - type: part.mimeType ?? undefined, - size: attachment.size ?? undefined, - lastModified: new Date().getTime() - } - ] - } - if (part.body?.data == null) return [] - return [ - { - file: part.body.data, - name: part.filename, - type: part.mimeType ?? undefined, - size: part.body.size ?? undefined, - lastModified: new Date().getTime() - } - ] - } - return await this.getPartsFiles(part.parts, messageId) - } - - private async getAttachmentById (attachmentId: string, messageId: string): Promise { - await this.rateLimiter.take(5) - const res = await this.gmail.messages.attachments.get({ id: attachmentId, userId: 'me', messageId }) - return res.data - } - - private async getPartsFiles ( - parts: gmail_v1.Schema$MessagePart[] | undefined, - messageId: string - ): Promise { - const result: AttachedFile[] = [] - if (parts !== undefined) { - const filtredParts = parts.filter((part) => part.filename != null && part.filename.length > 0) - for (const part of filtredParts ?? []) { - const res = await this.getPartFiles(part, messageId) - if (res.length > 0) { - result.push(...res) - } - } - } - return result - } - - private createTx ( - message: AttachedData & { modifiedOn: Timestamp }, - factory: TxFactory, - channel: Channel - ): TxCUD { - const tx = factory.createTxCollectionCUD( - channel._class, - channel._id, - channel.space, - 'items', - factory.createTxCreateDoc( - gmail.class.Message, - core.space.Workspace, - message as unknown as Data, - undefined, - message.modifiedOn - ), - message.modifiedOn - ) - return tx - } - - private updateTx ( - message: AttachedData & { modifiedOn: Timestamp }, - current: Message, - factory: TxFactory, - channel: Channel - ): TxCUD | undefined { - const operations = diffAttributes(current, message) - if (Object.keys(operations).length === 0) return undefined - const tx = factory.createTxCollectionCUD( - channel._class, - channel._id, - channel.space, - 'items', - factory.createTxUpdateDoc(current._class, current.space, current._id, operations) - ) - return tx + await this.syncManager.fullSync(this.socialId._id, this.email, q) } private async refreshToken (): Promise { @@ -752,9 +440,13 @@ export class GmailClient { 30 * 60 * 1000 ) } catch (err: any) { - console.log("Couldn't refresh token, error:", err.message) + this.ctx.error("Couldn't refresh token", { + workspaceUuid: this.user.workspace, + userId: this.user.userId, + message: err.message + }) if (err?.response?.data?.error === 'invalid_grant') { - await this.workspace.signoutByUserId(this.user.userId, true) + await this.workspace.signoutBySocialId(this.socialId._id, true) } else { this.refreshTimer = setTimeout( () => { @@ -777,62 +469,21 @@ export class GmailClient { } }) } catch (err) { - console.log('Watch error', (err as any).message) + this.ctx.error('Watch error', { + workspaceUuid: this.user.workspace, + userId: this.user.userId, + message: (err as any).message + }) } } private async makeAttachmentsBody (message: NewMessage, from: string): Promise { - const str = [ - 'Content-Type: multipart/mixed; boundary="mail"\n', - 'MIME-Version: 1.0\n', - `To: ${message.to} \n`, - `From: ${from} \n` - ] - - if (message.replyTo != null) { - str.push(`In-Reply-To: ${message.replyTo} \n`) + try { + return await this.attachmentHandler.makeAttachmentsBody(message, from) + } catch (err: any) { + this.ctx.error('Failed to make attachment body', err) + return makeHTMLBody(message, from) } - - if (message.copy != null && message.copy.length > 0) { - str.push(`Cc: ${message.copy.join(', ')} \n`) - } - - if (message.subject != null) { - str.push(`Subject: =?UTF-8?B?${encode64(message.subject)}?= \n`) - } - - str.push('\n\n') - str.push('--mail\n') - str.push('Content-Type: text/html; charset="UTF-8"\n') - str.push('MIME-Version: 1.0\n') - str.push('Content-Transfer-Encoding: 7bit\n\n') - str.push(addFooter(message.content)) - str.push('\n\n') - - const attachments = await this.client.findAll(attachment.class.Attachment, { attachedTo: message._id }) - - for (const attachment of attachments) { - const attachmentStrings = await this.makeAttachmentPart(attachment) - str.push(...attachmentStrings) - } - - str.push('--mail--') - - return encode64(str.join('')).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '') - } - - private async makeAttachmentPart (attachment: Attachment): Promise { - const buffer = await this.storageAdapter.read(this.ctx, this.workspaceId as any, attachment.file) // TODO: FIXME - const data = arrayBufferToBase64(Buffer.concat(buffer)) - const res: string[] = [] - res.push('--mail\n') - res.push(`Content-Type: ${attachment.type}\n`) - res.push('MIME-Version: 1.0\n') - res.push('Content-Transfer-Encoding: base64\n') - res.push(`Content-Disposition: attachment; filename="${attachment.name}"\n\n`) - res.push(data) - res.push('\n\n') - return res } async close (): Promise { @@ -844,7 +495,11 @@ export class GmailClient { userId: 'me' }) } catch (err) { - console.log('close error', (err as any).message) + this.ctx.error('close error', { + workspaceUuid: this.user.workspace, + userId: this.user.userId, + message: (err as any).message + }) } } } diff --git a/services/gmail/pod-gmail/src/gmail/auth.ts b/services/gmail/pod-gmail/src/gmail/auth.ts new file mode 100644 index 0000000000..4b70e692ef --- /dev/null +++ b/services/gmail/pod-gmail/src/gmail/auth.ts @@ -0,0 +1,46 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { google } from 'googleapis' +import { OAuth2Client } from 'google-auth-library' +import { ProjectCredentials } from '../types' + +export class AuthProvider { + private oAuth2Client: OAuth2Client | undefined + private readonly credentials: ProjectCredentials + + constructor (credentials: ProjectCredentials) { + this.credentials = credentials + this.initClient() + } + + initClient (): void { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { client_secret, client_id, redirect_uris } = this.credentials.web + this.oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]) + } + + getAuthUrl (redirectURL: string, user: { workspace: string, userId: string }): string { + const { workspace, userId } = user + if (this.oAuth2Client === undefined) { + throw new Error('OAuth client not initialized') + } + const authUrl = this.oAuth2Client.generateAuthUrl({ + access_type: 'offline', + scope: ['https://www.googleapis.com/auth/gmail.modify'], + state: Buffer.from(JSON.stringify({ workspace, userId, redirectURL })).toString('base64') + }) + return authUrl + } +} diff --git a/services/gmail/pod-gmail/src/gmail/utils.ts b/services/gmail/pod-gmail/src/gmail/utils.ts new file mode 100644 index 0000000000..49e3361d7d --- /dev/null +++ b/services/gmail/pod-gmail/src/gmail/utils.ts @@ -0,0 +1,28 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { wait } from '../utils' +import { gmail_v1 } from 'googleapis' + +export async function getEmail (gmail: gmail_v1.Resource$Users): Promise { + for (let i = 0; i < 3; i++) { + const profile = await gmail.getProfile({ + userId: 'me' + }) + const email = profile.data.emailAddress ?? undefined + if (email !== undefined) return email + await wait(5) + } + throw new Error("Can't get profile email after 3 attempts") +} diff --git a/services/gmail/pod-gmail/src/gmailController.ts b/services/gmail/pod-gmail/src/gmailController.ts index e900f59612..6b39b3cf68 100644 --- a/services/gmail/pod-gmail/src/gmailController.ts +++ b/services/gmail/pod-gmail/src/gmailController.ts @@ -13,41 +13,52 @@ // limitations under the License. // -import { isActiveMode, MeasureContext, RateLimiter, systemAccountUuid, WorkspaceUuid } from '@hcengineering/core' +import { + AccountUuid, + isActiveMode, + MeasureContext, + RateLimiter, + WorkspaceUuid, + type PersonId +} from '@hcengineering/core' import type { StorageAdapter } from '@hcengineering/server-core' -import { type Db } from 'mongodb' import { decode64 } from './base64' import config from './config' import { type GmailClient } from './gmail' import { type ProjectCredentials, type Token, type User } from './types' import { WorkspaceClient } from './workspaceClient' -import { generateToken } from '@hcengineering/server-token' import { getAccountClient } from '@hcengineering/server-client' +import { getIntegrations } from './integrations' +import { serviceToken } from './utils' +import { getWorkspaceTokens } from './tokens' + +import { AuthProvider } from './gmail/auth' export class GmailController { private readonly workspaces: Map = new Map() private readonly credentials: ProjectCredentials - private readonly clients: Map = new Map() + private readonly clients: Map = new Map() private readonly initLimitter = new RateLimiter(config.InitLimit) + private readonly authProvider protected static _instance: GmailController private constructor ( private readonly ctx: MeasureContext, - private readonly mongo: Db, private readonly storageAdapter: StorageAdapter ) { this.credentials = JSON.parse(config.Credentials) + this.authProvider = new AuthProvider(this.credentials) GmailController._instance = this } - static create (ctx: MeasureContext, mongo: Db, storageAdapter: StorageAdapter): GmailController { + static create (ctx: MeasureContext, storageAdapter: StorageAdapter): GmailController { if (GmailController._instance !== undefined) { throw new Error('GmailController already exists') } - return new GmailController(ctx, mongo, storageAdapter) + return new GmailController(ctx, storageAdapter) } static getGmailController (): GmailController { @@ -58,45 +69,43 @@ export class GmailController { } async startAll (): Promise { - const tokens = await this.mongo.collection('tokens').find().toArray() - const groups = new Map() - console.log('start gmail service', tokens.length) - for (const token of tokens) { - const group = groups.get(token.workspace) - if (group === undefined) { - groups.set(token.workspace, [token]) - } else { - group.push(token) - groups.set(token.workspace, group) - } - } + try { + const token = serviceToken() + const integrations = await getIntegrations(token) + this.ctx.info('Start integrations', { count: integrations.length }) - const limiter = new RateLimiter(config.InitLimit) - for (const [workspace, tokens] of groups) { - await limiter.add(async () => { - const wstok = generateToken(systemAccountUuid, workspace as any) // TODO: FIXME - const accountClient = getAccountClient(wstok) - const info = await accountClient.getWorkspaceInfo() + const limiter = new RateLimiter(config.InitLimit) + for (const integration of integrations) { + if (integration.workspaceUuid === null) continue + await limiter.add(async () => { + if (integration.workspaceUuid === null) return + const accountClient = getAccountClient(token) + const info = await accountClient.getWorkspaceInfo() - if (info === undefined) { - console.log('workspace not found', workspace) - return - } - if (!isActiveMode(info.mode)) { - console.log('workspace is not active', workspace) - return - } - const startPromise = this.startWorkspace(workspace as any, tokens) // TODO: FIXME - const timeoutPromise = new Promise((resolve) => { - setTimeout(() => { - resolve() - }, 60000) + if (info === undefined) { + this.ctx.info('workspace not found', { workspaceUuid: integration.workspaceUuid }) + return + } + if (!isActiveMode(info.mode)) { + this.ctx.info('workspace is not active', { workspaceUuid: integration.workspaceUuid }) + return + } + const tokens = await getWorkspaceTokens(accountClient, integration.workspaceUuid) + this.ctx.info('Use stored tokens', { count: tokens.length }) + const startPromise = this.startWorkspace(integration.workspaceUuid, tokens) + const timeoutPromise = new Promise((resolve) => { + setTimeout(() => { + resolve() + }, 60000) + }) + await Promise.race([startPromise, timeoutPromise]) }) - await Promise.race([startPromise, timeoutPromise]) - }) - } + } - await limiter.waitProcessing() + await limiter.waitProcessing() + } catch (err: any) { + this.ctx.error('Failed to start existing integrations', err) + } } async startWorkspace (workspace: WorkspaceUuid, tokens: Token[]): Promise { @@ -105,15 +114,20 @@ export class GmailController { for (const token of tokens) { try { const timeout = setTimeout(() => { - console.log('init client hang', token.workspace, token.userId) + this.ctx.info('init client hang', { workspaceUuid: token.workspace, userId: token.userId }) }, 60000) const client = await workspaceClient.createGmailClient(token) clearTimeout(timeout) clients.push(client) - } catch (err) { - console.error(`Couldn't create client for ${workspace} ${token.userId}`) + } catch (err: any) { + this.ctx.error("Couldn't create client", { + workspaceUuid: workspace, + userId: token.userId, + message: err.message + }) } } + this.ctx.info('Start clients sync', { workspaceUuid: workspace, count: clients.length }) for (const client of clients) { void this.initLimitter.add(async () => { await client.startSync() @@ -133,25 +147,30 @@ export class GmailController { } } - addClient (email: string, client: GmailClient): void { - const clients = this.clients.get(email) + addClient (socialId: PersonId, client: GmailClient): void { + const clients = this.clients.get(socialId) if (clients === undefined) { - this.clients.set(email, [client]) + this.clients.set(socialId, [client]) } else { clients.push(client) - this.clients.set(email, clients) + this.clients.set(socialId, clients) } } - async getGmailClient (email: string, workspace: WorkspaceUuid, token: string): Promise { + /* + async getGmailClient (userId: AccountUuid, workspace: WorkspaceUuid, token: string): Promise { const workspaceClient = await this.getWorkspaceClient(workspace) - const userId = await workspaceClient.getUserId(email) return await workspaceClient.createGmailClient({ userId, workspace, token }) } + */ - async signout (workspace: WorkspaceUuid, email: string): Promise { + getAuthProvider (): AuthProvider { + return this.authProvider + } + + async signout (workspace: WorkspaceUuid, account: AccountUuid): Promise { const workspaceClient = await this.getWorkspaceClient(workspace) - const clients = await workspaceClient.signout(email) + const clients = await workspaceClient.signoutByAccountId(account) if (clients === 0) { this.workspaces.delete(workspace) } @@ -164,9 +183,9 @@ export class GmailController { this.workspaces.clear() } - async createClient (user: User | Token): Promise { + async createClient (user: User | Token, authCode?: string): Promise { const workspace = await this.getWorkspaceClient(user.workspace) - const newClient = await workspace.createGmailClient(user) + const newClient = await workspace.createGmailClient(user, authCode) return newClient } @@ -174,12 +193,12 @@ export class GmailController { let res = this.workspaces.get(workspace) if (res === undefined) { try { - console.log('create workspace worker for', workspace) - res = await WorkspaceClient.create(this.ctx, this.credentials, this.mongo, this.storageAdapter, workspace) + this.ctx.info('create workspace worker', { workspaceUuid: workspace }) + res = await WorkspaceClient.create(this.ctx, this.credentials, this.storageAdapter, workspace) this.workspaces.set(workspace, res) - console.log('created workspace worker for', workspace) + this.ctx.info('created workspace worker', { workspaceUuid: workspace }) } catch (err) { - console.error(`Couldn't create workspace worker for ${workspace}, reason: `, err) + this.ctx.error("Couldn't create workspace worker", { workspaceUuid: workspace, reason: err }) throw err } } diff --git a/services/gmail/pod-gmail/src/integrations.ts b/services/gmail/pod-gmail/src/integrations.ts new file mode 100644 index 0000000000..8dc0b92902 --- /dev/null +++ b/services/gmail/pod-gmail/src/integrations.ts @@ -0,0 +1,132 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { AccountUuid, MeasureContext, PersonId, TxOperations, WorkspaceUuid } from '@hcengineering/core' +import { getAccountClient } from '@hcengineering/server-client' +import { Integration } from '@hcengineering/account-client' +import gmail from '@hcengineering/gmail' +import setting from '@hcengineering/setting' + +import { serviceToken } from './utils' +import { GMAIL_INTEGRATION } from './types' +import { getAccountSocialIds } from './accounts' + +export async function getIntegration (socialId: PersonId, workspaceUuid?: WorkspaceUuid): Promise { + const client = getAccountClient(serviceToken()) + return await client.getIntegration({ kind: GMAIL_INTEGRATION, socialId, workspaceUuid: workspaceUuid ?? null }) +} + +export async function getIntegrations (token: string): Promise { + const client = getAccountClient(token) + return (await client.listIntegrations({ kind: GMAIL_INTEGRATION })) ?? [] +} + +export async function getIntegrationByAccount ( + account: AccountUuid, + workspaceUuid?: WorkspaceUuid +): Promise { + const token = serviceToken() + const client = getAccountClient(token) + const integrations = await client.listIntegrations({ kind: GMAIL_INTEGRATION, workspaceUuid: workspaceUuid ?? null }) + if (integrations.length === 0) return null + const socialIds = await getAccountSocialIds(account) + + for (const integration of integrations) { + if (integration.workspaceUuid == null) continue + const socialId = socialIds.find((it) => it._id === integration.socialId) + if (socialId !== undefined) { + return integration + } + } + + return null +} + +export async function removeIntegration ( + socialId: PersonId | undefined | null, + workspaceUuid: WorkspaceUuid +): Promise { + if (socialId == null) return + const accountClient = getAccountClient(serviceToken()) + const integrations = await accountClient.listIntegrations({ kind: GMAIL_INTEGRATION, socialId, workspaceUuid }) + for (const integration of integrations) { + await accountClient.deleteIntegration({ + socialId, + kind: GMAIL_INTEGRATION, + workspaceUuid: integration.workspaceUuid + }) + } +} + +export async function deleteIntegration (integration: Integration): Promise { + const accountClient = getAccountClient(serviceToken()) + await accountClient.deleteIntegration(integration) +} + +export async function createIntegrationIfNotEsixts (socialId: PersonId, workspace: WorkspaceUuid): Promise { + const accountClient = getAccountClient(serviceToken()) + const integration = { + socialId, + kind: GMAIL_INTEGRATION, + workspaceUuid: workspace + } + const existingIntegration = await accountClient.getIntegration(integration) + if (existingIntegration != null) return existingIntegration + + await accountClient.createIntegration(integration) + return integration +} + +export async function disableIntegration (integration: Integration): Promise { + const client = getAccountClient(serviceToken()) + await client.updateIntegration({ + ...integration, + data: { + disabled: true + } + }) +} + +export async function enableIntegration (integration: Integration): Promise { + const client = getAccountClient(serviceToken()) + await client.updateIntegration({ + ...integration, + data: { + disabled: false + } + }) +} + +export async function cleanIntegrations ( + ctx: MeasureContext, + client: TxOperations, + userId: AccountUuid, + workspace: WorkspaceUuid +): Promise { + const integration = await getIntegrationByAccount(userId, workspace) + if (integration != null) { + await deleteIntegration(integration) + ctx.info('Deleted integration', { userId, workspace }) + } + const personIds = ((await getAccountSocialIds(userId)) ?? []).map((id) => id._id) + const integrationSetting = await client.findOne(setting.class.Integration, { + createdBy: { $in: personIds }, + type: gmail.integrationType.Gmail + }) + if (integrationSetting != null) { + await client.remove(integrationSetting) + ctx.info('Deleted integration setting', { userId, workspace }) + } +} diff --git a/services/gmail/pod-gmail/src/main.ts b/services/gmail/pod-gmail/src/main.ts index aae9f66128..7da8a39086 100644 --- a/services/gmail/pod-gmail/src/main.ts +++ b/services/gmail/pod-gmail/src/main.ts @@ -18,17 +18,17 @@ import { SplitLogger } from '@hcengineering/analytics-service' import { MeasureMetricsContext, newMetrics } from '@hcengineering/core' import { setMetadata } from '@hcengineering/platform' -import serverClient from '@hcengineering/server-client' +import serverClient, { getAccountClient } from '@hcengineering/server-client' +import { isWorkspaceLoginInfo } from '@hcengineering/account-client' import { initStatisticsContext, type StorageConfiguration } from '@hcengineering/server-core' import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' -import serverToken from '@hcengineering/server-token' +import serverToken, { decodeToken } from '@hcengineering/server-token' import { type IncomingHttpHeaders } from 'http' import { join } from 'path' import { decode64 } from './base64' import config from './config' import { GmailController } from './gmailController' import { createServer, listen } from './server' -import { closeDB, getDB } from './storage' import { type Endpoint, type State } from './types' const extractToken = (header: IncomingHttpHeaders): any => { @@ -61,8 +61,7 @@ export const main = async (): Promise => { const storageConfig: StorageConfiguration = storageConfigFromEnv() const storageAdapter = buildStorageFromConfig(storageConfig) - const db = await getDB() - const gmailController = GmailController.create(ctx, db, storageAdapter) + const gmailController = GmailController.create(ctx, storageAdapter) await gmailController.startAll() const endpoints: Endpoint[] = [ { @@ -70,22 +69,30 @@ export const main = async (): Promise => { type: 'get', handler: async (req, res) => { try { - // TODO: FIXME - throw new Error('Not implemented') - // const token = extractToken(req.headers) + ctx.info('Signin request received') + const token = extractToken(req.headers) - // if (token === undefined) { - // res.status(401).send() - // return - // } - // const redirectURL = req.query.redirectURL as string + if (token === undefined) { + res.status(401).send() + return + } + const redirectURL = req.query.redirectURL as string - // const { workspace } = decodeToken(token) - // const gmail = await gmailController.getGmailClient(email, workspace, token) - // const url = gmail.getAutUrl(redirectURL) - // res.send(url) + const accountClient = getAccountClient(token) + const wsLoginInfo = await accountClient.getLoginInfoByToken() + if (!isWorkspaceLoginInfo(wsLoginInfo)) { + res.status(400).send({ err: "Couldn't find workspace with the provided token" }) + return + } + + const authProvider = gmailController.getAuthProvider() + const url = authProvider.getAuthUrl(redirectURL, { + workspace: wsLoginInfo.workspace, + userId: wsLoginInfo.account + }) + res.send(url) } catch (err) { - console.log('signin error', (err as any).message) + ctx.error('signin error', { message: (err as any).message }) res.status(500).send() } } @@ -94,31 +101,36 @@ export const main = async (): Promise => { endpoint: '/signin/code', type: 'get', handler: async (req, res) => { - const code = req.query.code as string - const state = JSON.parse(decode64(req.query.state as string)) as unknown as State - const gmail = await gmailController.createClient(state) - await gmail.authorize(code) - res.redirect(state.redirectURL) + try { + ctx.info('Signin code request received') + const code = req.query.code as string + const state = JSON.parse(decode64(req.query.state as string)) as unknown as State + await gmailController.createClient(state, code) + res.redirect(state.redirectURL) + } catch (err) { + ctx.error('Failed to process signin code', { message: (err as any).message }) + res.status(500).send() + } } }, { endpoint: '/signout', type: 'get', handler: async (req, res) => { + ctx.info('Signout request received') try { - // TODO: FIXME - throw new Error('Not implemented') - // const token = extractToken(req.headers) + const token = extractToken(req.headers) - // if (token === undefined) { - // res.status(401).send() - // return - // } + if (token === undefined) { + res.status(401).send() + return + } - // const { email, workspace } = decodeToken(token) - // await gmailController.signout(workspace.name, email) + const { account, workspace } = decodeToken(token) + + await gmailController.signout(workspace, account) } catch (err) { - console.log('signout error', JSON.stringify(err)) + ctx.error('signout error', { message: JSON.stringify(err) }) } res.send() @@ -128,14 +140,19 @@ export const main = async (): Promise => { endpoint: '/push', type: 'post', handler: async (req, res) => { - const data = req.body?.message?.data - if (data === undefined) { - res.status(400).send({ err: "'data' is missing" }) - return - } - gmailController.push(data) + try { + const data = req.body?.message?.data + if (data === undefined) { + res.status(400).send({ err: "'data' is missing" }) + return + } + gmailController.push(data) - res.send() + res.send() + } catch (err) { + ctx.error('Push request failed', { message: JSON.stringify(err) }) + res.status(500).send() + } } } ] @@ -145,7 +162,6 @@ export const main = async (): Promise => { const asyncClose = async (): Promise => { await gmailController.close() await storageAdapter.close() - await closeDB() } const shutdown = (): void => { diff --git a/services/gmail/pod-gmail/src/message/attachments.ts b/services/gmail/pod-gmail/src/message/attachments.ts new file mode 100644 index 0000000000..6cf3c325f4 --- /dev/null +++ b/services/gmail/pod-gmail/src/message/attachments.ts @@ -0,0 +1,165 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import attachment, { Attachment } from '@hcengineering/attachment' +import { AttachedData, Blob, MeasureContext, Ref, TxOperations, WorkspaceUuid } from '@hcengineering/core' +import { StorageAdapter } from '@hcengineering/server-core' +import { gmail_v1 } from 'googleapis' +import { v4 as uuid } from 'uuid' +import { encode64 } from '../base64' +import type { AttachedFile } from '../types' +import { addFooter } from '../utils' + +export class AttachmentHandler { + constructor ( + private readonly ctx: MeasureContext, + private readonly workspaceId: WorkspaceUuid, + private readonly storageAdapter: StorageAdapter, + private readonly gmail: gmail_v1.Resource$Users, + private readonly client: TxOperations + ) {} + + async addAttachement (file: AttachedFile, message: any, currentAttachemtns: Attachment[]): Promise { + try { + if (currentAttachemtns.findIndex((p) => p.name === file.name && p.lastModified === file.lastModified) !== -1) { + return + } + const id = uuid() + const data: AttachedData = { + name: file.name, + file: id as Ref, + type: file.type ?? 'undefined', + size: file.size ?? Buffer.from(file.file, 'base64').length, + lastModified: file.lastModified + } + await this.storageAdapter.put(this.ctx, this.workspaceId as any, id, file.file, data.type, data.size) // TODO: FIXME + await this.client.addCollection( + attachment.class.Attachment, + message.space, + message._id, + message._class, + 'attachments', + data + ) + } catch (err: any) { + this.ctx.error('Add attachment error', { error: err.message }) + } + } + + async getPartFiles (part: gmail_v1.Schema$MessagePart | undefined, messageId: string): Promise { + if (part === undefined) return [] + if (part.filename != null && part.filename.length > 0) { + if (part.body?.attachmentId != null) { + const attachment = await this.getAttachmentById(part.body?.attachmentId, messageId) + if (attachment.data == null) return [] + return [ + { + file: attachment.data, + name: part.filename, + type: part.mimeType ?? undefined, + size: attachment.size ?? undefined, + lastModified: new Date().getTime() + } + ] + } + if (part.body?.data == null) return [] + return [ + { + file: part.body.data, + name: part.filename, + type: part.mimeType ?? undefined, + size: part.body.size ?? undefined, + lastModified: new Date().getTime() + } + ] + } + return await this.getPartsFiles(part.parts, messageId) + } + + private async getAttachmentById (attachmentId: string, messageId: string): Promise { + const res = await this.gmail.messages.attachments.get({ id: attachmentId, userId: 'me', messageId }) + return res.data + } + + private async getPartsFiles ( + parts: gmail_v1.Schema$MessagePart[] | undefined, + messageId: string + ): Promise { + const result: AttachedFile[] = [] + if (parts !== undefined) { + const filtredParts = parts.filter((part) => part.filename != null && part.filename.length > 0) + for (const part of filtredParts ?? []) { + const res = await this.getPartFiles(part, messageId) + if (res.length > 0) { + result.push(...res) + } + } + } + return result + } + + async makeAttachmentsBody (message: any, from: string): Promise { + const str = [ + 'Content-Type: multipart/mixed; boundary="mail"\n', + 'MIME-Version: 1.0\n', + `To: ${message.to} \n`, + `From: ${from} \n` + ] + + if (message.replyTo != null) { + str.push(`In-Reply-To: ${message.replyTo} \n`) + } + + if (message.copy != null && message.copy.length > 0) { + str.push(`Cc: ${message.copy.join(', ')} \n`) + } + + if (message.subject != null) { + str.push(`Subject: =?UTF-8?B?${encode64(message.subject)}?= \n`) + } + + str.push('\n\n') + str.push('--mail\n') + str.push('Content-Type: text/html; charset="UTF-8"\n') + str.push('MIME-Version: 1.0\n') + str.push('Content-Transfer-Encoding: 7bit\n\n') + str.push(addFooter(message.content)) + str.push('\n\n') + + const attachments = await this.client.findAll(attachment.class.Attachment, { attachedTo: message._id }) + + for (const attachment of attachments) { + const attachmentStrings = await this.makeAttachmentPart(attachment) + str.push(...attachmentStrings) + } + + str.push('--mail--') + + return encode64(str.join('')).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '') + } + + private async makeAttachmentPart (attachment: Attachment): Promise { + const buffer = await this.storageAdapter.read(this.ctx, this.workspaceId as any, attachment.file) // TODO: FIXME + const data = Buffer.concat(buffer.map((b) => new Uint8Array(b))).toString('base64') + const res: string[] = [] + res.push('--mail\n') + res.push(`Content-Type: ${attachment.type}\n`) + res.push('MIME-Version: 1.0\n') + res.push('Content-Transfer-Encoding: base64\n') + res.push(`Content-Disposition: attachment; filename="${attachment.name}"\n\n`) + res.push(data) + res.push('\n\n') + return res + } +} diff --git a/services/gmail/pod-gmail/src/message/message.ts b/services/gmail/pod-gmail/src/message/message.ts new file mode 100644 index 0000000000..87b7e7187a --- /dev/null +++ b/services/gmail/pod-gmail/src/message/message.ts @@ -0,0 +1,201 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { + type MeasureContext, + PersonId, + Timestamp, + TxCUD, + TxFactory, + TxOperations, + TxProcessor, + TxCreateDoc, + TxUpdateDoc, + AttachedData, + Data +} from '@hcengineering/core' +import gmail, { type Message } from '@hcengineering/gmail' +import { type GaxiosResponse } from 'gaxios' +import { gmail_v1 } from 'googleapis' +import core from '@hcengineering/core' +import attachment, { Attachment } from '@hcengineering/attachment' +import sanitizeHtml from 'sanitize-html' + +import { type Channel } from '../types' +import { AttachmentHandler } from './attachments' +import { decode64 } from '../base64' +import { diffAttributes } from '../utils' + +const EMAIL_REGEX = + /(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))/ + +export class MessageManager { + constructor ( + private readonly ctx: MeasureContext, + private readonly client: TxOperations, + private readonly attachmentHandler: AttachmentHandler, + private readonly socialId: PersonId, + private readonly workspace: { getChannel: (email: string) => Channel | undefined } + ) {} + + private findChannels (res: AttachedData): Channel[] { + const recievers = res.copy != null ? [res.to, ...res.copy] : [res.to] + const values = res.incoming ? [res.from] : recievers + const result: Channel[] = [] + for (const value of values) { + const res = value.match(EMAIL_REGEX) + if (res !== null) { + const email = res[0] + const channel = this.workspace.getChannel(email) + if (channel !== undefined) result.push(channel) + } + } + return result + } + + async saveMessage (message: GaxiosResponse, me: string): Promise { + const res = convertMessage(message, me) + const channels = this.findChannels(res) + if (channels.length === 0) return + const attachments = await this.attachmentHandler.getPartFiles(message.data.payload, message.data.id ?? '') + const factory = new TxFactory(this.socialId) + for (const channel of channels) { + const current = await this.client.findOne(gmail.class.Message, { + messageId: res.messageId, + attachedTo: channel._id + }) + const tx = current != null ? this.updateTx(res, current, factory, channel) : this.createTx(res, factory, channel) + if (tx !== undefined) { + const resultMessage = + current != null + ? TxProcessor.updateDoc2Doc(current, tx as TxUpdateDoc) + : TxProcessor.createDoc2Doc(tx as TxCreateDoc) + await this.client.tx(tx) + if (attachments.length > 0) { + const currentAttachemtns: Attachment[] = + current !== undefined + ? await this.client.findAll(attachment.class.Attachment, { attachedTo: current._id }) + : [] + for (const attachment of attachments) { + await this.attachmentHandler.addAttachement(attachment, resultMessage, currentAttachemtns) + } + } + } + } + } + + private createTx ( + message: AttachedData & { modifiedOn: Timestamp }, + factory: TxFactory, + channel: Channel + ): TxCUD { + const tx = factory.createTxCollectionCUD( + channel._class, + channel._id, + channel.space, + 'items', + factory.createTxCreateDoc( + gmail.class.Message, + core.space.Workspace, + message as unknown as Data, + undefined, + message.modifiedOn + ), + message.modifiedOn + ) + return tx + } + + private updateTx ( + message: AttachedData & { modifiedOn: Timestamp }, + current: Message, + factory: TxFactory, + channel: Channel + ): TxCUD | undefined { + const operations = diffAttributes(current, message) + if (Object.keys(operations).length === 0) return undefined + const tx = factory.createTxCollectionCUD( + channel._class, + channel._id, + channel.space, + 'items', + factory.createTxUpdateDoc(current._class, current.space, current._id, operations) + ) + return tx + } +} + +function getHeaderValue (payload: gmail_v1.Schema$MessagePart | undefined, name: string): string | undefined { + if (payload === undefined) return undefined + const headers = payload.headers + + return headers?.find((header) => header.name?.toLowerCase() === name.toLowerCase())?.value ?? undefined +} + +function getPartsMessage (parts: gmail_v1.Schema$MessagePart[] | undefined, mime: string): string { + let result = '' + if (parts !== undefined) { + const htmlPart = parts.find((part) => part.mimeType === mime) + const filtredParts = htmlPart !== undefined ? parts.filter((part) => part.mimeType === mime) : parts + for (const part of filtredParts ?? []) { + result += getPartMessage(part, mime) + } + } + return result +} + +const sanitizeOptions: sanitizeHtml.IOptions = { + allowedTags: [], + allowedAttributes: {} +} + +export function sanitizeText (input: string): string { + if (input == null) return '' + return sanitizeHtml(input, sanitizeOptions) +} + +function getPartMessage (part: gmail_v1.Schema$MessagePart | undefined, mime: string): string { + if (part === undefined) return '' + if (part.body?.data != null) { + return decode64(part.body.data) + } + return getPartsMessage(part.parts, mime) +} + +function convertMessage ( + message: GaxiosResponse, + me: string +): AttachedData & { modifiedOn: Timestamp } { + const date = message.data.internalDate != null ? new Date(Number.parseInt(message.data.internalDate)) : new Date() + const from = getHeaderValue(message.data.payload, 'From') ?? '' + const to = getHeaderValue(message.data.payload, 'To') ?? '' + const copy = + getHeaderValue(message.data.payload, 'Cc') + ?.split(',') + .map((p) => p.trim()) ?? undefined + const incoming = !from.includes(me) + return { + modifiedOn: date.getTime(), + messageId: getHeaderValue(message.data.payload, 'Message-ID') ?? '', + replyTo: getHeaderValue(message.data.payload, 'In-Reply-To'), + copy, + content: sanitizeHtml(getPartMessage(message.data.payload, 'text/html')), + textContent: sanitizeText(getPartMessage(message.data.payload, 'text/plain')), + from, + to, + incoming, + subject: getHeaderValue(message.data.payload, 'Subject') ?? '', + sendOn: date.getTime() + } +} diff --git a/services/gmail/pod-gmail/src/message/sync.ts b/services/gmail/pod-gmail/src/message/sync.ts new file mode 100644 index 0000000000..d3a26afae4 --- /dev/null +++ b/services/gmail/pod-gmail/src/message/sync.ts @@ -0,0 +1,246 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { type GaxiosResponse } from 'gaxios' +import { gmail_v1 } from 'googleapis' + +import { type MeasureContext, PersonId } from '@hcengineering/core' +import { type KeyValueClient } from '@hcengineering/kvs-client' + +import { RateLimiter } from '../rateLimiter' +import { MessageManager } from './message' + +interface History { + historyId: string + userId: string + workspace: string +} + +export class SyncMutex { + private readonly locks = new Map>() + + async lock (key: string): Promise<() => void> { + // Wait for any existing lock to be released + const currentLock = this.locks.get(key) + if (currentLock != null) { + await currentLock + } + + // Create a new lock + let releaseFn!: () => void + const newLock = new Promise((resolve) => { + releaseFn = resolve + }) + + // Store the lock + this.locks.set(key, newLock) + + // Return the release function + return () => { + if (this.locks.get(key) === newLock) { + this.locks.delete(key) + } + releaseFn() + } + } +} + +export class SyncManager { + private readonly syncMutex = new SyncMutex() + + constructor ( + private readonly ctx: MeasureContext, + private readonly messageManager: MessageManager, + private readonly gmail: gmail_v1.Resource$Users, + private readonly workspace: string, + private readonly keyValueClient: KeyValueClient, + private readonly rateLimiter: RateLimiter + ) {} + + private async getHistory (userId: PersonId): Promise { + const historyKey = this.getHistoryKey(userId) + return await this.keyValueClient.getValue(historyKey) + } + + private async clearHistory (userId: PersonId): Promise { + const historyKey = this.getHistoryKey(userId) + await this.keyValueClient.deleteKey(historyKey) + } + + private async setHistoryId (userId: PersonId, historyId: string): Promise { + const historyKey = this.getHistoryKey(userId) + const history: History = { + historyId, + userId, + workspace: this.workspace + } + await this.keyValueClient.setValue(historyKey, history) + } + + private async partSync (userId: PersonId, userEmail: string | undefined, historyId: string): Promise { + if (userEmail === undefined) { + throw new Error('Cannot sync without user email') + } + let pageToken: string | undefined + let histories: GaxiosResponse + while (true) { + try { + await this.rateLimiter.take(2) + histories = await this.gmail.history.list({ + userId: 'me', + historyTypes: ['messageAdded'], + startHistoryId: historyId, + pageToken + }) + } catch (err: any) { + this.ctx.error('Part sync get history error', { workspaceUuid: this.workspace, userId, error: err.message }) + await this.clearHistory(userId) + void this.sync(userId) + return + } + const nextPageToken = histories.data.nextPageToken + const array = histories.data.history ?? [] + this.ctx.info('Messages to migrate', { count: array.length }) + try { + for (const history of array) { + for (const message of history.messagesAdded ?? []) { + if (message.message?.id == null || (message.message.labelIds?.includes('DRAFT') ?? false)) { + continue + } + try { + const res = await this.getMessage(message.message.id) + await this.messageManager.saveMessage(res, userEmail) + } catch (err) { + this.ctx.error('Part sync message error', { + workspaceUuid: this.workspace, + userId, + messageId: message.message.id, + err + }) + } + } + if (history.id != null) { + await this.setHistoryId(userId, history.id) + } + } + if (nextPageToken == null) { + return + } else { + pageToken = nextPageToken + } + } catch (err) { + this.ctx.error('Part sync error', { workspaceUuid: this.workspace, userId, historyId, err }) + return + } + } + } + + async fullSync (userId: PersonId, userEmail?: string, q?: string): Promise { + this.ctx.info('Start full sync', { workspaceUuid: this.workspace, userId, userEmail }) + if (userEmail === undefined) { + throw new Error('Cannot sync without user email') + } + const pageToken: string | undefined = undefined + const query: gmail_v1.Params$Resource$Users$Messages$List = { + userId: 'me', + pageToken + } + if (q !== undefined) { + query.q = q + } + let currentHistoryId: string | undefined + try { + const messagesIds: string[] = [] + while (true) { + await this.rateLimiter.take(5) + const messages = await this.gmail.messages.list(query) + if (query.pageToken == null) { + this.ctx.info('Total messages', { + workspace: this.workspace, + userId, + resultSizeEstimate: messages.data.resultSizeEstimate + }) + } + const ids = messages.data.messages?.map((p) => p.id) ?? [] + for (let index = 0; index < ids.length; index++) { + const id = ids[index] + if (id == null) continue + messagesIds.push(id) + } + if (messages.data.nextPageToken == null) { + this.ctx.info('Break', { totalNewMessages: messagesIds.length }) + break + } + query.pageToken = messages.data.nextPageToken + } + for (let index = messagesIds.length - 1; index >= 0; index--) { + const id = messagesIds[index] + try { + const message = await this.getMessage(id) + const historyId = message.data.historyId + await this.messageManager.saveMessage(message, userEmail) + if (historyId != null && q === undefined) { + if (currentHistoryId == null || Number(currentHistoryId) < Number(historyId)) { + await this.setHistoryId(userId, historyId) + currentHistoryId = historyId + } + } + if (index % 500 === 0) { + this.ctx.info('Remaining messages to sync', { workspace: this.workspace, userId, count: index }) + } + } catch (err: any) { + this.ctx.error('Full sync message error', { workspace: this.workspace, userId, err }) + } + } + this.ctx.info('Full sync finished', { workspaceUuid: this.workspace, userId, userEmail }) + } catch (err) { + this.ctx.error('Full sync error', { workspace: this.workspace, userId, err }) + } + } + + private async getMessage (id: string): Promise> { + await this.rateLimiter.take(5) + return await this.gmail.messages.get({ + id, + userId: 'me', + format: 'FULL' + }) + } + + async sync (userId: PersonId, userEmail?: string): Promise { + const mutexKey = `${this.workspace}:${userId}` + const releaseLock = await this.syncMutex.lock(mutexKey) + + try { + this.ctx.info('Sync history', { workspaceUuid: this.workspace, userId, userEmail }) + + const history = await this.getHistory(userId) + if (history?.historyId != null && history?.historyId !== '') { + this.ctx.info('Start part sync', { workspaceUuid: this.workspace, userId, historyId: history.historyId }) + await this.partSync(userId, userEmail, history.historyId) + } else { + this.ctx.info('Start full sync', { workspaceUuid: this.workspace, userId }) + await this.fullSync(userId, userEmail) + } + } catch (err) { + this.ctx.error('Sync error', { workspace: this.workspace, userId, err }) + } finally { + releaseLock() + } + } + + private getHistoryKey (userId: PersonId): string { + return `history:${this.workspace}:${userId}` + } +} diff --git a/services/gmail/pod-gmail/src/storage.ts b/services/gmail/pod-gmail/src/storage.ts deleted file mode 100644 index 6dc4e8d6de..0000000000 --- a/services/gmail/pod-gmail/src/storage.ts +++ /dev/null @@ -1,36 +0,0 @@ -// -// Copyright © 2020, 2021 Anticrm Platform Contributors. -// Copyright © 2021 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { MongoClientReference, getMongoClient } from '@hcengineering/mongo' -import { MongoClient } from 'mongodb' - -import config from './config' - -const clientRef: MongoClientReference = getMongoClient(config.MongoURI) -let client: MongoClient | undefined -export const getDB = (() => { - return async () => { - if (client === undefined) { - client = await clientRef.getClient() - } - - return client.db(config.MongoDB) - } -})() - -export const closeDB: () => Promise = async () => { - clientRef.close() -} diff --git a/services/gmail/pod-gmail/src/tokens.ts b/services/gmail/pod-gmail/src/tokens.ts new file mode 100644 index 0000000000..0120c91ae5 --- /dev/null +++ b/services/gmail/pod-gmail/src/tokens.ts @@ -0,0 +1,86 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { MeasureContext, PersonId, WorkspaceUuid } from '@hcengineering/core' +import type { AccountClient } from '@hcengineering/account-client' +import { getAccountClient } from '@hcengineering/server-client' +import { GMAIL_INTEGRATION, SecretType, Token } from './types' + +export class TokenStorage { + private readonly accountClient: AccountClient + + constructor ( + private readonly ctx: MeasureContext, + private readonly workspace: WorkspaceUuid, + platformToken: string + ) { + this.accountClient = getAccountClient(platformToken) + } + + async getToken (socialId: PersonId): Promise { + const secret = await this.accountClient.getIntegrationSecret({ + key: SecretType.TOKEN, + kind: GMAIL_INTEGRATION, + socialId, + workspaceUuid: this.workspace + }) + return secret?.secret !== undefined ? JSON.parse(secret.secret) : null + } + + async saveToken (socialId: PersonId, token: Token): Promise { + const existingToken = await this.getToken(socialId) + + if (existingToken !== null) { + const updatedToken = { + ...existingToken, + ...token + } + await this.accountClient.updateIntegrationSecret({ + key: SecretType.TOKEN, + kind: GMAIL_INTEGRATION, + socialId, + secret: JSON.stringify(updatedToken), + workspaceUuid: this.workspace + }) + this.ctx.info('Updated integration secret', { socialId, workspaceUuid: this.workspace }) + } else { + await this.accountClient.addIntegrationSecret({ + key: SecretType.TOKEN, + kind: GMAIL_INTEGRATION, + socialId, + secret: JSON.stringify(token), + workspaceUuid: this.workspace + }) + this.ctx.info('Created integration secret', { socialId, workspaceUuid: this.workspace }) + } + } + + async deleteToken (socialId: PersonId): Promise { + await this.accountClient.deleteIntegrationSecret({ + key: SecretType.TOKEN, + kind: GMAIL_INTEGRATION, + socialId, + workspaceUuid: this.workspace + }) + } +} + +export async function getWorkspaceTokens (accountClient: AccountClient, workspace: WorkspaceUuid): Promise { + const secrets = await accountClient.listIntegrationsSecrets({ + kind: GMAIL_INTEGRATION, + workspaceUuid: workspace + }) + return secrets.map((secret: { secret: string }) => JSON.parse(secret.secret)) +} diff --git a/services/gmail/pod-gmail/src/types.ts b/services/gmail/pod-gmail/src/types.ts index 60214cb334..2e0b3b3184 100644 --- a/services/gmail/pod-gmail/src/types.ts +++ b/services/gmail/pod-gmail/src/types.ts @@ -14,7 +14,7 @@ // limitations under the License. // -import type { PersonId, Doc, WorkspaceUuid } from '@hcengineering/core' +import type { AccountUuid, Doc, SocialId, WorkspaceUuid } from '@hcengineering/core' import type { NextFunction, Request, Response } from 'express' import type { Credentials } from 'google-auth-library' import type { Channel as PlatformChannel } from '@hcengineering/contact' @@ -26,7 +26,9 @@ export type History = User & { } export interface User { - userId: PersonId + userId: AccountUuid + email?: string + socialId: SocialId workspace: WorkspaceUuid token: string } @@ -68,3 +70,8 @@ export interface ProjectCredentialsData { client_secret: string redirect_uris: string[] } +export const GMAIL_INTEGRATION = 'gmail' + +export enum SecretType { + TOKEN = 'token' +} diff --git a/services/gmail/pod-gmail/src/utils.ts b/services/gmail/pod-gmail/src/utils.ts index 193fd39860..d5de1d1d1b 100644 --- a/services/gmail/pod-gmail/src/utils.ts +++ b/services/gmail/pod-gmail/src/utils.ts @@ -14,9 +14,12 @@ // limitations under the License. // -import { type Data, type Doc, type DocumentUpdate } from '@hcengineering/core' +import { type Data, type Doc, type DocumentUpdate, systemAccountUuid } from '@hcengineering/core' +import { generateToken } from '@hcengineering/server-token' import { deepEqual } from 'fast-equals' +import { type KeyValueClient, getClient as getKeyValueClient } from '@hcengineering/kvs-client' import { type Token, type User } from './types' +import config from './config' export class DeferredPromise { public readonly promise: Promise @@ -63,3 +66,27 @@ function toUndef (value: any): any { export function isToken (user: User | Token): user is Token { return (user as Token).access_token !== undefined } + +export function addFooter (message: string): string { + if (config.FooterMessage === undefined || config.FooterMessage.trim() === '') return message + return message + config.FooterMessage.trim() +} + +export function serviceToken (): string { + return generateToken(systemAccountUuid, undefined, { service: 'gmail' }) +} + +export async function wait (sec: number): Promise { + await new Promise((resolve) => { + setTimeout(() => { + resolve() + }, sec * 1000) + }) +} + +let keyValueClient: KeyValueClient | undefined +export function getKvsClient (token: string): KeyValueClient { + if (keyValueClient !== undefined) return keyValueClient + keyValueClient = getKeyValueClient('gmail', config.KvsUrl, token) + return keyValueClient +} diff --git a/services/gmail/pod-gmail/src/workspaceClient.ts b/services/gmail/pod-gmail/src/workspaceClient.ts index f26c537d77..08d9fc546b 100644 --- a/services/gmail/pod-gmail/src/workspaceClient.ts +++ b/services/gmail/pod-gmail/src/workspaceClient.ts @@ -14,10 +14,9 @@ // limitations under the License. // -import contact, { type Employee, type Channel as PlatformChannel } from '@hcengineering/contact' +import contact, { type Channel as PlatformChannel, type Person, Employee } from '@hcengineering/contact' import core, { type WorkspaceUuid, - type PersonId, type Client, type Doc, MeasureContext, @@ -27,15 +26,19 @@ import core, { type TxCreateDoc, TxProcessor, type TxRemoveDoc, - type TxUpdateDoc + type TxUpdateDoc, + PersonId, + AccountUuid, + TxOperations } from '@hcengineering/core' import gmailP, { type NewMessage } from '@hcengineering/gmail' import type { StorageAdapter } from '@hcengineering/server-core' import { generateToken } from '@hcengineering/server-token' -import { type Db } from 'mongodb' import { getClient } from './client' import { GmailClient } from './gmail' import { type Channel, type ProjectCredentials, type User } from './types' +import { getAccountSocialIds } from './accounts' +import { cleanIntegrations } from './integrations' export class WorkspaceClient { private messageSubscribed: boolean = false @@ -49,7 +52,6 @@ export class WorkspaceClient { private constructor ( private readonly ctx: MeasureContext, private readonly credentials: ProjectCredentials, - private readonly mongo: Db, private readonly storageAdapter: StorageAdapter, private readonly workspace: WorkspaceUuid ) {} @@ -57,29 +59,30 @@ export class WorkspaceClient { static async create ( ctx: MeasureContext, credentials: ProjectCredentials, - mongo: Db, storageAdapter: StorageAdapter, workspace: WorkspaceUuid ): Promise { - const instance = new WorkspaceClient(ctx, credentials, mongo, storageAdapter, workspace) + const instance = new WorkspaceClient(ctx, credentials, storageAdapter, workspace) await instance.initClient(workspace) return instance } - async createGmailClient (user: User): Promise { - const current = this.getGmailClient(user.userId) + async createGmailClient (user: User, authCode?: string): Promise { + const current = user.socialId?._id !== undefined ? this.getGmailClient(user.socialId?._id) : undefined if (current !== undefined) return current + this.ctx.info('Creating new gmail client', { workspaceUuid: this.workspace, userId: user.userId }) + this.ctx.info('Creating new gmail user', { user }) const newClient = await GmailClient.create( this.ctx, this.credentials, user, - this.mongo, this.client, this, this.workspace, - this.storageAdapter + this.storageAdapter, + authCode ) - this.clients.set(user.userId, newClient) + this.clients.set(user.socialId._id, newClient) return newClient } @@ -91,32 +94,34 @@ export class WorkspaceClient { await this.client?.close() } - async getUserId (email: string): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // const user = this.client.getModel().getAccountByEmail(email) - // if (user === undefined) { - // throw new Error('User not found') - // } - // return user._id - } - - async signout (email: string, byError: boolean = false): Promise { - const userId = await this.getUserId(email) - const client = this.clients.get(userId) - if (client !== undefined) { - await client.signout(byError) + async signoutByAccountId (userId: AccountUuid, byError: boolean = false): Promise { + const socialIds = await getAccountSocialIds(userId) + this.ctx.info('socialIds', { socialIds }) + this.ctx.info('clients', { clients: this.clients }) + let deleted = false + for (const socialId of socialIds) { + const client = this.clients.get(socialId._id) + if (client !== undefined) { + await client.signout(byError) + this.clients.delete(socialId._id) + deleted = true + } } - this.clients.delete(userId) + if (!deleted && socialIds.length > 0) { + this.ctx.info('Clean up integrations without clients') + const tx = new TxOperations(this.client, socialIds[0]._id) + await cleanIntegrations(this.ctx, tx, userId, this.workspace) + } + return this.clients.size } - async signoutByUserId (userId: PersonId, byError: boolean = false): Promise { - const client = this.clients.get(userId) + async signoutBySocialId (socialId: PersonId, byError: boolean = false): Promise { + const client = this.clients.get(socialId) if (client !== undefined) { await client.signout(byError) + this.clients.delete(socialId) } - this.clients.delete(userId) return this.clients.size } @@ -126,7 +131,7 @@ export class WorkspaceClient { private async initClient (workspace: WorkspaceUuid): Promise { const token = generateToken(systemAccountUuid, workspace, { service: 'gmail' }) - console.log('token', token, workspace) + this.ctx.info('Init client', { workspaceUuid: workspace }) const client = await getClient(token) client.notify = (...tx: Tx[]) => { void this.txHandler(...tx) @@ -161,7 +166,7 @@ export class WorkspaceClient { const newMessages = await this.client.findAll(gmailP.class.NewMessage, { status: 'new' }) - console.log('get new messages, recieved', this.workspace, newMessages.length) + this.ctx.info('get new messages', { workspaceUuid: this.workspace, count: newMessages.length }) await this.subscribeMessages() for (const message of newMessages) { const from = message.from ?? message.createdBy ?? message.modifiedBy @@ -169,7 +174,11 @@ export class WorkspaceClient { if (client !== undefined) { await client.createMessage(message) } else { - console.log('client not found, skip message', this.workspace, from, message._id) + this.ctx.error('client not found, skip message', { + workspaceUuid: this.workspace, + from, + messageId: message._id + }) } } } @@ -326,41 +335,36 @@ export class WorkspaceClient { // #region Users async checkUsers (): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // const removedEmployees = await this.client.findAll(contact.mixin.Employee, { - // active: false - // }) - // const accounts = await this.client.findAll(contact.class.PersonAccount, { - // person: { $in: removedEmployees.map((p) => p._id) } - // }) - // for (const acc of accounts) { - // await this.deactivateUser(acc) - // } - // this.txHandlers.push(async (...txes: Tx[]) => { - // for (const tx of txes) { - // await this.txEmployeeHandler(tx) - // } - // }) - // console.log('deactivate users', this.workspace, accounts.length) + const removedEmployees = await this.client.findAll(contact.mixin.Employee, { + active: false + }) + + for (const person of removedEmployees) { + await this.deactivateUser(person) + } + this.txHandlers.push(async (...txes: Tx[]) => { + for (const tx of txes) { + await this.txEmployeeHandler(tx) + } + }) + this.ctx.info('deactivate users', { workspaceUuid: this.workspace, count: removedEmployees.length }) } - // private async deactivateUser (acc: PersonAccount): Promise { - // await this.signout(acc.email, true) - // } + private async deactivateUser (acc: Person): Promise { + if (acc.personUuid === undefined) return + await this.signoutByAccountId(acc.personUuid as AccountUuid, true) + } private async txEmployeeHandler (tx: Tx): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // if (tx._class !== core.class.TxUpdateDoc) return - // const ctx = tx as TxUpdateDoc - // if (!this.client.getHierarchy().isDerived(ctx.objectClass, contact.mixin.Employee)) return - // if (ctx.operations.active === false) { - // const acc = await this.client.findOne(contact.class.PersonAccount, { person: ctx.objectId }) - // if (acc !== undefined) { - // await this.deactivateUser(acc) - // } - // } + if (tx._class !== core.class.TxUpdateDoc) return + const ctx = tx as TxUpdateDoc + if (!this.client.getHierarchy().isDerived(ctx.objectClass, contact.mixin.Employee)) return + if (ctx.operations.active === false) { + const acc = await this.client.findOne(contact.class.Person, { _id: ctx.objectId }) + if (acc !== undefined) { + await this.deactivateUser(acc) + } + } } // #endregion