diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 35c0e09c52..9990a68772 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -2227,6 +2227,9 @@ importers: snappyjs: specifier: ^0.7.0 version: 0.7.0 + striptags: + specifier: ^3.2.0 + version: 3.2.0 style-loader: specifier: ^3.3.1 version: 3.3.4(webpack@5.97.1) @@ -4417,7 +4420,7 @@ packages: version: 0.0.0 '@rush-temp/calendar-resources@file:projects/calendar-resources.tgz': - resolution: {integrity: sha512-RkeOzU5zVyx9sAt73Zf/5NXrv8NeDh8AlNCBqv3lFPv23wvvsEUyrGJkeIiz1FU8JXg6ARR/vT3t9+IzNNNFQQ==, tarball: file:projects/calendar-resources.tgz} + resolution: {integrity: sha512-eWYLBlRXnQFqGd3AoaHW837h+8bmx+E7xihore4xPshh7F3AZJ+LuRdnuJhiTQ3YQVp+x1OED3u6+Wn9JzRd9w==, tarball: file:projects/calendar-resources.tgz} version: 0.0.0 '@rush-temp/calendar@file:projects/calendar.tgz': @@ -4497,7 +4500,7 @@ packages: version: 0.0.0 '@rush-temp/communication-resources@file:projects/communication-resources.tgz': - resolution: {integrity: sha512-fbzeC4XwbwgeXRdRXsguOzBB7ioJUCr+k1xNqw9d7y7mxhz7rtDsZIF7Qj8xm7gs+P5y0L2gsiGiwe9OMDgAaw==, tarball: file:projects/communication-resources.tgz} + resolution: {integrity: sha512-tNTGgOqQ1EOdChaNs4PcbEZwaxInVLb1c04YLtc0k/8m046n69zTQhwu/OJo3mjuDcvvkF0BGgtwnDC1iFwKDA==, tarball: file:projects/communication-resources.tgz} version: 0.0.0 '@rush-temp/communication-rest-client@file:projects/communication-rest-client.tgz': @@ -4673,7 +4676,7 @@ packages: version: 0.0.0 '@rush-temp/github-resources@file:projects/github-resources.tgz': - resolution: {integrity: sha512-SoxO9GDPj0TrU4JL8RnOYlp/wNx6ZBzTL8xcIx+fES66e1zdsYt4K0Eq1/uV0EnCI+d+tdYeD4sQUMqlYERAkw==, tarball: file:projects/github-resources.tgz} + resolution: {integrity: sha512-tQ5ZcI9uIYDVpiT2aOZAV1iIprahWnBThvBfmiCdBt9A1zAH7UdA2od9UwjAOgefnAaaCtVnSZcMtZyRBwY6Dg==, tarball: file:projects/github-resources.tgz} version: 0.0.0 '@rush-temp/github@file:projects/github.tgz': @@ -4685,7 +4688,7 @@ packages: version: 0.0.0 '@rush-temp/gmail-resources@file:projects/gmail-resources.tgz': - resolution: {integrity: sha512-xr9UPYNDxlO/FM2U7inZHlwuMdgdkymGhff2rpurI0uB8fIczQkegJmSqu8olMC7xgMTvdWyp9vkBP7xBpRprw==, tarball: file:projects/gmail-resources.tgz} + resolution: {integrity: sha512-CtPpz9Jx7rGJXdAon9+eI5gUIvhFU/ztjHUUeBYsPA4pjBXZH5xPC0YWqxqbp6LWdnIwy9wL7BTAXRPlAR6eQA==, tarball: file:projects/gmail-resources.tgz} version: 0.0.0 '@rush-temp/gmail@file:projects/gmail.tgz': @@ -4753,7 +4756,7 @@ packages: version: 0.0.0 '@rush-temp/integration-client@file:projects/integration-client.tgz': - resolution: {integrity: sha512-r46KMTTe/Xy/8c7L81e3fmLX/ESI5ChiCmz+KTBVnHqeuM788FYZVdJhb+JebbXoi+lSHhqE0kWm825RhPiUEA==, tarball: file:projects/integration-client.tgz} + resolution: {integrity: sha512-73s3IbZbC5cv7DGfYnGP9LXNNsTTn9n+Bdvdmj/xqy3hayPTvQOFuzQHTJXBtoFh9RKATvI8v802e7BcJF3D+w==, tarball: file:projects/integration-client.tgz} version: 0.0.0 '@rush-temp/inventory-assets@file:projects/inventory-assets.tgz': @@ -4821,7 +4824,7 @@ packages: version: 0.0.0 '@rush-temp/mail-common@file:projects/mail-common.tgz': - resolution: {integrity: sha512-5bI4jSUvdgJu9WZfMBixVXIGqXr6vkojFQ05Ruk52dYc+bo6PvPHlQNPgzyDJjAnoX7lBUvsKJwKgY2TbbwcWQ==, tarball: file:projects/mail-common.tgz} + resolution: {integrity: sha512-TEKnSqRlNdp+CAZC5Oq5IC9fODEsemF6FboiNgYxwgUcEJeYMEOd9difhrv+BR3e2ok2cMmbYHDnLIVO9VFKOA==, tarball: file:projects/mail-common.tgz} version: 0.0.0 '@rush-temp/mail@file:projects/mail.tgz': @@ -5293,7 +5296,7 @@ packages: version: 0.0.0 '@rush-temp/pod-calendar@file:projects/pod-calendar.tgz': - resolution: {integrity: sha512-S3ygrMi6bQUc9U7sfujx5iv8t7gHC+Kl/u/NjRtcmEPDdF47zdmr8mr0THnZNfpgpabFcnBaPcTm9mvXXe/Ftg==, tarball: file:projects/pod-calendar.tgz} + resolution: {integrity: sha512-WrzMM2DguMpVeRdZ+OfM2/55ZtMNFmfRpXMN7V2i3+KpsAUKDQBFqHAibiTJJAiI6uuXN8mkZiSfn6Bh6eCvYA==, tarball: file:projects/pod-calendar.tgz} version: 0.0.0 '@rush-temp/pod-collaborator@file:projects/pod-collaborator.tgz': @@ -5325,7 +5328,7 @@ packages: version: 0.0.0 '@rush-temp/pod-gmail@file:projects/pod-gmail.tgz': - resolution: {integrity: sha512-HAat9sFoOjj5Shx3UlRU9ACh2mH4gLlp8CEkxv3gbuoQwVQsgJ0VLCe4FwUjj9DPHP8cpBTek6h75DOXbbvN3g==, tarball: file:projects/pod-gmail.tgz} + resolution: {integrity: sha512-gIKoyoL1gSWpWR35vPCMOpYY/Z+TkO0YJvKHU9ysRpH5Uqm8vjBD91RckK86xNoAHCC6Tuk/xsHnLQfSuQTv3w==, tarball: file:projects/pod-gmail.tgz} version: 0.0.0 '@rush-temp/pod-inbound-mail@file:projects/pod-inbound-mail.tgz': @@ -5841,11 +5844,11 @@ packages: version: 0.0.0 '@rush-temp/setting-resources@file:projects/setting-resources.tgz': - resolution: {integrity: sha512-WTrqgpE93g8zjpDYMxBcNlR57SdyerXgTwcm/ffH9Rq6g6jeci4d3Ccg866G/ByekstdqoKmL0PsayYuMzcR1g==, tarball: file:projects/setting-resources.tgz} + resolution: {integrity: sha512-Ypujv1wWqwY3NEz1nnDPTQRPBCMuz71yh8XBeaO/R8LQoxbXNrawIJPSkPhskudFYmmY1yKvWGuV8YO1vYiR2w==, tarball: file:projects/setting-resources.tgz} version: 0.0.0 '@rush-temp/setting@file:projects/setting.tgz': - resolution: {integrity: sha512-+aMwLYnIOHywBM7Qx0U8oJFBAFDukVI6aW9XllMbQj4YVjxXMQuZ+IqdW7PdclDq+xMC8u8noU30nzAKIj45eg==, tarball: file:projects/setting.tgz} + resolution: {integrity: sha512-HWEdbpzjpofObK2fv0WkpSZeFVpLROeA6s6/7tsJ/51IHCSxbsuqbJO+CGc+g1fZhbLdHUdIKH+J63kwsRaENA==, tarball: file:projects/setting.tgz} version: 0.0.0 '@rush-temp/sign@file:projects/sign.tgz': @@ -5909,7 +5912,7 @@ packages: version: 0.0.0 '@rush-temp/telegram-resources@file:projects/telegram-resources.tgz': - resolution: {integrity: sha512-oNF++UNPuNik4Ak7sGISighoWJtiPGG+x9+Bop7xHYpPk+j4QTobQlqDKBSAXJdXYE2fe4U64QL5vvIeZT/dDQ==, tarball: file:projects/telegram-resources.tgz} + resolution: {integrity: sha512-sC43Z4u+6Q0Rvj7UUk/T0ndVDEhBEgV777XTsoae+Zrsir6w1zXnSJ1gdoM4dmUVrsUiDN3NMeWFHiZODWYURg==, tarball: file:projects/telegram-resources.tgz} version: 0.0.0 '@rush-temp/telegram@file:projects/telegram.tgz': @@ -5997,7 +6000,7 @@ packages: version: 0.0.0 '@rush-temp/tool@file:projects/tool.tgz': - resolution: {integrity: sha512-Vi/BPbM8K+6iZBNQRSsjIAtu4vtocPqM89TWdTxQx4xgUuQqptiNCH/OFzCyGEE7PbY3Ng/dAJcafPhAg4UKKw==, tarball: file:projects/tool.tgz} + resolution: {integrity: sha512-9s0FnsEBN/WrQoIcDgSH/hbgkV49BSHIp/TTZqiRVzmV/yZY9dqaSgYhJxTfWZQnELkKWYIRwNTniqipMobGXw==, tarball: file:projects/tool.tgz} version: 0.0.0 '@rush-temp/tracker-assets@file:projects/tracker-assets.tgz': @@ -13245,6 +13248,9 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + striptags@3.2.0: + resolution: {integrity: sha512-g45ZOGzHDMe2bdYMdIvdAfCQkCTDMGBazSw1ypMowwGIee7ZQ5dU0rBJ8Jqgl+jAKIv4dbeE1jscZq9wid1Tkw==} + strnum@1.0.5: resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==} @@ -13748,8 +13754,8 @@ packages: resolution: {integrity: sha512-XiBOnM/UpUq21ZZ91q2AVDOnGROE6UQd37WrO9WBgw4u2eGvUCNOheMmZ3EfEUj7DLHr8tre+Um/436Of/Vwzg==} hasBin: true - uc.micro@2.0.0: - resolution: {integrity: sha512-DffL94LsNOccVn4hyfRe5rdKa273swqeA5DJpMOeFmEn1wCDc7nAbbB0gXlgBCL7TNzeTv6G7XVWzan7iJtfig==} + uc.micro@2.1.0: + resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==} ufo@1.5.4: resolution: {integrity: sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ==} @@ -21254,6 +21260,7 @@ snapshots: '@tsconfig/node16': 1.0.4 '@types/express': 4.17.21 '@types/jest': 29.5.12 + '@types/markdown-it': 13.0.8 '@types/node': 22.15.29 '@types/sanitize-html': 2.15.0 '@types/turndown': 5.0.5 @@ -21270,8 +21277,10 @@ snapshots: image-size: 1.1.1 jest: 29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.8.3)) kafkajs: 2.2.4 + markdown-it: 14.0.0 prettier: 3.2.5 sanitize-html: 2.16.0 + striptags: 3.2.0 ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.8.3)))(typescript@5.8.3) ts-node: 10.9.2(@types/node@22.15.29)(typescript@5.8.3) turndown: 7.2.0 @@ -35521,7 +35530,7 @@ snapshots: linkify-it@5.0.0: dependencies: - uc.micro: 2.0.0 + uc.micro: 2.1.0 linkifyjs@4.2.0: {} @@ -35795,7 +35804,7 @@ snapshots: linkify-it: 5.0.0 mdurl: 2.0.0 punycode.js: 2.3.1 - uc.micro: 2.0.0 + uc.micro: 2.1.0 marked@13.0.3: {} @@ -37974,6 +37983,8 @@ snapshots: strip-json-comments@3.1.1: {} + striptags@3.2.0: {} + strnum@1.0.5: {} strtok3@6.3.0: @@ -38604,7 +38615,7 @@ snapshots: transitivePeerDependencies: - encoding - uc.micro@2.0.0: {} + uc.micro@2.1.0: {} ufo@1.5.4: {} diff --git a/services/gmail/pod-gmail/package.json b/services/gmail/pod-gmail/package.json index dd1fd86fa6..449981db38 100644 --- a/services/gmail/pod-gmail/package.json +++ b/services/gmail/pod-gmail/package.json @@ -65,6 +65,7 @@ "@hcengineering/client": "^0.6.18", "@hcengineering/client-resources": "^0.6.27", "@hcengineering/communication-sdk-types": "^0.1.0", + "@hcengineering/communication-rest-client": "^0.1.0", "@hcengineering/contact": "^0.6.24", "@hcengineering/kvs-client": "^0.6.0", "@hcengineering/mail-common": "^0.6.0", @@ -77,6 +78,7 @@ "@hcengineering/server-storage": "^0.6.0", "@hcengineering/server-token": "^0.6.11", "@hcengineering/integration-client": "^0.6.0", + "@hcengineering/kafka": "^0.6.0", "cors": "^2.8.5", "dotenv": "~16.0.0", "express": "^4.21.2", diff --git a/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts b/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts index 70c188a928..7aa34f09c6 100644 --- a/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts +++ b/services/gmail/pod-gmail/src/__tests__/gmailClient.test.ts @@ -39,11 +39,23 @@ jest.mock('@hcengineering/core', () => { WorkspaceUuid: String, // Provide missing properties configUserAccountUuid: 'test-user-id', + systemAccountUuid: 'system-user-id', core: { space: { Workspace: 'workspace' } }, + // Add toFindResult function to fix the import error + toFindResult: jest.fn().mockImplementation((items: any[]) => ({ + items, + total: items.length + })), + // Add withContext decorator mock + withContext: jest.fn().mockImplementation((name: string) => { + return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) { + return descriptor + } + }), TxOperations: jest.fn().mockImplementation(() => ({ findAll: jest.fn().mockResolvedValue([]), findOne: jest.fn().mockResolvedValue(undefined), @@ -60,7 +72,9 @@ jest.mock('@hcengineering/core', () => { }) jest.mock('@hcengineering/mail-common', () => ({ - createMessages: jest.fn().mockResolvedValue(undefined) + createMessages: jest.fn().mockResolvedValue(undefined), + getChannel: jest.fn().mockResolvedValue({ _id: 'test-channel-id' }), + isSyncedMessage: jest.fn().mockReturnValue(false) })) jest.mock('googleapis', () => ({ @@ -79,8 +93,12 @@ jest.mock('../tokens') jest.mock('../message/adapter') jest.mock('../message/sync') jest.mock('../message/attachments') -jest.mock('@hcengineering/server-client', () => ({ - getAccountClient: jest.fn() +jest.mock('@hcengineering/server-core', () => ({ + withContext: jest.fn().mockImplementation((name: string) => { + return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) { + return descriptor + } + }) })) jest.mock('../accounts', () => ({ getOrCreateSocialId: jest.fn().mockResolvedValue({ _id: 'test-social-id' }) @@ -89,7 +107,7 @@ jest.mock('../gmail/utils', () => ({ getEmail: jest.fn().mockResolvedValue('test@example.com') })) jest.mock('../integrations', () => ({ - createIntegrationIfNotEsixts: jest.fn().mockResolvedValue({ _id: 'test-integration-id' }), + createIntegrationIfNotExists: jest.fn().mockResolvedValue({ _id: 'test-integration-id' }), disableIntegration: jest.fn(), removeIntegration: jest.fn() })) @@ -133,6 +151,15 @@ jest.mock('@hcengineering/account-client', () => ({ endpoint: 'wss://test-endpoint.com', workspace: 'mockWorkspaceId', token: 'test-token' + }), + getPersonInfo: jest.fn().mockResolvedValue({ + socialIds: [ + { + _id: 'test-social-id', + value: 'test@example.com', + type: 'email' + } + ] }) })), isWorkspaceLoginInfo: jest.fn().mockImplementation(() => true) diff --git a/services/gmail/pod-gmail/src/client.ts b/services/gmail/pod-gmail/src/client.ts index f1635f0188..656ece1b88 100644 --- a/services/gmail/pod-gmail/src/client.ts +++ b/services/gmail/pod-gmail/src/client.ts @@ -14,11 +14,33 @@ // limitations under the License. // -import { type Client } from '@hcengineering/core' +import { WorkspaceUuid, type Client } from '@hcengineering/core' import { createClient, getTransactorEndpoint } from '@hcengineering/server-client' +import { generateToken } from '@hcengineering/server-token' +import { systemAccountUuid } from '@hcengineering/core' +import { + createRestClient as createCommunicationRestClient, + RestClient as CommunicationRestClient +} from '@hcengineering/communication-rest-client' export async function getClient (token: string): Promise { const endpoint = await getTransactorEndpoint(token) console.log('connecting to', endpoint) return await createClient(endpoint, token) } + +export async function getCommunicationClient (workspace: WorkspaceUuid): Promise { + const token = generateToken(systemAccountUuid, workspace, { service: 'gmail' }) + const endpoint = toHttpUrl(await getTransactorEndpoint(token)) + return createCommunicationRestClient(endpoint, workspace, token) +} + +function toHttpUrl (url: string): string { + if (url.startsWith('ws://')) { + return url.replace('ws://', 'http://') + } + if (url.startsWith('wss://')) { + return url.replace('wss://', 'https://') + } + return url +} diff --git a/services/gmail/pod-gmail/src/gmail.ts b/services/gmail/pod-gmail/src/gmail.ts index 307934cee2..676a1ede02 100644 --- a/services/gmail/pod-gmail/src/gmail.ts +++ b/services/gmail/pod-gmail/src/gmail.ts @@ -17,6 +17,7 @@ import core, { AccountUuid, Client, MeasureContext, + PersonId, SocialId, SocialIdType, TxOperations, @@ -34,14 +35,22 @@ import { isWorkspaceLoginInfo, AccountClient } from '@hcengineering/account-client' -import { MailRecipient, type SyncOptions } from '@hcengineering/mail-common' +import { MailRecipient, type SyncOptions, getChannel, isSyncedMessage } from '@hcengineering/mail-common' +import chat from '@hcengineering/chat' import { encode64 } from './base64' import config from './config' import { GmailController } from './gmailController' import { RateLimiter } from './rateLimiter' -import type { ProjectCredentials, Token, User, SyncState } from './types' -import { addFooter, isToken, serviceToken, getKvsClient } from './utils' +import { + type ProjectCredentials, + type Token, + type User, + type SyncState, + HulyMailHeader, + HulyMessageIdHeader +} from './types' +import { addFooter, isToken, serviceToken, getKvsClient, createGmailSearchQuery } from './utils' import type { WorkspaceClient } from './workspaceClient' import { getOrCreateSocialId } from './accounts' import { createIntegrationIfNotExists, disableIntegration, removeIntegration } from './integrations' @@ -51,6 +60,9 @@ import { createMessageManager } from './message/adapter' import { SyncManager } from './message/sync' import { getEmail } from './gmail/utils' import { IMessageManager } from './message/types' +import { CreateMessageEvent } from '@hcengineering/communication-sdk-types' +import { Card } from '@hcengineering/card' +import { makeHTMLBodyV2 } from './message/v2/send' const SCOPES = ['https://www.googleapis.com/auth/gmail.modify'] @@ -60,7 +72,9 @@ function makeHTMLBody (message: NewMessage, from: string): string { 'MIME-Version: 1.0\n', 'Content-Transfer-Encoding: 7bit\n', `To: ${message.to} \n`, - `From: ${from} \n` + `From: ${from} \n`, + `${HulyMailHeader}: true\n`, + `${HulyMessageIdHeader}: ${message._id}\n` ] if (message.replyTo != null) { @@ -103,6 +117,7 @@ export class GmailClient { private readonly integrationToken: string private integration: Integration | undefined = undefined private syncStarted: boolean = false + private channel: Card | undefined = undefined private constructor ( private readonly ctx: MeasureContext, @@ -110,12 +125,13 @@ export class GmailClient { private readonly gmail: gmail_v1.Resource$Users, private readonly user: User, client: Client, - accountClient: AccountClient, + private readonly accountClient: AccountClient, wsInfo: WorkspaceLoginInfo, storageAdapter: StorageAdapter, private readonly workspace: WorkspaceClient, email: string, - private socialId: SocialId + private socialId: SocialId, + private readonly allSocialIds: Set ) { this.email = email this.integrationToken = serviceToken(wsInfo.workspace) @@ -163,6 +179,7 @@ export class GmailClient { const { client_secret, client_id, redirect_uris } = credentials.web const oAuth2Client = new google.auth.OAuth2(client_id, client_secret, redirect_uris[0]) const googleClient = google.gmail({ version: 'v1', auth: oAuth2Client }).users + ctx.info('Google client created:', { client_id, workspace: user.workspace, socialId: user.socialId }) let token: Credentials | undefined = isToken(user) ? user : undefined if (token === undefined && authCode !== undefined) { @@ -193,6 +210,8 @@ export class GmailClient { ctx.error('Unable to get workspace info', { workspaceId, email }) throw new Error('Unable to get workspace info') } + const info = await accountClient.getPersonInfo(user.userId) + const allSocialIds = new Set(info.socialIds.map((socialId) => socialId._id)) const gmailClient = new GmailClient( ctx, @@ -205,7 +224,8 @@ export class GmailClient { storageAdapter, workspace, email, - socialId + socialId, + allSocialIds ) await gmailClient.createIntregration() if (token !== undefined) { @@ -343,6 +363,92 @@ export class GmailClient { } } + async handleNewMessage (message: CreateMessageEvent): Promise { + try { + const personId = message.socialId + if (personId !== this.socialId._id && !this.allSocialIds.has(personId)) { + return + } + const email = await this.getEmail() + const thread = await this.client.findOne(chat.masterTag.Thread, { _id: message.cardId }) + const mailChannel = await this.getMailChannel() + if (mailChannel === undefined) { + this.ctx.error('Mail channel is not defined', { email, workspace: this.workspace }) + } + if (thread === undefined || thread?.parent !== mailChannel?._id) { + return + } + const isExisting = await this.isExistingGmailMessage(message, email) + if (isExisting) { + this.ctx.info('Skip existing message', { id: message._id, email }) + return + } + + this.ctx.info('Sending gmail message', { id: message._id, email }) + const gmailBody = await makeHTMLBodyV2(this.accountClient, message, thread, this.socialId._id, email) + await this.rateLimiter.take(100) + await this.gmail.messages.send({ + userId: 'me', + requestBody: { + raw: gmailBody + } + }) + } catch (err: any) { + this.ctx.error('Send gmail message v2 error', { + workspaceUuid: this.user.workspace, + userId: this.user.userId, + message: err.message + }) + if (err?.response?.data?.error === 'invalid_grant') { + await this.refreshToken() + } + } + } + + async getMailChannel (): Promise { + if (this.channel === undefined) { + this.channel = await getChannel(this.client, this.email) + } + return this.channel + } + + async isExistingGmailMessage (message: CreateMessageEvent, from: string): Promise { + try { + if (isSyncedMessage(message)) { + return true + } + await this.rateLimiter.take(10) + + const rawDate = message.date ?? new Date() + const messageDate = rawDate instanceof Date ? rawDate : new Date(rawDate) + + const startDate = new Date(messageDate.getTime() - 1000 * 60 * 60) // 1 hour before + const endDate = new Date(messageDate.getTime() + 1000 * 60 * 60) // 1 hour after + + const query: gmail_v1.Params$Resource$Users$Messages$List = { + userId: 'me', + q: createGmailSearchQuery(startDate, endDate, from), + maxResults: 20 // Limit results for performance + } + + const existingMessages = await this.gmail.messages.list(query) + + for (const msg of existingMessages.data.messages ?? []) { + const gmailDate = msg.internalDate != null ? new Date(Number.parseInt(msg.internalDate)) : undefined + if (gmailDate !== undefined && gmailDate?.getTime() === message.date?.getTime()) { + return true + } + } + return false + } catch (err: any) { + this.ctx.error('Error checking existing Gmail message', { + messageId: message.messageId, + error: err.message + }) + return false + } + } + private checkError (err: any): boolean { return err?.response?.data?.error === 'invalid_grant' } diff --git a/services/gmail/pod-gmail/src/gmailController.ts b/services/gmail/pod-gmail/src/gmailController.ts index e1f6a7be04..d9f193e858 100644 --- a/services/gmail/pod-gmail/src/gmailController.ts +++ b/services/gmail/pod-gmail/src/gmailController.ts @@ -15,30 +15,41 @@ import { AccountUuid, + Doc, isActiveMode, isArchivingMode, isDeletingMode, MeasureContext, RateLimiter, + TxCUD, WorkspaceInfoWithStatus, WorkspaceUuid, type PersonId } from '@hcengineering/core' -import { normalizeEmail } from '@hcengineering/mail-common' -import type { StorageAdapter } from '@hcengineering/server-core' +import { toMessageEvent, normalizeEmail } from '@hcengineering/mail-common' +import { ConsumerHandle, PlatformQueue, QueueTopic, type StorageAdapter } from '@hcengineering/server-core' +import { getPlatformQueue } from '@hcengineering/kafka' import { getAccountClient } from '@hcengineering/server-client' import { decode64 } from './base64' import config from './config' import { type GmailClient } from './gmail' import { getWorkspaceTokens } from './tokens' -import { SyncState, type ProjectCredentials, type Token, type User, type WorkspaceStateInfo } from './types' +import { + SyncState, + IntegrationVersion, + type ProjectCredentials, + type Token, + type User, + type WorkspaceStateInfo +} from './types' import { serviceToken } from './utils' import { WorkspaceClient } from './workspaceClient' import { getIntegrationClient } from './integrations' import { AuthProvider } from './gmail/auth' import { AccountClient } from '@hcengineering/account-client' +import { CreateMessageEvent } from '@hcengineering/communication-sdk-types' export class GmailController { private readonly workspaces: Map = new Map() @@ -53,6 +64,8 @@ export class GmailController { private readonly initLimitter = new RateLimiter(config.InitLimit) private readonly authProvider + private queue: PlatformQueue | undefined + private txConsumer: ConsumerHandle | undefined protected static _instance: GmailController @@ -103,9 +116,60 @@ export class GmailController { void this.checkPendingWorkspaces(pendingWorkspaces, sysClient) } catch (err: any) { this.ctx.error('Failed to start existing integrations', { error: err.message }) + } finally { + await this.startQueue() } } + async startQueue (): Promise { + try { + if (config.Version === IntegrationVersion.V1) { + // For old gmail there is already handler for NewMessage class + return + } + if (this.queue !== undefined) { + this.ctx.warn('Queue is already started') + return + } + this.queue = getPlatformQueue('gmail-service', config.QueueRegion) + if (this.queue === undefined) { + this.ctx.error('Queue not found') + return + } + this.txConsumer = this.queue.createConsumer>( + this.ctx, + QueueTopic.Tx, + this.queue.getClientId(), + async (msgs) => { + for (const msg of msgs) { + const workspaceUuid = msg.workspace + for (const tx of msg.value) { + const messageEvent = toMessageEvent(tx) + if (messageEvent !== undefined) { + await this.handleNewMessage(workspaceUuid, messageEvent) + } + } + } + }, + { + fromBegining: false // Set to true to process all historical messages + } + ) + this.ctx.info('Queue consumer started', { topic: QueueTopic.Tx }) + } catch (err: any) { + this.ctx.error('Failed to start queue consumer', { topic: QueueTopic.Tx }) + } + } + + async handleNewMessage (workspaceUuid: WorkspaceUuid, message: CreateMessageEvent): Promise { + const client = this.workspaces.get(workspaceUuid) + if (client === undefined) { + this.ctx.warn('No workspace client found', { socialId: message.socialId, workspaceUuid }) + return + } + await client.handleNewMessage(message) + } + async checkPendingWorkspaces (workspaceIds: Set, sysClient: AccountClient): Promise { try { let unprocessedWorkspaces = new Set(workspaceIds) @@ -203,14 +267,18 @@ export class GmailController { const workspaceClient = await this.getWorkspaceClient(workspace) const clients: GmailClient[] = [] for (const token of tokens) { + let timeout: NodeJS.Timeout | undefined try { - const timeout = setTimeout(() => { + timeout = setTimeout(() => { this.ctx.info('init client hang', { workspaceUuid: token.workspace, userId: token.userId }) }, 60000) const client = await workspaceClient.createGmailClient(token) clearTimeout(timeout) clients.push(client) } catch (err: any) { + if (timeout !== undefined) { + clearTimeout(timeout) + } this.ctx.error("Couldn't create client", { workspaceUuid: workspace, userId: token.userId, @@ -288,6 +356,12 @@ export class GmailController { await workspace.close() } this.workspaces.clear() + if (this.txConsumer !== undefined) { + await this.txConsumer.close() + } + if (this.queue !== undefined) { + await this.queue.shutdown() + } } async createClient (user: User | Token, authCode?: string): Promise { diff --git a/services/gmail/pod-gmail/src/message/v1/send.ts b/services/gmail/pod-gmail/src/message/v1/send.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/gmail/pod-gmail/src/message/v2/message.ts b/services/gmail/pod-gmail/src/message/v2/message.ts index a44a013ffb..25ad957f68 100644 --- a/services/gmail/pod-gmail/src/message/v2/message.ts +++ b/services/gmail/pod-gmail/src/message/v2/message.ts @@ -23,7 +23,8 @@ import { parseNameFromEmailHeader, EmailMessage, getProducer, - MailRecipient + MailRecipient, + getMessageExtra } from '@hcengineering/mail-common' import { type KeyValueClient } from '@hcengineering/kvs-client' import { AccountClient, isWorkspaceLoginInfo, WorkspaceLoginInfo } from '@hcengineering/account-client' @@ -32,6 +33,7 @@ import { IMessageManager } from '../types' import config from '../../config' import { AttachmentHandler } from '../attachments' import { decode64 } from '../../base64' +import { GmailMessageType, HulyMailHeader, HulyMessageIdHeader } from '../../types' export class MessageManagerV2 implements IMessageManager { private wsInfo: WorkspaceLoginInfo | undefined = undefined @@ -46,6 +48,10 @@ export class MessageManagerV2 implements IMessageManager { ) {} async saveMessage (message: GaxiosResponse, me: string): Promise { + if (isHulyMessage(message.data.payload)) { + this.ctx.info('Skipping Huly message', { mailId: message.data.id, me }) + return + } const res = convertMessage(message, me) const attachments = await this.attachmentHandler.getPartFiles(message.data.payload, message.data.id ?? '') @@ -80,6 +86,15 @@ function getHeaderValue (payload: gmail_v1.Schema$MessagePart | undefined, name: return headers?.find((header) => header.name?.toLowerCase() === name.toLowerCase())?.value ?? undefined } +export function isHulyMessage (payload: gmail_v1.Schema$MessagePart | undefined): boolean { + const hulyHeader = getHeaderValue(payload, HulyMailHeader) + if (hulyHeader !== undefined) { + return true + } + const hulyMessage = getHeaderValue(payload, HulyMessageIdHeader) + return hulyMessage !== undefined +} + function getPartsMessage (parts: gmail_v1.Schema$MessagePart[] | undefined, mime: string): string { let result = '' if (parts !== undefined) { @@ -128,6 +143,7 @@ function convertMessage (message: GaxiosResponse, me: s to, incoming, subject: getHeaderValue(message.data.payload, 'Subject') ?? '', - sendOn: date.getTime() + sendOn: date.getTime(), + extra: getMessageExtra(GmailMessageType, true) } } diff --git a/services/gmail/pod-gmail/src/message/v2/send.ts b/services/gmail/pod-gmail/src/message/v2/send.ts new file mode 100644 index 0000000000..1faaf52e27 --- /dev/null +++ b/services/gmail/pod-gmail/src/message/v2/send.ts @@ -0,0 +1,100 @@ +import { CreateMessageEvent } from '@hcengineering/communication-sdk-types' +import { type GaxiosResponse } from 'gaxios' +import { gmail_v1 } from 'googleapis' +import { markdownToHtml, getReplySubject } from '@hcengineering/mail-common' + +import { encode64 } from '../../base64' +import { addFooter } from '../../utils' +import { Card } from '@hcengineering/card' +import { PersonId, SocialIdType } from '@hcengineering/core' +import { AccountClient } from '@hcengineering/account-client' +import { HulyMailHeader, HulyMessageIdHeader } from '../../types' + +export async function makeHTMLBodyV2 ( + accountClient: AccountClient, + message: CreateMessageEvent, + thread: Card, + personId: PersonId, + from: string +): Promise { + const collaborators: PersonId[] = (thread as any).members ?? [] + if (collaborators.length === 0) { + return undefined + } + const recipients = collaborators.length > 1 ? collaborators.filter((c) => c !== personId) : collaborators + const mailSocialIds = (await accountClient.findFullSocialIds(recipients)).filter( + (id) => id.type === SocialIdType.EMAIL + ) + if (mailSocialIds.length === 0) { + console.warn('No social IDs found for recipients', { recipients }) + return undefined + } + const to = mailSocialIds[0].value + const copy = mailSocialIds.length > 1 ? mailSocialIds.slice(1).map((s) => s.value) : [] + + const str = [ + 'Content-Type: text/html; charset="UTF-8"\n', + 'MIME-Version: 1.0\n', + 'Content-Transfer-Encoding: 7bit\n', + `To: ${to} \n`, + `From: ${from} \n`, + `${HulyMailHeader}: true\n`, + `${HulyMessageIdHeader}: ${message._id}\n` + ] + + // TODO: get reply-to from channel + + if (copy != null && copy.length > 0) { + str.push(`Cc: ${copy.join(', ')} \n`) + } + + const subject = getReplySubject(thread.title) + if (subject != null) { + str.push(`Subject: =?UTF-8?B?${encode64(subject)}?= \n`) + } + + str.push('\n\n') + str.push(addFooter(markdownToHtml(message.content))) + const res = str.join('') + return encode64(res).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '') +} + +/** + * Check if a Gmail message was sent by the platform + */ +export function isPlatformSentMessage (message: gmail_v1.Schema$Message): boolean { + if (message.payload?.headers == null) return false + + // Check for custom platform headers + const headers = message.payload.headers + const platformSentHeader = headers.find((h) => h.name === HulyMailHeader) + if (platformSentHeader?.value === 'true') { + return true + } + + // Check for platform message ID header + const platformMessageIdHeader = headers.find((h) => h.name === HulyMessageIdHeader) + if (platformMessageIdHeader?.value != null) { + return true + } + + return false +} + +/** + * Check if message has platform footer signature + */ +export function hasPlatformFooter (messageBody: string): boolean { + const platformFooterPattern = /Sent via Huly/i + return platformFooterPattern.test(messageBody) +} + +/** + * Extract platform message ID from headers if available + */ +export function getPlatformMessageId (message: GaxiosResponse): string | undefined { + const headers = message.data?.payload?.headers + if (headers == null) return undefined + const platformMessageIdHeader = headers.find((h) => h.name === HulyMessageIdHeader) + return platformMessageIdHeader?.value ?? undefined +} diff --git a/services/gmail/pod-gmail/src/types.ts b/services/gmail/pod-gmail/src/types.ts index d73edf0e6e..f4dde63c34 100644 --- a/services/gmail/pod-gmail/src/types.ts +++ b/services/gmail/pod-gmail/src/types.ts @@ -92,3 +92,7 @@ export interface SyncState { isActive: boolean } } + +export const GmailMessageType = 'gmail-message' +export const HulyMailHeader = 'X-Huly-Sent' +export const HulyMessageIdHeader = 'X-Huly-Message-Id' diff --git a/services/gmail/pod-gmail/src/utils.ts b/services/gmail/pod-gmail/src/utils.ts index ce5e1ea581..4a53be065f 100644 --- a/services/gmail/pod-gmail/src/utils.ts +++ b/services/gmail/pod-gmail/src/utils.ts @@ -90,3 +90,20 @@ export function getKvsClient (token: string): KeyValueClient { keyValueClient = getKeyValueClient('gmail', config.KvsUrl, token) return keyValueClient } + +export function createGmailSearchQuery (fromDate: Date, toDate: Date, fromEmail: string): string { + const formatDate = (date: Date): string => { + const year = date.getFullYear() + const month = (date.getMonth() + 1).toString().padStart(2, '0') + const day = date.getDate().toString().padStart(2, '0') + return `${year}/${month}/${day}` + } + + const afterDate = formatDate(fromDate) + // Add one day to toDate for proper before date range (Gmail's before is exclusive) + const adjustedToDate = new Date(toDate) + adjustedToDate.setDate(adjustedToDate.getDate() + 1) + const beforeDate = formatDate(adjustedToDate) + + return `after:${afterDate} before:${beforeDate} from:${fromEmail}` +} diff --git a/services/gmail/pod-gmail/src/workspaceClient.ts b/services/gmail/pod-gmail/src/workspaceClient.ts index 43c0b9be3e..3a15dd5400 100644 --- a/services/gmail/pod-gmail/src/workspaceClient.ts +++ b/services/gmail/pod-gmail/src/workspaceClient.ts @@ -39,6 +39,7 @@ import { GmailClient } from './gmail' import { type Channel, type ProjectCredentials, type User } from './types' import { getAccountSocialIds } from './accounts' import { cleanIntegrations } from './integrations' +import { CreateMessageEvent } from '@hcengineering/communication-sdk-types' export class WorkspaceClient { private messageSubscribed: boolean = false @@ -129,6 +130,12 @@ export class WorkspaceClient { return this.clients.size } + async handleNewMessage (message: CreateMessageEvent): Promise { + for (const client of this.clients.values()) { + await client.handleNewMessage(message) + } + } + getGmailClient (userId: PersonId): GmailClient | undefined { return this.clients.get(userId) } diff --git a/services/mail/mail-common/package.json b/services/mail/mail-common/package.json index fcc2357b81..608a1f8c93 100644 --- a/services/mail/mail-common/package.json +++ b/services/mail/mail-common/package.json @@ -25,9 +25,10 @@ "@tsconfig/node16": "^1.0.4", "@types/express": "^4.17.13", "@types/jest": "^29.5.5", + "@types/markdown-it": "~13.0.0", "@types/node": "^22.15.29", - "@types/turndown": "^5.0.5", "@types/sanitize-html": "^2.15.0", + "@types/turndown": "^5.0.5", "@types/uuid": "^8.3.1", "@typescript-eslint/eslint-plugin": "^6.11.0", "@typescript-eslint/parser": "^6.11.0", @@ -55,15 +56,17 @@ "@hcengineering/communication-types": "^0.1.0", "@hcengineering/contact": "^0.6.24", "@hcengineering/core": "^0.6.32", - "@hcengineering/kvs-client": "^0.6.0", "@hcengineering/kafka": "^0.6.0", + "@hcengineering/kvs-client": "^0.6.0", "@hcengineering/mail": "^0.6.0", "@hcengineering/server-core": "^0.6.1", "@hcengineering/server-storage": "^0.6.0", + "image-size": "^1.1.1", "kafkajs": "^2.2.4", + "markdown-it": "^14.0.0", "sanitize-html": "^2.15.0", "turndown": "^7.2.0", "uuid": "^8.3.2", - "image-size": "^1.1.1" + "striptags": "^3.2.0" } } diff --git a/services/mail/mail-common/src/__tests__/md.test.ts b/services/mail/mail-common/src/__tests__/md.test.ts index 3ad6d820e7..66659d8e4d 100644 --- a/services/mail/mail-common/src/__tests__/md.test.ts +++ b/services/mail/mail-common/src/__tests__/md.test.ts @@ -18,6 +18,7 @@ import fs from 'fs' import path from 'path' import { getMdContent } from '../utils' +import { markdownToHtml, markdownToText } from '../md' import { EmailMessage } from '../types' describe('getMdContent', () => { @@ -122,3 +123,170 @@ describe('getMdContent', () => { expect(mockCtx.warn).not.toHaveBeenCalled() }) }) + +describe('markdownToHtml', () => { + it('should convert simple markdown to HTML', async () => { + const markdown = '# Hello World\n\nThis is **bold** text.' + const result = markdownToHtml(markdown) + + expect(result).toContain('

Hello World

') + expect(result).toContain('bold') + expect(result).toContain('

This is') + }) + + it('should handle empty string', async () => { + const result = markdownToHtml('') + expect(result).toBe('') + }) + + it('should handle non-string input gracefully', async () => { + const result = markdownToHtml(null as any) + expect(result).toBe('') + }) + + it('should enable linkify by default (GFM-like behavior)', async () => { + const markdown = 'Visit https://example.com for more info' + const result = markdownToHtml(markdown) + + expect(result).toContain('https://example.com') + }) + + it('should handle line breaks when enabled', async () => { + const markdown = 'Line 1\nLine 2' + const result = markdownToHtml(markdown, { breaks: true }) + + expect(result).toContain('
') + }) + + it('should disable linkify when GFM is disabled', async () => { + const markdown = 'Visit https://example.com for more info' + const result = markdownToHtml(markdown, { gfm: false }) + + // Without GFM (linkify), URLs should not be converted to links + expect(result).not.toContain('') + expect(result).toContain('https://example.com') + }) + + it('should handle markdown with links', async () => { + const markdown = '[Google](https://google.com)' + const result = markdownToHtml(markdown) + + expect(result).toContain('Google') + }) + + it('should handle markdown with code blocks', async () => { + const markdown = '```javascript\nconsole.log("hello");\n```' + const result = markdownToHtml(markdown) + + expect(result).toContain('

')
+    expect(result).toContain(' {
+    const markdown = 'Use `console.log()` for debugging'
+    const result = markdownToHtml(markdown)
+
+    expect(result).toContain('console.log()')
+  })
+
+  it('should handle lists', async () => {
+    const markdown = '- Item 1\n- Item 2\n- Item 3'
+    const result = markdownToHtml(markdown)
+
+    expect(result).toContain('
    ') + expect(result).toContain('
  • Item 1
  • ') + expect(result).toContain('
  • Item 2
  • ') + expect(result).toContain('
  • Item 3
  • ') + }) + + it('should handle ordered lists', async () => { + const markdown = '1. First\n2. Second\n3. Third' + const result = markdownToHtml(markdown) + + expect(result).toContain('
      ') + expect(result).toContain('
    1. First
    2. ') + expect(result).toContain('
    3. Second
    4. ') + expect(result).toContain('
    5. Third
    6. ') + }) + + it('should handle special characters in markdown', async () => { + const markdown = 'Text with & < > " \' characters' + const result = markdownToHtml(markdown) + + // markdown-it encodes single quotes as ' instead of ' + expect(result).toContain("

      Text with & < > " ' characters

      ") + }) +}) + +describe('markdownToText', () => { + it('should convert markdown to plain text', async () => { + const markdown = '# Hello World\n\nThis is **bold** text with [a link](https://example.com).' + const result = markdownToText(markdown) + + expect(result).toBe('Hello World\nThis is bold text with a link.') + expect(result).not.toContain('<') + expect(result).not.toContain('>') + expect(result).not.toContain('**') + expect(result).not.toContain('#') + }) + + it('should handle empty string', async () => { + const result = markdownToText('') + expect(result).toBe('') + }) + + it('should handle non-string input gracefully', async () => { + const result = markdownToText(undefined as any) + expect(result).toBe('') + }) + + it('should remove HTML tags from converted markdown', async () => { + const markdown = '**Bold** and *italic* text' + const result = markdownToText(markdown) + + expect(result).toBe('Bold and italic text') + expect(result).not.toContain('') + expect(result).not.toContain('') + }) + + it('should handle complex markdown with multiple elements', async () => { + const markdown = ` +# Title + +## Subtitle + +- List item 1 +- List item 2 + +> Blockquote text + +\`inline code\` + +[Link text](https://example.com) + `.trim() + + const result = markdownToText(markdown) + + expect(result).toContain('Title') + expect(result).toContain('Subtitle') + expect(result).toContain('List item 1') + expect(result).toContain('List item 2') + expect(result).toContain('Blockquote text') + expect(result).toContain('inline code') + expect(result).toContain('Link text') + + // Should not contain HTML tags + expect(result).not.toContain('<') + expect(result).not.toContain('>') + }) + + it('should trim whitespace from converted text', async () => { + const markdown = ' # Title \n\n Content ' + const result = markdownToText(markdown) + + expect(result.trim()).toBe('Title\nContent') + expect(result.startsWith(' ')).toBe(false) + expect(result.endsWith(' ')).toBe(false) + }) +}) diff --git a/services/mail/mail-common/src/index.ts b/services/mail/mail-common/src/index.ts index df82f64f6b..9a001f4dae 100644 --- a/services/mail/mail-common/src/index.ts +++ b/services/mail/mail-common/src/index.ts @@ -18,3 +18,5 @@ export * from './types' export * from './utils' export * from './mutex' export * from './queue' +export * from './txHandler' +export * from './md' diff --git a/services/mail/mail-common/src/md.ts b/services/mail/mail-common/src/md.ts new file mode 100644 index 0000000000..cc8051b9ff --- /dev/null +++ b/services/mail/mail-common/src/md.ts @@ -0,0 +1,87 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import MarkdownIt from 'markdown-it' +import striptags from 'striptags' +/** + * Configuration options for markdown to HTML conversion + */ +export interface MarkdownOptions { + /** Enable GitHub Flavored Markdown extensions */ + gfm?: boolean + /** Enable line breaks */ + breaks?: boolean +} + +/** + * Convert markdown text to HTML + * + * @param markdown The markdown text to convert + * @param options Optional configuration for the conversion + * @returns The converted HTML string + */ +export function markdownToHtml (markdown: string, options: MarkdownOptions = {}): string { + if (markdown === '' || typeof markdown !== 'string') { + return '' + } + + try { + // Create markdown-it instance with options + const md = new MarkdownIt({ + html: true, // Enable HTML tags in source + breaks: options.breaks ?? true, // Convert '\n' in paragraphs into
      + linkify: options.gfm ?? true, // Autoconvert URL-like text to links (part of GFM) + typographer: false // Disable some language-neutral replacement + quotes beautification + }) + + return md.render(markdown) + } catch (error) { + console.error('Error converting markdown to HTML:', error) + return escapeHtml(markdown) + } +} + +/** + * Convert markdown to plain text (strip all HTML tags) + */ +export function markdownToText (markdown: string): string { + if (markdown === '' || typeof markdown !== 'string') { + return '' + } + + try { + const html = markdownToHtml(markdown) + // Remove HTML tags using a robust library + return striptags(html).trim() + } catch (error) { + console.error('Error converting markdown to text:', error) + return markdown + } +} + +/** + * Escape HTML characters in plain text + */ +function escapeHtml (text: string): string { + const htmlEscapes: Record = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''' + } + + return text.replace(/[&<>"']/g, (match) => htmlEscapes[match] ?? match) +} diff --git a/services/mail/mail-common/src/message.ts b/services/mail/mail-common/src/message.ts index 6b5c0abfaa..54ce09f231 100644 --- a/services/mail/mail-common/src/message.ts +++ b/services/mail/mail-common/src/message.ts @@ -28,7 +28,6 @@ import core, { type TxOperations, AccountUuid, generateId, - OperationDomain, RateLimiter } from '@hcengineering/core' import { type KeyValueClient } from '@hcengineering/kvs-client' @@ -45,15 +44,13 @@ import { import { generateMessageId } from '@hcengineering/communication-shared' import { BaseConfig, SyncOptions, type Attachment } from './types' -import { EmailMessage, MailRecipient, MessageData } from './types' +import { COMMUNICATION_DOMAIN, EmailMessage, MailRecipient, MessageData } from './types' import { getBlobMetadata, getMdContent, MessageTimeShift } from './utils' import { PersonCacheFactory } from './person' import { PersonSpacesCacheFactory } from './personSpaces' import { ChannelCache, ChannelCacheFactory } from './channel' import { ThreadLookupService } from './thread' -const COMMUNICATION_DOMAIN = 'communication' as OperationDomain - /** * Creates mail messages in the platform * @@ -289,7 +286,8 @@ async function createMailThread ( messageId: subjectId, options: { noNotify: options?.noNotify - } + }, + extra: data.extra } const createSubjectData = toEventBuffer(createSubjectEvent) await sendToCommunicationTopic(producer, config, data, createSubjectData) diff --git a/services/mail/mail-common/src/person.ts b/services/mail/mail-common/src/person.ts index 70947afea5..2a7e2c8cdb 100644 --- a/services/mail/mail-common/src/person.ts +++ b/services/mail/mail-common/src/person.ts @@ -30,6 +30,7 @@ export interface CachedPerson { */ export class PersonCache { private readonly cache = new Map>() + private readonly emailCache = new Map() constructor ( private readonly ctx: MeasureContext, @@ -53,15 +54,21 @@ export class PersonCache { if (result === undefined) { throw new Error(`Failed to ensure person exists for email: ${email}`) } + this.emailCache.set(result.socialId, email) return result } + async getEmailBySocialId (socialId: PersonId): Promise { + return this.emailCache.get(socialId) + } + size (): number { return this.cache.size } clearCache (): void { this.cache.clear() + this.emailCache.clear() } private async fetchAndCachePerson ( diff --git a/services/mail/mail-common/src/txHandler.ts b/services/mail/mail-common/src/txHandler.ts new file mode 100644 index 0000000000..ebfddccfb4 --- /dev/null +++ b/services/mail/mail-common/src/txHandler.ts @@ -0,0 +1,48 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import core, { Tx, TxDomainEvent, TxOperations } from '@hcengineering/core' +import { CreateMessageEvent, MessageEventType } from '@hcengineering/communication-sdk-types' +import chat from '@hcengineering/chat' + +import { MessageType } from '@hcengineering/communication-types' +import { Card } from '@hcengineering/card' +import mail from '@hcengineering/mail' + +import { normalizeEmail } from './utils' +import { COMMUNICATION_DOMAIN } from './types' + +export function toMessageEvent (tx: Tx): CreateMessageEvent | undefined { + if (tx._class !== core.class.TxDomainEvent) { + return undefined + } + const domainTx = tx as TxDomainEvent + const isCreateMessage = + domainTx.domain === COMMUNICATION_DOMAIN && domainTx.event?.type === MessageEventType.CreateMessage + if (!isCreateMessage) { + return undefined + } + const event: CreateMessageEvent = domainTx.event + const isMessage = event.cardType === chat.masterTag.Thread && event.messageType === MessageType.Message + if (!isMessage) { + return undefined + } + return event +} + +export async function getChannel (client: TxOperations, email: string): Promise { + const normalizedEmail = normalizeEmail(email) + return await client.findOne(mail.tag.MailChannel, { title: normalizedEmail }) +} diff --git a/services/mail/mail-common/src/types.ts b/services/mail/mail-common/src/types.ts index ab97cf0fad..8793e092c3 100644 --- a/services/mail/mail-common/src/types.ts +++ b/services/mail/mail-common/src/types.ts @@ -13,8 +13,9 @@ // limitations under the License. import { Card } from '@hcengineering/card' +import { MessageExtra } from '@hcengineering/communication-types' import { PersonSpace } from '@hcengineering/contact' -import { PersonId, PersonUuid, Ref, WorkspaceUuid } from '@hcengineering/core' +import { OperationDomain, PersonId, PersonUuid, Ref, WorkspaceUuid } from '@hcengineering/core' // export interface Attachment { @@ -44,6 +45,7 @@ export interface EmailMessage { incoming: boolean subject: string sendOn: number + extra?: MessageExtra } export interface MailRecipient { @@ -73,8 +75,11 @@ export interface MessageData { threadId: Ref recipient: MailRecipient isReply: boolean + extra?: MessageExtra } export interface SyncOptions { noNotify?: boolean } + +export const COMMUNICATION_DOMAIN = 'communication' as OperationDomain diff --git a/services/mail/mail-common/src/utils.ts b/services/mail/mail-common/src/utils.ts index 2c64a37b43..e0034532de 100644 --- a/services/mail/mail-common/src/utils.ts +++ b/services/mail/mail-common/src/utils.ts @@ -18,6 +18,8 @@ import { imageSize } from 'image-size' import { BlobMetadata, MeasureContext } from '@hcengineering/core' import { Attachment, EmailContact, EmailMessage } from './types' +import { MessageExtra } from '@hcengineering/communication-types' +import { CreateMessageEvent } from '@hcengineering/communication-sdk-types' const NAME_EMAIL_PATTERN = /^(?:"?([^"<]+)"?\s*)?<([^>]+)>$/ const NAME_SEGMENT_REGEX = /[\s,;]+/ @@ -160,3 +162,35 @@ export enum MessageTimeShift { Collaborator = -2, Subject = -1 } + +export function getMessageExtra (type: string, synced: boolean): MessageExtra { + return { + type, + mailSynced: synced + } +} + +export function isSyncedMessage (message: CreateMessageEvent): boolean { + return message.extra?.mailSynced ?? false +} + +export function getReplySubject (threadName: string | undefined): string | undefined { + if (threadName === undefined) { + return undefined + } + + const trimmedSubject = threadName.trim() + if (trimmedSubject === '') { + return undefined + } + + // Check if subject already has a reply/forward prefix (case-insensitive) + const replyPrefixes = /^(re|aw|sv|antw|resp):\s*/i + const forwardPrefixes = /^(fwd?|fw|wg|tr|vs):\s*/i + + if (replyPrefixes.test(trimmedSubject) || forwardPrefixes.test(trimmedSubject)) { + return trimmedSubject + } + + return `Re: ${trimmedSubject}` +}