From 8a4524fef50477dbfcacda17952b11ca04c27b68 Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Fri, 27 Jun 2025 00:01:17 +0500 Subject: [PATCH] Fix calendar sync (#9392) Signed-off-by: Denis Bykhov --- .vscode/launch.json | 19 +++++++ services/calendar/pod-calendar/src/auth.ts | 2 +- .../calendar/pod-calendar/src/calendar.ts | 16 ------ .../pod-calendar/src/calendarController.ts | 8 ++- services/calendar/pod-calendar/src/client.ts | 2 +- services/calendar/pod-calendar/src/index.ts | 4 +- .../calendar/pod-calendar/src/kvsUtils.ts | 10 ++-- .../pod-calendar/src/outcomingClient.ts | 2 +- services/calendar/pod-calendar/src/sync.ts | 46 +++++++++++++-- .../pod-calendar/src/workspaceClient.ts | 56 +++++++++++++------ 10 files changed, 116 insertions(+), 49 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 190f1d8ae1..28541417da 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -723,6 +723,25 @@ "outputCapture": "std", "cwd": "${workspaceRoot}/services/analytics-collector/pod-analytics-collector" }, + { + "name": "Debug calendar", + "type": "node", + "request": "launch", + "args": ["src/index.ts"], + "env": { + "ACCOUNTS_URL" : "http://localhost:3000", + "Credentials" : "", + "KVS_URL" : "http://localhost:8094", + "SECRET" : "secret", + "WATCH_URL" : "https://calendar.hc.engineering/push" + }, + "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], + "runtimeVersion": "20", + "sourceMaps": true, + "showAsyncStacks": true, + "outputCapture": "std", + "cwd": "${workspaceRoot}/services/calendar/pod-calendar" + }, { "name": "Debug AI bot", "type": "node", diff --git a/services/calendar/pod-calendar/src/auth.ts b/services/calendar/pod-calendar/src/auth.ts index dbfe4a9034..82109b8df0 100644 --- a/services/calendar/pod-calendar/src/auth.ts +++ b/services/calendar/pod-calendar/src/auth.ts @@ -140,7 +140,7 @@ export class AuthController { const authRes = await this.authorize(code) await this.setWorkspaceIntegration(authRes) if (authRes.success) { - void IncomingSyncManager.sync( + void IncomingSyncManager.initSync( this.ctx, this.accountClient, this.client, diff --git a/services/calendar/pod-calendar/src/calendar.ts b/services/calendar/pod-calendar/src/calendar.ts index 31a4944a38..a41c17e18a 100644 --- a/services/calendar/pod-calendar/src/calendar.ts +++ b/services/calendar/pod-calendar/src/calendar.ts @@ -22,7 +22,6 @@ import { OAuth2Client } from 'google-auth-library' import { calendar_v3 } from 'googleapis' import { removeUserByEmail } from './kvsUtils' import { getRateLimitter, RateLimiter } from './rateLimiter' -import { IncomingSyncManager } from './sync' import { CALENDAR_INTEGRATION, type Token } from './types' import { convertDate, @@ -77,21 +76,6 @@ export class CalendarClient { return calendarClient } - async startSync (): Promise { - try { - await IncomingSyncManager.sync( - this.ctx, - this.accountClient, - this.client, - this.user, - this.user.email, - this.calendar - ) - } catch (err) { - this.ctx.error('Start sync error', { workspace: this.user.workspace, user: this.user.userId, err }) - } - } - private areDatesEqual (first: calendar_v3.Schema$EventDateTime, second: calendar_v3.Schema$EventDateTime): boolean { if (first.date != null && second.date != null) { return new Date(first.date).getTime() === new Date(second.date).getTime() diff --git a/services/calendar/pod-calendar/src/calendarController.ts b/services/calendar/pod-calendar/src/calendarController.ts index 79bace9046..0a648c08cb 100644 --- a/services/calendar/pod-calendar/src/calendarController.ts +++ b/services/calendar/pod-calendar/src/calendarController.ts @@ -68,8 +68,12 @@ export class CalendarController { const integrations = groups.get(info.uuid) ?? [] if (await this.checkWorkspace(info, integrations)) { await limiter.add(async () => { - this.ctx.info('start workspace', { workspace: info.uuid }) - await WorkspaceClient.run(this.ctx, this.accountClient, info.uuid) + try { + this.ctx.info('start workspace', { workspace: info.uuid }) + await WorkspaceClient.run(this.ctx, this.accountClient, info.uuid) + } catch (err) { + this.ctx.error('Failed to start workspace', { workspace: info.uuid, error: err }) + } }) } } diff --git a/services/calendar/pod-calendar/src/client.ts b/services/calendar/pod-calendar/src/client.ts index 530bafab4e..75fcc87fa6 100644 --- a/services/calendar/pod-calendar/src/client.ts +++ b/services/calendar/pod-calendar/src/client.ts @@ -19,7 +19,7 @@ import { setMetadata } from '@hcengineering/platform' import { createClient, getTransactorEndpoint } from '@hcengineering/server-client' export async function getClient (token: string): Promise { - const endpoint = await getTransactorEndpoint(token) + const endpoint = await getTransactorEndpoint(token, 'external') setMetadata(client.metadata.FilterModel, 'client') return await createClient(endpoint, token) } diff --git a/services/calendar/pod-calendar/src/index.ts b/services/calendar/pod-calendar/src/index.ts index 6606ad0d66..9d00642b55 100644 --- a/services/calendar/pod-calendar/src/index.ts +++ b/services/calendar/pod-calendar/src/index.ts @@ -14,7 +14,7 @@ // // eslint-disable-next-line -require('dotenv').config() +import 'dotenv/config' // eslint-disable-next-line -const { main } = require('./main') +import { main } from './main' void main() diff --git a/services/calendar/pod-calendar/src/kvsUtils.ts b/services/calendar/pod-calendar/src/kvsUtils.ts index 1cd0e476b8..f5f368ab1e 100644 --- a/services/calendar/pod-calendar/src/kvsUtils.ts +++ b/services/calendar/pod-calendar/src/kvsUtils.ts @@ -15,14 +15,16 @@ export function getKvsClient (): KeyValueClient { export async function getSyncHistory (workspace: WorkspaceUuid): Promise { const client = getKvsClient() const key = `${CALENDAR_INTEGRATION}:calendarSync:${workspace}` - const res = await client.getValue(key) - return res ?? undefined + try { + const res = await client.getValue(key) + return res ?? undefined + } catch {} } -export async function setSyncHistory (workspace: WorkspaceUuid): Promise { +export async function setSyncHistory (workspace: WorkspaceUuid, value: number): Promise { const client = getKvsClient() const key = `${CALENDAR_INTEGRATION}:calendarSync:${workspace}` - await client.setValue(key, Date.now()) + await client.setValue(key, value) } function calendarsHistoryKey (user: User, email: GoogleEmail): string { diff --git a/services/calendar/pod-calendar/src/outcomingClient.ts b/services/calendar/pod-calendar/src/outcomingClient.ts index 84326cad33..afb0d3069a 100644 --- a/services/calendar/pod-calendar/src/outcomingClient.ts +++ b/services/calendar/pod-calendar/src/outcomingClient.ts @@ -65,7 +65,7 @@ export class OutcomingClient { } else if (type === 'create') { await this.create(event, calendar) } - await setSyncHistory(this.workspace) + await setSyncHistory(this.workspace, Date.now()) } private async convertBody (event: Event): Promise { diff --git a/services/calendar/pod-calendar/src/sync.ts b/services/calendar/pod-calendar/src/sync.ts index dca1006553..7122792f81 100644 --- a/services/calendar/pod-calendar/src/sync.ts +++ b/services/calendar/pod-calendar/src/sync.ts @@ -41,10 +41,23 @@ import setting from '@hcengineering/setting' import { htmlToMarkup } from '@hcengineering/text' import { deepEqual } from 'fast-equals' import { calendar_v3 } from 'googleapis' -import { getCalendarsSyncHistory, getEventHistory, setCalendarsSyncHistory, setEventHistory } from './kvsUtils' +import { getClient } from './client' +import { + getCalendarsSyncHistory, + getEventHistory, + removeUserByEmail, + setCalendarsSyncHistory, + setEventHistory +} from './kvsUtils' import { getRateLimitter, RateLimiter } from './rateLimiter' -import { GoogleEmail, Token, User } from './types' -import { parseRecurrenceStrings } from './utils' +import { CALENDAR_INTEGRATION, GoogleEmail, Token, User } from './types' +import { + getGoogleClient, + getWorkspaceToken, + parseRecurrenceStrings, + removeIntegrationSecret, + setCredentials +} from './utils' import { WatchController } from './watch' const locks = new Map>() @@ -91,7 +104,7 @@ export class IncomingSyncManager { this.systemClient = new TxOperations(client.client, core.account.System) } - static async sync ( + static async initSync ( ctx: MeasureContext, accountClient: AccountClient, client: TxOperations, @@ -108,6 +121,31 @@ export class IncomingSyncManager { } } + static async sync (ctx: MeasureContext, accountClient: AccountClient, user: Token, email: GoogleEmail): Promise { + const client = await getClient(getWorkspaceToken(user.workspace)) + const txOp = new TxOperations(client, user.userId) + const google = getGoogleClient() + const mutex = await lock(`${user.workspace}:${user.userId}:${email}`) + try { + const authSucces = await setCredentials(google.auth, user) + if (!authSucces) { + await removeUserByEmail(user, user.email) + await removeIntegrationSecret(ctx, accountClient, { + socialId: user.userId, + kind: CALENDAR_INTEGRATION, + workspaceUuid: user.workspace, + key: user.email + }) + return + } + const syncManager = new IncomingSyncManager(ctx, accountClient, txOp, user, email, google.google) + await syncManager.startSync() + } finally { + mutex() + await txOp.close() + } + } + private async fillParticipants (): Promise { const personsBySocialId = await getPersonRefsBySocialIds(this.client) const emailSocialIds = await this.client.findAll(contact.class.SocialIdentity, { type: SocialIdType.EMAIL }) diff --git a/services/calendar/pod-calendar/src/workspaceClient.ts b/services/calendar/pod-calendar/src/workspaceClient.ts index f4e5764762..ae052518e5 100644 --- a/services/calendar/pod-calendar/src/workspaceClient.ts +++ b/services/calendar/pod-calendar/src/workspaceClient.ts @@ -28,6 +28,7 @@ import { CalendarClient } from './calendar' import { getClient } from './client' import config from './config' import { addUserByEmail, getSyncHistory, setSyncHistory } from './kvsUtils' +import { IncomingSyncManager } from './sync' import { getWorkspaceTokens } from './tokens' import { GoogleEmail, Token } from './types' import { getWorkspaceToken } from './utils' @@ -37,6 +38,7 @@ export class WorkspaceClient { private readonly calendarsByExternal = new Map() readonly calendarsById = new Map, ExternalCalendar>() readonly participants = new Map, string>() + private lastSync: number = 0 private constructor ( private readonly ctx: MeasureContext, @@ -86,14 +88,16 @@ export class WorkspaceClient { const tokens = await getWorkspaceTokens(this.accountClient, this.workspace) for (const token of tokens) { if (token.workspaceUuid === null) continue - await addUserByEmail(JSON.parse(token.secret), token.key as GoogleEmail) - await this.createCalendarClient(JSON.parse(token.secret)) + const parsedToken = JSON.parse(token.secret) + await addUserByEmail(parsedToken, token.key as GoogleEmail) + await this.createCalendarClient(parsedToken) } await this.getNewEvents() const limiter = new RateLimiter(config.InitLimit) - for (const client of this.clients.values()) { + for (const token of tokens) { await limiter.add(async () => { - await client.startSync() + const parsedToken = JSON.parse(token.secret) + await IncomingSyncManager.sync(this.ctx, this.accountClient, parsedToken, parsedToken.email) }) } } @@ -118,27 +122,43 @@ export class WorkspaceClient { // #region Events - private async getSyncTime (): Promise { - return (await getSyncHistory(this.workspace)) ?? undefined - } - - private async updateSyncTime (): Promise { - await setSyncHistory(this.workspace) + private async updateSyncHistory (): Promise { + try { + await setSyncHistory(this.workspace, this.lastSync) + } catch (err) { + this.ctx.error('Failed to update sync history', { + workspace: this.workspace, + error: err instanceof Error ? err.message : String(err) + }) + } } private async getNewEvents (): Promise { - const lastSync = await this.getSyncTime() + const lastSync = await getSyncHistory(this.workspace) + this.lastSync = lastSync ?? 0 const query = lastSync !== undefined ? { modifiedOn: { $gt: lastSync } } : {} - const newEvents = await this.client.findAll(calendar.class.Event, query) + const newEvents = await this.client.findAll(calendar.class.Event, query, { sort: { modifiedOn: 1 } }) + const interval = setInterval(() => { + void this.updateSyncHistory() + }, 5000) for (const newEvent of newEvents) { - const client = this.getCalendarClientByCalendar(newEvent.calendar as Ref) - if (client === undefined) { - this.ctx.warn('Client not found', { calendar: newEvent.calendar, workspace: this.workspace }) - continue + try { + const client = this.getCalendarClientByCalendar(newEvent.calendar as Ref) + if (client === undefined) { + this.ctx.warn('Client not found', { calendar: newEvent.calendar, workspace: this.workspace }) + continue + } + await client.syncMyEvent(newEvent) + this.lastSync = newEvent.modifiedOn + } catch (err) { + this.ctx.error('Failed to sync event', { + event: newEvent._id, + workspace: this.workspace, + error: err instanceof Error ? err.message : String(err) + }) } - await client.syncMyEvent(newEvent) - await this.updateSyncTime() } + clearInterval(interval) this.ctx.info('all outcoming messages synced', { workspace: this.workspace }) }