Calendar extra logs (#9425)

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

services/calendar/pod-calendar/src/auth.ts

@@ -32,10 +32,11 @@ import { calendar_v3, google } from 'googleapis'
 import { encode64 } from './base64'
 import { getClient } from './client'
 import { addUserByEmail, removeUserByEmail, setSyncHistory } from './kvsUtils'
-import { IncomingSyncManager, lock } from './sync'
+import { IncomingSyncManager } from './sync'
 import { CALENDAR_INTEGRATION, GoogleEmail, SCOPES, State, Token, User } from './types'
 import { getGoogleClient, getWorkspaceToken, removeIntegrationSecret } from './utils'
 import { WatchController } from './watch'
+import { lock } from './mutex'
 
 interface AuthResult {
   success: boolean

services/calendar/pod-calendar/src/calendar.ts

@@ -16,7 +16,7 @@
 import { AccountClient } from '@hcengineering/account-client'
 import calendar, { Event, ExternalCalendar, ReccuringEvent, ReccuringInstance } from '@hcengineering/calendar'
 import { Client, MeasureContext, Ref, TxOperations } from '@hcengineering/core'
-import { jsonToHTML, markupToJSON } from '@hcengineering/text'
+import { areEqualMarkups, htmlToMarkup, isEmptyMarkup, jsonToHTML, markupToJSON } from '@hcengineering/text'
 import { deepEqual } from 'fast-equals'
 import { OAuth2Client } from 'google-auth-library'
 import { calendar_v3 } from 'googleapis'
@@ -29,6 +29,8 @@ import {
   getGoogleClient,
   getMixinFields,
   getTimezone,
+  parseEventDate,
+  parseRecurrenceStrings,
   removeIntegrationSecret,
   setCredentials
 } from './utils'
@@ -209,6 +211,9 @@ export class CalendarClient {
         const current = await this.calendar.events.get({ calendarId, eventId: event.eventId })
         if (current !== undefined && current.data.status !== 'cancelled') {
           const ev = this.applyUpdate(current.data, event)
+          if (ev === undefined) {
+            return true // No changes to apply
+          }
           await this.rateLimiter.take(1)
           await this.calendar.events.update({
             calendarId,
@@ -292,6 +297,15 @@ export class CalendarClient {
   private applyUpdate (event: calendar_v3.Schema$Event, current: Event): calendar_v3.Schema$Event | undefined {
     let res: boolean = false
     if (current.title !== event.summary) {
+      this.ctx.info('Update event diff: title', {
+        calendarId: current.calendar,
+        eventId: current.eventId,
+        user: this.user.email,
+        workspace: this.user.workspace,
+        event: current._id,
+        prev: event.summary,
+        current: current.title
+      })
       event.summary = current.title
       res = true
     }
@@ -299,6 +313,15 @@ export class CalendarClient {
       const newVisibility = current.visibility === 'public' ? 'public' : 'private'
       if (newVisibility !== event.visibility) {
         event.visibility = newVisibility
+        this.ctx.info('Update event diff: visibility', {
+          calendarId: current.calendar,
+          eventId: current.eventId,
+          user: this.user.email,
+          workspace: this.user.workspace,
+          event: current._id,
+          prev: event.visibility,
+          current: newVisibility
+        })
         res = true
       }
       if (current.visibility === 'freeBusy' && event?.extendedProperties?.private?.visibility !== 'freeBusy') {
@@ -312,12 +335,34 @@ export class CalendarClient {
       }
     }
     const description = jsonToHTML(markupToJSON(current.description))
-    if ((event.description ?? '') !== description) {
+    const originMarkup = htmlToMarkup(event.description ?? '')
+    if (
+      isEmptyMarkup(description) !== isEmptyMarkup(originMarkup) &&
+      !areEqualMarkups(current.description, originMarkup)
+    ) {
       res = true
+      this.ctx.info('Update event diff: description', {
+        calendarId: current.calendar,
+        eventId: current.eventId,
+        user: this.user.email,
+        workspace: this.user.workspace,
+        event: current._id,
+        prev: event.description,
+        current: description
+      })
       event.description = description
     }
     if (current.location !== event.location) {
       res = true
+      this.ctx.info('Update event diff: location', {
+        calendarId: current.calendar,
+        eventId: current.eventId,
+        user: this.user.email,
+        workspace: this.user.workspace,
+        event: current._id,
+        prev: event.location,
+        current: current.location
+      })
       event.location = current.location
     }
     const attendees = this.getAttendees(current)
@@ -325,25 +370,68 @@ export class CalendarClient {
       for (const attendee of attendees) {
         if (event.attendees.findIndex((p) => p.email === attendee) === -1) {
           res = true
+          this.ctx.info('Update event diff: attendees', {
+            calendarId: current.calendar,
+            eventId: current.eventId,
+            user: this.user.email,
+            workspace: this.user.workspace,
+            event: current._id,
+            prev: event.attendees,
+            current: attendee
+          })
           event.attendees.push({ email: attendee })
         }
       }
     }
-    const newStart = convertDate(current.date, event.start?.date !== undefined, getTimezone(current))
-    if (!deepEqual(newStart, event.start)) {
+    const currentStart = parseEventDate(event.start)
+    if (currentStart !== current.date) {
+      const newStart = convertDate(current.date, event.start?.date !== undefined, getTimezone(current))
       res = true
+      this.ctx.info('Update event diff: start', {
+        calendarId: current.calendar,
+        eventId: current.eventId,
+        user: this.user.email,
+        workspace: this.user.workspace,
+        event: current._id,
+        prev: event.start,
+        current: newStart
+      })
       event.start = newStart
     }
-    const newEnd = convertDate(current.dueDate, event.end?.date !== undefined, getTimezone(current))
-    if (!deepEqual(newEnd, event.end)) {
+    const currentEnd = parseEventDate(event.end)
+    if (currentEnd !== current.dueDate) {
       res = true
+      const newEnd = convertDate(current.dueDate, event.end?.date !== undefined, getTimezone(current))
+      this.ctx.info('Update event diff: end', {
+        calendarId: current.calendar,
+        eventId: current.eventId,
+        user: this.user.email,
+        workspace: this.user.workspace,
+        event: current._id,
+        prev: event.end,
+        current: newEnd
+      })
       event.end = newEnd
     }
     if (current._class === calendar.class.ReccuringEvent) {
       const rec = current as ReccuringEvent
-      const newRec = encodeReccuring(rec.rules, rec.rdate, rec.exdate)
-      if (!deepEqual(newRec, event.recurrence)) {
+      const parsed = parseRecurrenceStrings(event.recurrence ?? [])
+      if (
+        !deepEqual(rec.rules, parsed.rules) ||
+        !deepEqual(rec.rdate, parsed.rdate) ||
+        !deepEqual(rec.exdate, parsed.exdate)
+      ) {
         res = true
+        const newRec = encodeReccuring(rec.rules, rec.rdate, rec.exdate)
+        this.ctx.info('Update event diff: recurrence', {
+          calendarId: current.calendar,
+          eventId: current.eventId,
+          user: this.user.email,
+          workspace: this.user.workspace,
+          event: current._id,
+          prev: event.recurrence,
+          current: newRec
+        })
         event.recurrence = newRec
       }
     }

services/calendar/pod-calendar/src/mutex.ts

@@ -0,0 +1,30 @@
+import { WorkspaceUuid } from '@hcengineering/core'
+
+export const synced = new Set<WorkspaceUuid>()
+
+const locks = new Map<string, Promise<void>>()
+
+export async function lock (key: string): Promise<() => void> {
+  // Wait for any existing lock to be released
+  const currentLock = locks.get(key)
+  if (currentLock != null) {
+    await currentLock
+  }
+
+  // Create a new lock
+  let releaseFn!: () => void
+  const newLock = new Promise<void>((resolve) => {
+    releaseFn = resolve
+  })
+
+  // Store the lock
+  locks.set(key, newLock)
+
+  // Return the release function
+  return () => {
+    if (locks.get(key) === newLock) {
+      locks.delete(key)
+    }
+    releaseFn()
+  }
+}

services/calendar/pod-calendar/src/outcomingClient.ts

@@ -17,7 +17,7 @@ import { AccountClient } from '@hcengineering/account-client'
 import calendar, { Event, ExternalCalendar, ReccuringEvent, ReccuringInstance } from '@hcengineering/calendar'
 import contact, { Contact } from '@hcengineering/contact'
 import core, { MeasureContext, Ref, SocialIdType, TxOperations, WorkspaceUuid } from '@hcengineering/core'
-import { jsonToHTML, markupToJSON } from '@hcengineering/text'
+import { areEqualMarkups, htmlToMarkup, isEmptyMarkup, jsonToHTML, markupToJSON } from '@hcengineering/text'
 import { deepEqual } from 'fast-equals'
 import { OAuth2Client } from 'google-auth-library'
 import { calendar_v3 } from 'googleapis'
@@ -32,10 +32,12 @@ import {
   getMixinFields,
   getTimezone,
   getWorkspaceToken,
+  parseEventDate,
+  parseRecurrenceStrings,
   removeIntegrationSecret,
   setCredentials
 } from './utils'
-import { synced } from './sync'
+import { lock, synced } from './mutex'
 
 export class OutcomingClient {
   private readonly calendar: calendar_v3.Calendar
@@ -57,6 +59,14 @@ export class OutcomingClient {
   async push (event: Event, type: 'create' | 'update' | 'delete'): Promise<void> {
     const calendar = await this.getCalendar(event)
     if (calendar === undefined) return
+    this.ctx.info('Push outcoming event', {
+      calendarId: calendar.externalId,
+      eventId: event.eventId,
+      user: this.user.email,
+      workspace: this.workspace,
+      event: event._id,
+      type
+    })
     if (type === 'delete') {
       await this.remove(event.eventId, calendar.externalId)
     } else if (type === 'update') {
@@ -151,7 +161,25 @@ export class OutcomingClient {
         await this.rateLimiter.take(1)
         const current = await this.calendar.events.get({ calendarId, eventId: event.eventId })
         if (current !== undefined && current.data.status !== 'cancelled') {
+          this.ctx.info('Update event', {
+            calendarId,
+            eventId: event.eventId,
+            user: this.user.email,
+            workspace: this.workspace,
+            event: event._id,
+            current: current.data.id
+          })
           const ev = await this.applyUpdate(current.data, event)
+          if (ev === undefined) {
+            this.ctx.info('No changes to update', {
+              calendarId,
+              eventId: event.eventId,
+              user: this.user.email,
+              workspace: this.workspace,
+              event: event._id
+            })
+            return true
+          }
           await this.rateLimiter.take(1)
           await this.calendar.events.update({
             calendarId,
@@ -177,12 +205,30 @@ export class OutcomingClient {
   ): Promise<calendar_v3.Schema$Event | undefined> {
     let res: boolean = false
     if (current.title !== event.summary) {
+      this.ctx.info('Update event diff: title', {
+        calendarId: current.calendar,
+        eventId: current.eventId,
+        user: this.user.email,
+        workspace: this.workspace,
+        event: current._id,
+        prev: event.summary,
+        current: current.title
+      })
       event.summary = current.title
       res = true
     }
     if (current.visibility !== undefined) {
       const newVisibility = current.visibility === 'public' ? 'public' : 'private'
       if (newVisibility !== event.visibility) {
+        this.ctx.info('Update event diff: visibility', {
+          calendarId: current.calendar,
+          eventId: current.eventId,
+          user: this.user.email,
+          workspace: this.workspace,
+          event: current._id,
+          prev: event.visibility,
+          current: newVisibility
+        })
         event.visibility = newVisibility
         res = true
       }
@@ -197,12 +243,34 @@ export class OutcomingClient {
       }
     }
     const description = jsonToHTML(markupToJSON(current.description))
-    if ((event.description ?? '') !== description) {
+    const originMarkup = htmlToMarkup(event.description ?? '')
+    if (
+      isEmptyMarkup(description) !== isEmptyMarkup(originMarkup) &&
+      !areEqualMarkups(current.description, originMarkup)
+    ) {
       res = true
+      this.ctx.info('Update event diff: description', {
+        calendarId: current.calendar,
+        eventId: current.eventId,
+        user: this.user.email,
+        workspace: this.workspace,
+        event: current._id,
+        prev: event.description,
+        current: description
+      })
       event.description = description
     }
     if (current.location !== event.location) {
       res = true
+      this.ctx.info('Update event diff: location', {
+        calendarId: current.calendar,
+        eventId: current.eventId,
+        user: this.user.email,
+        workspace: this.workspace,
+        event: current._id,
+        prev: event.location,
+        current: current.location
+      })
       event.location = current.location
     }
     const attendees = await this.getAttendees(current)
@@ -210,25 +278,68 @@ export class OutcomingClient {
       for (const attendee of attendees) {
         if (event.attendees.findIndex((p) => p.email === attendee) === -1) {
           res = true
+          this.ctx.info('Update event diff: attendees', {
+            calendarId: current.calendar,
+            eventId: current.eventId,
+            user: this.user.email,
+            workspace: this.workspace,
+            event: current._id,
+            prev: event.attendees,
+            current: attendee
+          })
           event.attendees.push({ email: attendee })
         }
       }
     }
-    const newStart = convertDate(current.date, event.start?.date !== undefined, getTimezone(current))
-    if (!deepEqual(newStart, event.start)) {
+    const currentStart = parseEventDate(event.start)
+    if (currentStart !== current.date) {
       res = true
+      const newStart = convertDate(current.date, event.start?.date !== undefined, getTimezone(current))
+      this.ctx.info('Update event diff: start', {
+        calendarId: current.calendar,
+        eventId: current.eventId,
+        user: this.user.email,
+        workspace: this.workspace,
+        event: current._id,
+        prev: event.start,
+        current: newStart
+      })
       event.start = newStart
     }
-    const newEnd = convertDate(current.dueDate, event.end?.date !== undefined, getTimezone(current))
-    if (!deepEqual(newEnd, event.end)) {
+    const currentEnd = parseEventDate(event.end)
+    if (currentEnd !== current.dueDate) {
       res = true
+      const newEnd = convertDate(current.dueDate, event.end?.date !== undefined, getTimezone(current))
+      this.ctx.info('Update event diff: end', {
+        calendarId: current.calendar,
+        eventId: current.eventId,
+        user: this.user.email,
+        workspace: this.workspace,
+        event: current._id,
+        prev: event.end,
+        current: newEnd
+      })
       event.end = newEnd
     }
     if (current._class === calendar.class.ReccuringEvent) {
       const rec = current as ReccuringEvent
-      const newRec = encodeReccuring(rec.rules, rec.rdate, rec.exdate)
-      if (!deepEqual(newRec, event.recurrence)) {
+      const parsed = parseRecurrenceStrings(event.recurrence ?? [])
+      if (
+        !deepEqual(rec.rules, parsed.rules) ||
+        !deepEqual(rec.rdate, parsed.rdate) ||
+        !deepEqual(rec.exdate, parsed.exdate)
+      ) {
         res = true
+        const newRec = encodeReccuring(rec.rules, rec.rdate, rec.exdate)
+        this.ctx.info('Update event diff: recurrence', {
+          calendarId: current.calendar,
+          eventId: current.eventId,
+          user: this.user.email,
+          workspace: this.workspace,
+          event: current._id,
+          prev: event.recurrence,
+          current: newRec
+        })
         event.recurrence = newRec
       }
     }
@@ -267,10 +378,12 @@ export class OutcomingClient {
     if (current?.data !== undefined) {
       if (current.data.organizer?.self === true) {
         await this.rateLimiter.take(1)
-        await this.calendar.events.delete({
-          eventId,
-          calendarId
-        })
+        try {
+          await this.calendar.events.delete({
+            eventId,
+            calendarId
+          })
+        } catch {}
       }
     }
   }
@@ -289,6 +402,7 @@ export class OutcomingClient {
     type: 'create' | 'update' | 'delete'
   ): Promise<void> {
     if (event.access === 'owner' || event.access === 'writer') {
+      const mutex = await lock(`outcoming:${workspace}`)
       const client = await getClient(getWorkspaceToken(workspace))
       const txOp = new TxOperations(client, core.account.System)
       try {
@@ -299,6 +413,11 @@ export class OutcomingClient {
         const calendarClient = new OutcomingClient(ctx, workspace, txOp, user)
         const authSucces = await setCredentials(calendarClient.oAuth2Client, user)
         if (!authSucces) {
+          ctx.warn('OutcomingClient push: remove user', {
+            user: user.userId,
+            workspace: user.workspace,
+            email: user.email
+          })
           await removeUserByEmail(user, user.email)
           await removeIntegrationSecret(ctx, accountClient, {
             socialId: user.userId,
@@ -311,6 +430,7 @@ export class OutcomingClient {
         await calendarClient.push(event, type)
       } finally {
         await txOp.close()
+        mutex()
       }
     }
   }

services/calendar/pod-calendar/src/sync.ts

@@ -35,8 +35,7 @@ import core, {
   Ref,
   SocialIdType,
   TxOperations,
-  TxProcessor,
-  WorkspaceUuid
+  TxProcessor
 } from '@hcengineering/core'
 import setting from '@hcengineering/setting'
 import { htmlToMarkup } from '@hcengineering/text'
@@ -50,46 +49,19 @@ import {
   setCalendarsSyncHistory,
   setEventHistory
 } from './kvsUtils'
+import { lock } from './mutex'
 import { getRateLimitter, RateLimiter } from './rateLimiter'
 import { CALENDAR_INTEGRATION, GoogleEmail, Token, User } from './types'
 import {
   getGoogleClient,
   getWorkspaceToken,
+  parseEventDate,
   parseRecurrenceStrings,
   removeIntegrationSecret,
   setCredentials
 } from './utils'
 import { WatchController } from './watch'
 
-export const synced = new Set<WorkspaceUuid>()
-
-const locks = new Map<string, Promise<void>>()
-
-export async function lock (key: string): Promise<() => void> {
-  // Wait for any existing lock to be released
-  const currentLock = locks.get(key)
-  if (currentLock != null) {
-    await currentLock
-  }
-
-  // Create a new lock
-  let releaseFn!: () => void
-  const newLock = new Promise<void>((resolve) => {
-    releaseFn = resolve
-  })
-
-  // Store the lock
-  locks.set(key, newLock)
-
-  // Return the release function
-  return () => {
-    if (locks.get(key) === newLock) {
-      locks.delete(key)
-    }
-    releaseFn()
-  }
-}
-
 export class IncomingSyncManager {
   private readonly rateLimiter: RateLimiter
   private calendars: ExternalCalendar[] = []
@@ -346,7 +318,7 @@ export class IncomingSyncManager {
         {
           ...data,
           recurringEventId: event.recurringEventId as Ref<ReccuringEvent>,
-          originalStartTime: parseDate(event.originalStartTime),
+          originalStartTime: parseEventDate(event.originalStartTime),
           isCancelled: event.status === 'cancelled'
         },
         current as ReccuringInstance
@@ -431,8 +403,8 @@ export class IncomingSyncManager {
   ): Promise<AttachedData<Event>> {
     const participants = await this.getParticipants(event)
     const res: AttachedData<Event> = {
-      date: parseDate(event.start),
-      dueDate: parseDate(event.end),
+      date: parseEventDate(event.start),
+      dueDate: parseEventDate(event.end),
       allDay: event.start?.date != null,
       description: htmlToMarkup(event.description ?? ''),
       title: event.summary ?? '',
@@ -531,10 +503,10 @@ export class IncomingSyncManager {
       res.title = event.summary
     }
     if (event.start != null) {
-      res.date = parseDate(event.start)
+      res.date = parseEventDate(event.start)
     }
     if (event.end != null) {
-      res.dueDate = parseDate(event.end)
+      res.dueDate = parseEventDate(event.end)
     }
     if (event.visibility != null && event.visibility !== 'default') {
       res.visibility =
@@ -566,7 +538,7 @@ export class IncomingSyncManager {
         {
           ...data,
           recurringEventId: event.recurringEventId,
-          originalStartTime: parseDate(event.originalStartTime),
+          originalStartTime: parseEventDate(event.originalStartTime),
           isCancelled: event.status === 'cancelled',
           rules: parseRule.rules,
           exdate: parseRule.exdate,
@@ -741,13 +713,3 @@ export class IncomingSyncManager {
     }
   }
 }
-
-function parseDate (date: calendar_v3.Schema$EventDateTime | undefined): number {
-  if (date?.dateTime != null) {
-    return new Date(date.dateTime).getTime()
-  }
-  if (date?.date != null) {
-    return new Date(date.date).getTime()
-  }
-  return 0
-}

services/calendar/pod-calendar/src/utils.ts

@@ -353,3 +353,13 @@ export function getMixinFields (h: Hierarchy, event: Event): Record<string, any>
 
   return res
 }
+
+export function parseEventDate (date: calendar_v3.Schema$EventDateTime | undefined): number {
+  if (date?.dateTime != null) {
+    return new Date(date.dateTime).getTime()
+  }
+  if (date?.date != null) {
+    return new Date(date.date).getTime()
+  }
+  return 0
+}

services/calendar/pod-calendar/src/workspaceClient.ts

@@ -28,10 +28,11 @@ import { CalendarClient } from './calendar'
 import { getClient } from './client'
 import config from './config'
 import { addUserByEmail, getSyncHistory, setSyncHistory } from './kvsUtils'
-import { IncomingSyncManager, synced } from './sync'
+import { IncomingSyncManager } from './sync'
 import { getWorkspaceTokens } from './tokens'
 import { GoogleEmail, Token } from './types'
 import { getWorkspaceToken } from './utils'
+import { synced } from './mutex'
 
 export class WorkspaceClient {
   private readonly clients = new Map<GoogleEmail, CalendarClient>()