diff --git a/server/server/src/sessionManager.ts b/server/server/src/sessionManager.ts index f74dc61bf7..500eb3f5d8 100644 --- a/server/server/src/sessionManager.ts +++ b/server/server/src/sessionManager.ts @@ -176,7 +176,14 @@ export class TSessionManager implements SessionManager { if (this.ticks % (60 * ticksPerSecond) === workspace.tickHash) { try { // update account lastVisit every minute per every workspace.∏ - void this.getWorkspaceInfo(this.ctx, workspace.token).catch(() => { + let connected: boolean = false + for (const val of workspace.sessions.values()) { + if (val.session.getUser() !== systemAccountEmail) { + connected = true + break + } + } + void this.getWorkspaceInfo(this.ctx, workspace.token, connected).catch(() => { // Ignore }) } catch (err: any) { @@ -284,7 +291,11 @@ export class TSessionManager implements SessionManager { return this.sessionFactory(token, workspace) } - async getWorkspaceInfo (ctx: MeasureContext, token: string): Promise { + async getWorkspaceInfo ( + ctx: MeasureContext, + token: string, + updateLastVisit: boolean + ): Promise { try { const userInfo = await ( await fetch(this.accountsUrl, { @@ -296,7 +307,7 @@ export class TSessionManager implements SessionManager { }, body: JSON.stringify({ method: 'getWorkspaceInfo', - params: [true] + params: [updateLastVisit] }) }) ).json() @@ -328,7 +339,7 @@ export class TSessionManager implements SessionManager { let workspaceInfo: WorkspaceLoginInfo | undefined try { - workspaceInfo = await this.getWorkspaceInfo(ctx, rawToken) + workspaceInfo = await this.getWorkspaceInfo(ctx, rawToken, token.email !== systemAccountEmail) } catch (err: any) { this.updateConnectErrorInfo(token) return { error: err } diff --git a/services/calendar/pod-calendar/src/calendar.ts b/services/calendar/pod-calendar/src/calendar.ts index 8ffe987ded..a1e99f1169 100644 --- a/services/calendar/pod-calendar/src/calendar.ts +++ b/services/calendar/pod-calendar/src/calendar.ts @@ -52,8 +52,8 @@ export class CalendarClient { private readonly systemTxOp: TxOperations private readonly activeSync: Record = {} private readonly dummyWatches: DummyWatch[] = [] - // to do< find!!!! - private readonly googleClient + private readonly googleClient: GoogleClient + private stayAlive: boolean private inactiveTimer: NodeJS.Timeout @@ -63,8 +63,10 @@ export class CalendarClient { private readonly user: User, private readonly mongo: Db, client: Client, - private readonly workspace: WorkspaceClient + private readonly workspace: WorkspaceClient, + stayAlive: boolean = false ) { + this.stayAlive = stayAlive this.client = new TxOperations(client, this.user.userId) this.systemTxOp = new TxOperations(client, core.account.System) this.googleClient = new GoogleClient(user, mongo, this) @@ -92,16 +94,17 @@ export class CalendarClient { clearTimeout(this.inactiveTimer) this.inactiveTimer = setTimeout(() => { this.closeByTimer() - }, 60 * 1000) + }, 30 * 1000) } static async create ( user: User | Token, mongo: Db, client: Client, - workspace: WorkspaceClient + workspace: WorkspaceClient, + stayAlive: boolean = false ): Promise { - const calendarClient = new CalendarClient(user, mongo, client, workspace) + const calendarClient = new CalendarClient(user, mongo, client, workspace, stayAlive) if (isToken(user)) { await calendarClient.googleClient.init(user) calendarClient.updateTimer() @@ -109,10 +112,23 @@ export class CalendarClient { return calendarClient } + async release (): Promise { + this.stayAlive = false + this.updateTimer() + } + async authorize (code: string): Promise { this.updateTimer() const me = await this.googleClient.authorize(code) if (me === undefined) { + const alreadyExistsIntegration = await this.client.findOne(setting.class.Integration, { + type: calendar.integrationType.Calendar, + disabled: false, + value: me + }) + if (alreadyExistsIntegration !== undefined) { + throw new Error('Client already exist') + } const integrations = await this.client.findAll(setting.class.Integration, { createdBy: this.user.userId, type: calendar.integrationType.Calendar @@ -215,6 +231,11 @@ export class CalendarClient { } private closeByTimer (): void { + if (this.stayAlive) { + console.log("Couldn't close calendar client, stay alive", this.user.workspace, this.user.userId) + this.updateTimer() + return + } this.close() this.workspace.removeClient(this.user.email) } @@ -429,13 +450,14 @@ export class CalendarClient { if (res.data.nextSyncToken != null) { await this.setEventHistoryId(calendarId, res.data.nextSyncToken) } + // if resync } catch (err: any) { if (err?.response?.status === 410) { await this.eventsSync(calendarId) return } await this.googleClient.checkError(err) - console.error('Event sync error', this.user.workspace, this.user.userId, err) + console.error('Event sync error', this.user.workspace, this.user.userId, err.message) } } @@ -820,8 +842,7 @@ export class CalendarClient { } } catch (err: any) { await this.googleClient.checkError(err) - // eslint-disable-next-line - throw new Error(`Create event error, ${this.user.workspace}, ${this.user.userId}, ${event._id}, ${err?.message}`) + console.error(`Create event error, ${this.user.workspace}, ${this.user.userId}, ${event._id}, ${err?.message}`) } } @@ -836,12 +857,14 @@ export class CalendarClient { if (current?.data !== undefined) { if (current.data.organizer?.self === true) { const ev = this.applyUpdate(current.data, event, me) - await this.googleClient.rateLimiter.take(1) - await this.calendar.events.update({ - calendarId, - eventId: event.eventId, - requestBody: ev - }) + if (ev !== undefined) { + await this.googleClient.rateLimiter.take(1) + await this.calendar.events.update({ + calendarId, + eventId: event.eventId, + requestBody: ev + }) + } } } } catch (err: any) { @@ -875,8 +898,8 @@ export class CalendarClient { if (_calendar !== undefined) { await this.remove(event.eventId, _calendar.externalId) } - } catch (err) { - console.error('Remove event error', this.user.workspace, this.user.userId, err) + } catch (err: any) { + console.error('Remove event error', this.user.workspace, this.user.userId, err.message) } } @@ -935,12 +958,14 @@ export class CalendarClient { const current = await this.calendar.events.get({ calendarId, eventId: event.eventId }) if (current !== undefined) { const ev = this.applyUpdate(current.data, event, me) - await this.googleClient.rateLimiter.take(1) - await this.calendar.events.update({ - calendarId, - eventId: event.eventId, - requestBody: ev - }) + if (ev !== undefined) { + await this.googleClient.rateLimiter.take(1) + await this.calendar.events.update({ + calendarId, + eventId: event.eventId, + requestBody: ev + }) + } } return true } catch (err: any) { @@ -1033,46 +1058,69 @@ export class CalendarClient { return res } - private applyUpdate (event: calendar_v3.Schema$Event, current: Event, me: string): calendar_v3.Schema$Event { + private applyUpdate ( + event: calendar_v3.Schema$Event, + current: Event, + me: string + ): calendar_v3.Schema$Event | undefined { + let res: boolean = false if (current.title !== event.summary) { event.summary = current.title + res = true } if (current.visibility !== undefined) { const newVisibility = current.visibility === 'public' ? 'public' : 'private' if (newVisibility !== event.visibility) { event.visibility = newVisibility + res = true } - if (current.visibility === 'freeBusy') { + if (current.visibility === 'freeBusy' && event?.extendedProperties?.private?.visibility !== 'freeBusy') { event.extendedProperties = { ...event.extendedProperties, private: { visibility: 'freeBusy' } } + res = true } } const description = htmlToMarkup(event.description ?? '') if (current.description !== description) { + res = true event.description = description } if (current.location !== event.location) { + res = true event.location = current.location } const attendees = this.getAttendees(current, me) if (attendees.length > 0 && event.attendees !== undefined) { for (const attendee of attendees) { if (event.attendees.findIndex((p) => p.email === attendee) === -1) { + res = true event.attendees.push({ email: attendee }) } } } - event.start = convertDate(current.date, event.start?.date !== undefined, getTimezone(current)) - event.end = convertDate(current.dueDate, event.end?.date !== undefined, getTimezone(current)) + const newStart = convertDate(current.date, event.start?.date !== undefined, getTimezone(current)) + if (!deepEqual(newStart, event.start)) { + res = true + event.start = newStart + } + const newEnd = convertDate(current.dueDate, event.end?.date !== undefined, getTimezone(current)) + if (!deepEqual(newEnd, event.end)) { + res = true + event.end = newEnd + } if (current._class === calendar.class.ReccuringEvent) { const rec = current as ReccuringEvent - event.recurrence = encodeReccuring(rec.rules, rec.rdate, rec.exdate) + const newRec = encodeReccuring(rec.rules, rec.rdate, rec.exdate) + if (!deepEqual(newRec, event.recurrence)) { + res = true + event.recurrence = newRec + } } - return event + return res ? event : undefined } private getAttendees (event: Event, me: string): string[] { diff --git a/services/calendar/pod-calendar/src/calendarController.ts b/services/calendar/pod-calendar/src/calendarController.ts index 9d7d15d16f..18676b487b 100644 --- a/services/calendar/pod-calendar/src/calendarController.ts +++ b/services/calendar/pod-calendar/src/calendarController.ts @@ -29,6 +29,8 @@ export class CalendarController { WorkspaceClient | Promise >() + private readonly syncedWorkspaces = new Set() + private readonly tokens: Collection protected static _instance: CalendarController @@ -68,10 +70,12 @@ export class CalendarController { const limiter = new RateLimiter(config.InitLimit) const token = generateToken(systemAccountEmail, { name: '' }) const ids = [...groups.keys()] - console.log('start workspaces', ids) const infos = await getWorkspacesInfo(token, ids) - console.log('infos', infos) - for (const info of infos) { + infos.sort((a, b) => b.lastVisit - a.lastVisit) + + let progress = 0 + for (let i = 0; i < infos.length; i++) { + const info = infos[i] const tokens = groups.get(info.workspaceId) if (tokens === undefined) { console.log('no tokens for workspace', info.workspaceId) @@ -89,10 +93,15 @@ export class CalendarController { continue } await limiter.add(async () => { - console.log('start workspace', info.workspaceId) const workspace = await this.startWorkspace(info.workspaceId, tokens) await workspace.sync() + this.syncedWorkspaces.add(info.workspaceId) }) + const newProgress = Math.round((i * 100) / infos.length) + if (newProgress > progress) { + progress = newProgress + console.log(`starting workspaces ${progress}%`) + } } } @@ -104,7 +113,7 @@ export class CalendarController { console.warn('init client hang', token.workspace, token.userId) }, 60000) console.log('init client', token.workspace, token.userId) - await workspaceClient.createCalendarClient(token) + await workspaceClient.createCalendarClient(token, true) clearTimeout(timeout) } catch (err) { console.error(`Couldn't create client for ${workspace} ${token.userId} ${token.email}`) @@ -116,7 +125,8 @@ export class CalendarController { async push (email: string, mode: 'events' | 'calendar', calendarId?: string): Promise { const tokens = await this.tokens.find({ email, access_token: { $exists: true } }).toArray() const token = generateToken(systemAccountEmail, { name: '' }) - const workspaces = [...new Set(tokens.map((p) => p.workspace))] + const workspaces = [...new Set(tokens.map((p) => p.workspace).filter((p) => this.syncedWorkspaces.has(p)))] + if (workspaces.length === 0) return const infos = await getWorkspacesInfo(token, workspaces) for (const token of tokens) { const info = infos.find((p) => p.workspaceId === token.workspace) @@ -142,8 +152,10 @@ export class CalendarController { } async pushEvent (workspace: string, event: Event, type: 'create' | 'update' | 'delete'): Promise { - const workspaceController = await this.getWorkspaceClient(workspace) - await workspaceController.pushEvent(event, type) + if (this.syncedWorkspaces.has(workspace)) { + const workspaceController = await this.getWorkspaceClient(workspace) + await workspaceController.pushEvent(event, type) + } } async getUserId (email: string, workspace: string): Promise> { @@ -195,6 +207,9 @@ export class CalendarController { } try { const client = WorkspaceClient.create(this.mongo, workspace, this) + if (this.workspaces.has(workspace)) { + console.error('Workspace already exists', workspace) + } this.workspaces.set(workspace, client) const res = await client this.workspaces.set(workspace, res) diff --git a/services/calendar/pod-calendar/src/googleClient.ts b/services/calendar/pod-calendar/src/googleClient.ts index 7f4c6be317..a07158f948 100644 --- a/services/calendar/pod-calendar/src/googleClient.ts +++ b/services/calendar/pod-calendar/src/googleClient.ts @@ -41,7 +41,7 @@ export class GoogleClient { private refreshTimer: NodeJS.Timeout | undefined = undefined - readonly rateLimiter = new RateLimiter(1000, 500) + readonly rateLimiter = new RateLimiter(100, 30) constructor ( private readonly user: User, @@ -214,7 +214,9 @@ export class GoogleClient { }) if (current != null) { await this.rateLimiter.take(1) - await this.calendar.channels.stop({ requestBody: { id: current.channelId, resourceId: current.resourceId } }) + try { + await this.calendar.channels.stop({ requestBody: { id: current.channelId, resourceId: current.resourceId } }) + } catch {} } const channelId = generateId() const me = await this.getMe() @@ -230,9 +232,11 @@ export class GoogleClient { calendarId: null }, { - channelId, - expired: Number.parseInt(res.data.expiration), - resourceId: res.data.resourceId ?? '' + $set: { + channelId, + expired: Number.parseInt(res.data.expiration), + resourceId: res.data.resourceId ?? '' + } } ) } else { @@ -246,8 +250,8 @@ export class GoogleClient { }) } } - } catch (err) { - console.error('Calendar watch error', err) + } catch (err: any) { + console.error('Calendar watch error', err.message) } } @@ -260,9 +264,11 @@ export class GoogleClient { }) if (current != null) { await this.rateLimiter.take(1) - await this.calendar.channels.stop({ - requestBody: { id: current.channelId, resourceId: current.resourceId } - }) + try { + await this.calendar.channels.stop({ + requestBody: { id: current.channelId, resourceId: current.resourceId } + }) + } catch {} } const channelId = generateId() const me = await this.getMe() @@ -283,9 +289,11 @@ export class GoogleClient { calendarId }, { - channelId, - expired: Number.parseInt(res.data.expiration), - resourceId: res.data.resourceId ?? '' + $set: { + channelId, + expired: Number.parseInt(res.data.expiration), + resourceId: res.data.resourceId ?? '' + } } ) } else { @@ -304,7 +312,7 @@ export class GoogleClient { if (err?.errors?.[0]?.reason === 'pushNotSupportedForRequestedResource') { return false } else { - console.error('Watch error', err) + console.error('Watch error', err.message) await this.checkError(err) return false } diff --git a/services/calendar/pod-calendar/src/main.ts b/services/calendar/pod-calendar/src/main.ts index 2367745f37..5b6073933c 100644 --- a/services/calendar/pod-calendar/src/main.ts +++ b/services/calendar/pod-calendar/src/main.ts @@ -41,9 +41,10 @@ export const main = async (): Promise => { const db = await getDB() const calendarController = CalendarController.getCalendarController(db) - await calendarController.startAll() const watchController = WatchController.get(db) - watchController.startCheck() + void calendarController.startAll().then(() => { + watchController.startCheck() + }) const endpoints: Endpoint[] = [ { endpoint: '/signin', diff --git a/services/calendar/pod-calendar/src/utils.ts b/services/calendar/pod-calendar/src/utils.ts index fa7d70185d..1238363a79 100644 --- a/services/calendar/pod-calendar/src/utils.ts +++ b/services/calendar/pod-calendar/src/utils.ts @@ -74,7 +74,6 @@ export function parseRecurrenceStrings (recurrenceStrings: string[]): ReccuringD } } } - console.log('parseRecurrenceStrings', recurrenceStrings, JSON.stringify(res)) return res } diff --git a/services/calendar/pod-calendar/src/watch.ts b/services/calendar/pod-calendar/src/watch.ts index 940343e552..6229cc83e1 100644 --- a/services/calendar/pod-calendar/src/watch.ts +++ b/services/calendar/pod-calendar/src/watch.ts @@ -14,9 +14,12 @@ export class WatchClient { private readonly calendar: calendar_v3.Calendar private readonly user: Token private me: string = '' - readonly rateLimiter = new RateLimiter(1000, 500) - private constructor (mongo: Db, token: Token) { + private constructor ( + mongo: Db, + token: Token, + private readonly rateLimiter: RateLimiter + ) { this.user = token this.watches = mongo.collection('watch') const credentials = JSON.parse(config.Credentials) @@ -25,8 +28,8 @@ export class WatchClient { this.calendar = google.calendar({ version: 'v3', auth: this.oAuth2Client }) } - static async Create (mongo: Db, token: Token): Promise { - const watchClient = new WatchClient(mongo, token) + static async Create (mongo: Db, token: Token, rateLimiter: RateLimiter): Promise { + const watchClient = new WatchClient(mongo, token, rateLimiter) await watchClient.init(token) return watchClient } @@ -82,8 +85,7 @@ export class WatchClient { await this.rateLimiter.take(1) const res = await this.calendar.calendarList.watch({ requestBody: body }) if (res.data.expiration != null && res.data.resourceId !== null) { - // eslint-disable-next-line - this.watches.updateOne( + await this.watches.updateOne( { userId: current.userId, workspace: current.workspace, @@ -116,8 +118,7 @@ export class WatchClient { await this.rateLimiter.take(1) const res = await this.calendar.events.watch({ calendarId: current.calendarId, requestBody: body }) if (res.data.expiration != null && res.data.resourceId != null) { - // eslint-disable-next-line - this.watches.updateOne( + await this.watches.updateOne( { userId: current.userId, workspace: current.workspace, @@ -142,6 +143,7 @@ export class WatchClient { export class WatchController { private readonly watches: Collection private readonly tokens: Collection + readonly rateLimiter = new RateLimiter(1000, 500) private timer: NodeJS.Timeout | undefined = undefined protected static _instance: WatchController @@ -149,7 +151,6 @@ export class WatchController { private constructor (private readonly mongo: Db) { this.watches = mongo.collection('watch') this.tokens = mongo.collection('tokens') - console.log('watch started') } static get (mongo: Db): WatchController { @@ -164,7 +165,7 @@ export class WatchController { await this.watches.deleteMany({ userId: user.userId, workspae: user.workspace }) const token = this.tokens.findOne({ user: user.userId, workspace: user.workspace }) if (token == null) return - const watchClient = await WatchClient.Create(this.mongo, user) + const watchClient = await WatchClient.Create(this.mongo, user, this.rateLimiter) await watchClient.unsubscribe(allWatches) } @@ -221,7 +222,7 @@ export class WatchController { await this.watches.deleteMany({ userId, workspace }) continue } - const watchClient = await WatchClient.Create(this.mongo, token) + const watchClient = await WatchClient.Create(this.mongo, token, this.rateLimiter) await watchClient.subscribe(group) } catch {} } diff --git a/services/calendar/pod-calendar/src/workspaceClient.ts b/services/calendar/pod-calendar/src/workspaceClient.ts index ddb995e45c..ade9e4f5af 100644 --- a/services/calendar/pod-calendar/src/workspaceClient.ts +++ b/services/calendar/pod-calendar/src/workspaceClient.ts @@ -50,6 +50,7 @@ export class WorkspaceClient { private readonly syncHistory: Collection private readonly tokens: Collection + private closeTimer: NodeJS.Timeout | undefined = undefined private channels = new Map, Channel>() private readonly calendarsByEmail = new Map() readonly calendars = { @@ -84,11 +85,12 @@ export class WorkspaceClient { static async create (mongo: Db, workspace: string, serviceController: CalendarController): Promise { const instance = new WorkspaceClient(mongo, workspace, serviceController) + console.log('create workspace client', workspace) await instance.initClient(workspace) return instance } - async createCalendarClient (user: User): Promise { + async createCalendarClient (user: User, stayAlive: boolean = false): Promise { const current = this.getCalendarClient(user.email) if (current !== undefined) { if (current instanceof Promise) { @@ -96,7 +98,10 @@ export class WorkspaceClient { } return current } - const newClient = CalendarClient.create(user, this.mongo, this.client, this) + const newClient = CalendarClient.create(user, this.mongo, this.client, this, stayAlive) + if (this.clients.has(user.email)) { + console.error('Calendar client already exists', user.workspace, user.userId) + } this.clients.set(user.email, newClient) const res = await newClient this.clients.set(user.email, res) @@ -161,8 +166,12 @@ export class WorkspaceClient { removeClient (email: string): void { this.clients.delete(email) if (this.clients.size > 0) return - void this.close() - this.serviceController.removeWorkspace(this.workspace) + if (this.closeTimer !== undefined) clearTimeout(this.closeTimer) + this.closeTimer = setTimeout(() => { + if (this.clients.size > 0) return + void this.close() + this.serviceController.removeWorkspace(this.workspace) + }, 20000) } private getCalendarClient (email: string): CalendarClient | Promise | undefined { @@ -175,7 +184,6 @@ export class WorkspaceClient { ): Promise { const calendar = this.calendars.byId.get(id) if (calendar === undefined) { - console.warn("couldn't find calendar by id", id) return } const client = this.clients.get(calendar.externalUser) @@ -223,14 +231,19 @@ export class WorkspaceClient { } async sync (): Promise { - await this.getNewEvents() + try { + await this.getNewEvents() + } catch (err) { + console.error('sync error', err) + } const limiter = new RateLimiter(config.InitLimit) for (let client of this.clients.values()) { - void limiter.add(async () => { + await limiter.add(async () => { if (client instanceof Promise) { client = await client } await client.startSync() + await client.release() }) } } @@ -244,8 +257,8 @@ export class WorkspaceClient { return res?.timestamp } - async updateSyncTime (): Promise { - const timestamp = Date.now() + async updateSyncTime (to: number | undefined = undefined): Promise { + const timestamp = to ?? Date.now() await this.syncHistory.updateOne( { workspace: this.workspace @@ -262,15 +275,17 @@ export class WorkspaceClient { async pushEvent (event: Event, type: 'create' | 'update' | 'delete'): Promise { const client = await this.getCalendarClientByCalendar(event.calendar as Ref, true) if (client === undefined) { - console.warn('Client not found', event.calendar, this.workspace) return } - if (type === 'delete') { - await client.removeEvent(event) - } else { - await client.syncMyEvent(event) + // if client synced our events just call resync + if (event.access === 'owner' || event.access === 'writer') { + if (type === 'delete') { + await client.removeEvent(event) + } else { + await client.syncMyEvent(event) + } + await this.updateSyncTime() } - await this.updateSyncTime() } async getNewEvents (): Promise { @@ -283,13 +298,12 @@ export class WorkspaceClient { for (const newEvent of newEvents) { const client = await this.getCalendarClientByCalendar(newEvent.calendar as Ref) if (client === undefined) { - console.warn('Client not found', newEvent.calendar, this.workspace) - return + continue } await client.syncMyEvent(newEvent) - await this.updateSyncTime() + await this.updateSyncTime(newEvent.modifiedOn) } - console.log('all outcoming messages synced', this.workspace) + await this.updateSyncTime() } private async txEventHandler (...txes: Tx[]): Promise { @@ -297,11 +311,11 @@ export class WorkspaceClient { switch (tx._class) { case core.class.TxCreateDoc: { await this.txCreateEvent(tx as TxCreateDoc) - return + continue } case core.class.TxUpdateDoc: { await this.txUpdateEvent(tx as TxUpdateDoc) - return + continue } case core.class.TxRemoveDoc: { await this.txRemoveEvent(tx as TxRemoveDoc)