Calendar fixes (#8092)

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>
This commit is contained in:
Denis Bykhov
2025-02-25 23:48:29 +07:00
committed by GitHub
parent dcbeb1e043
commit 3f3b3c5b29
8 changed files with 187 additions and 90 deletions
+15 -4
View File
@@ -176,7 +176,14 @@ export class TSessionManager implements SessionManager {
if (this.ticks % (60 * ticksPerSecond) === workspace.tickHash) {
try {
// update account lastVisit every minute per every workspace.∏
void this.getWorkspaceInfo(this.ctx, workspace.token).catch(() => {
let connected: boolean = false
for (const val of workspace.sessions.values()) {
if (val.session.getUser() !== systemAccountEmail) {
connected = true
break
}
}
void this.getWorkspaceInfo(this.ctx, workspace.token, connected).catch(() => {
// Ignore
})
} catch (err: any) {
@@ -284,7 +291,11 @@ export class TSessionManager implements SessionManager {
return this.sessionFactory(token, workspace)
}
async getWorkspaceInfo (ctx: MeasureContext, token: string): Promise<WorkspaceLoginInfo | undefined> {
async getWorkspaceInfo (
ctx: MeasureContext,
token: string,
updateLastVisit: boolean
): Promise<WorkspaceLoginInfo | undefined> {
try {
const userInfo = await (
await fetch(this.accountsUrl, {
@@ -296,7 +307,7 @@ export class TSessionManager implements SessionManager {
},
body: JSON.stringify({
method: 'getWorkspaceInfo',
params: [true]
params: [updateLastVisit]
})
})
).json()
@@ -328,7 +339,7 @@ export class TSessionManager implements SessionManager {
let workspaceInfo: WorkspaceLoginInfo | undefined
try {
workspaceInfo = await this.getWorkspaceInfo(ctx, rawToken)
workspaceInfo = await this.getWorkspaceInfo(ctx, rawToken, token.email !== systemAccountEmail)
} catch (err: any) {
this.updateConnectErrorInfo(token)
return { error: err }
+77 -29
View File
@@ -52,8 +52,8 @@ export class CalendarClient {
private readonly systemTxOp: TxOperations
private readonly activeSync: Record<string, boolean> = {}
private readonly dummyWatches: DummyWatch[] = []
// to do< find!!!!
private readonly googleClient
private readonly googleClient: GoogleClient
private stayAlive: boolean
private inactiveTimer: NodeJS.Timeout
@@ -63,8 +63,10 @@ export class CalendarClient {
private readonly user: User,
private readonly mongo: Db,
client: Client,
private readonly workspace: WorkspaceClient
private readonly workspace: WorkspaceClient,
stayAlive: boolean = false
) {
this.stayAlive = stayAlive
this.client = new TxOperations(client, this.user.userId)
this.systemTxOp = new TxOperations(client, core.account.System)
this.googleClient = new GoogleClient(user, mongo, this)
@@ -92,16 +94,17 @@ export class CalendarClient {
clearTimeout(this.inactiveTimer)
this.inactiveTimer = setTimeout(() => {
this.closeByTimer()
}, 60 * 1000)
}, 30 * 1000)
}
static async create (
user: User | Token,
mongo: Db,
client: Client,
workspace: WorkspaceClient
workspace: WorkspaceClient,
stayAlive: boolean = false
): Promise<CalendarClient> {
const calendarClient = new CalendarClient(user, mongo, client, workspace)
const calendarClient = new CalendarClient(user, mongo, client, workspace, stayAlive)
if (isToken(user)) {
await calendarClient.googleClient.init(user)
calendarClient.updateTimer()
@@ -109,10 +112,23 @@ export class CalendarClient {
return calendarClient
}
async release (): Promise<void> {
this.stayAlive = false
this.updateTimer()
}
async authorize (code: string): Promise<string> {
this.updateTimer()
const me = await this.googleClient.authorize(code)
if (me === undefined) {
const alreadyExistsIntegration = await this.client.findOne(setting.class.Integration, {
type: calendar.integrationType.Calendar,
disabled: false,
value: me
})
if (alreadyExistsIntegration !== undefined) {
throw new Error('Client already exist')
}
const integrations = await this.client.findAll(setting.class.Integration, {
createdBy: this.user.userId,
type: calendar.integrationType.Calendar
@@ -215,6 +231,11 @@ export class CalendarClient {
}
private closeByTimer (): void {
if (this.stayAlive) {
console.log("Couldn't close calendar client, stay alive", this.user.workspace, this.user.userId)
this.updateTimer()
return
}
this.close()
this.workspace.removeClient(this.user.email)
}
@@ -429,13 +450,14 @@ export class CalendarClient {
if (res.data.nextSyncToken != null) {
await this.setEventHistoryId(calendarId, res.data.nextSyncToken)
}
// if resync
} catch (err: any) {
if (err?.response?.status === 410) {
await this.eventsSync(calendarId)
return
}
await this.googleClient.checkError(err)
console.error('Event sync error', this.user.workspace, this.user.userId, err)
console.error('Event sync error', this.user.workspace, this.user.userId, err.message)
}
}
@@ -820,8 +842,7 @@ export class CalendarClient {
}
} catch (err: any) {
await this.googleClient.checkError(err)
// eslint-disable-next-line
throw new Error(`Create event error, ${this.user.workspace}, ${this.user.userId}, ${event._id}, ${err?.message}`)
console.error(`Create event error, ${this.user.workspace}, ${this.user.userId}, ${event._id}, ${err?.message}`)
}
}
@@ -836,12 +857,14 @@ export class CalendarClient {
if (current?.data !== undefined) {
if (current.data.organizer?.self === true) {
const ev = this.applyUpdate(current.data, event, me)
await this.googleClient.rateLimiter.take(1)
await this.calendar.events.update({
calendarId,
eventId: event.eventId,
requestBody: ev
})
if (ev !== undefined) {
await this.googleClient.rateLimiter.take(1)
await this.calendar.events.update({
calendarId,
eventId: event.eventId,
requestBody: ev
})
}
}
}
} catch (err: any) {
@@ -875,8 +898,8 @@ export class CalendarClient {
if (_calendar !== undefined) {
await this.remove(event.eventId, _calendar.externalId)
}
} catch (err) {
console.error('Remove event error', this.user.workspace, this.user.userId, err)
} catch (err: any) {
console.error('Remove event error', this.user.workspace, this.user.userId, err.message)
}
}
@@ -935,12 +958,14 @@ export class CalendarClient {
const current = await this.calendar.events.get({ calendarId, eventId: event.eventId })
if (current !== undefined) {
const ev = this.applyUpdate(current.data, event, me)
await this.googleClient.rateLimiter.take(1)
await this.calendar.events.update({
calendarId,
eventId: event.eventId,
requestBody: ev
})
if (ev !== undefined) {
await this.googleClient.rateLimiter.take(1)
await this.calendar.events.update({
calendarId,
eventId: event.eventId,
requestBody: ev
})
}
}
return true
} catch (err: any) {
@@ -1033,46 +1058,69 @@ export class CalendarClient {
return res
}
private applyUpdate (event: calendar_v3.Schema$Event, current: Event, me: string): calendar_v3.Schema$Event {
private applyUpdate (
event: calendar_v3.Schema$Event,
current: Event,
me: string
): calendar_v3.Schema$Event | undefined {
let res: boolean = false
if (current.title !== event.summary) {
event.summary = current.title
res = true
}
if (current.visibility !== undefined) {
const newVisibility = current.visibility === 'public' ? 'public' : 'private'
if (newVisibility !== event.visibility) {
event.visibility = newVisibility
res = true
}
if (current.visibility === 'freeBusy') {
if (current.visibility === 'freeBusy' && event?.extendedProperties?.private?.visibility !== 'freeBusy') {
event.extendedProperties = {
...event.extendedProperties,
private: {
visibility: 'freeBusy'
}
}
res = true
}
}
const description = htmlToMarkup(event.description ?? '')
if (current.description !== description) {
res = true
event.description = description
}
if (current.location !== event.location) {
res = true
event.location = current.location
}
const attendees = this.getAttendees(current, me)
if (attendees.length > 0 && event.attendees !== undefined) {
for (const attendee of attendees) {
if (event.attendees.findIndex((p) => p.email === attendee) === -1) {
res = true
event.attendees.push({ email: attendee })
}
}
}
event.start = convertDate(current.date, event.start?.date !== undefined, getTimezone(current))
event.end = convertDate(current.dueDate, event.end?.date !== undefined, getTimezone(current))
const newStart = convertDate(current.date, event.start?.date !== undefined, getTimezone(current))
if (!deepEqual(newStart, event.start)) {
res = true
event.start = newStart
}
const newEnd = convertDate(current.dueDate, event.end?.date !== undefined, getTimezone(current))
if (!deepEqual(newEnd, event.end)) {
res = true
event.end = newEnd
}
if (current._class === calendar.class.ReccuringEvent) {
const rec = current as ReccuringEvent
event.recurrence = encodeReccuring(rec.rules, rec.rdate, rec.exdate)
const newRec = encodeReccuring(rec.rules, rec.rdate, rec.exdate)
if (!deepEqual(newRec, event.recurrence)) {
res = true
event.recurrence = newRec
}
}
return event
return res ? event : undefined
}
private getAttendees (event: Event, me: string): string[] {
@@ -29,6 +29,8 @@ export class CalendarController {
WorkspaceClient | Promise<WorkspaceClient>
>()
private readonly syncedWorkspaces = new Set<string>()
private readonly tokens: Collection<Token>
protected static _instance: CalendarController
@@ -68,10 +70,12 @@ export class CalendarController {
const limiter = new RateLimiter(config.InitLimit)
const token = generateToken(systemAccountEmail, { name: '' })
const ids = [...groups.keys()]
console.log('start workspaces', ids)
const infos = await getWorkspacesInfo(token, ids)
console.log('infos', infos)
for (const info of infos) {
infos.sort((a, b) => b.lastVisit - a.lastVisit)
let progress = 0
for (let i = 0; i < infos.length; i++) {
const info = infos[i]
const tokens = groups.get(info.workspaceId)
if (tokens === undefined) {
console.log('no tokens for workspace', info.workspaceId)
@@ -89,10 +93,15 @@ export class CalendarController {
continue
}
await limiter.add(async () => {
console.log('start workspace', info.workspaceId)
const workspace = await this.startWorkspace(info.workspaceId, tokens)
await workspace.sync()
this.syncedWorkspaces.add(info.workspaceId)
})
const newProgress = Math.round((i * 100) / infos.length)
if (newProgress > progress) {
progress = newProgress
console.log(`starting workspaces ${progress}%`)
}
}
}
@@ -104,7 +113,7 @@ export class CalendarController {
console.warn('init client hang', token.workspace, token.userId)
}, 60000)
console.log('init client', token.workspace, token.userId)
await workspaceClient.createCalendarClient(token)
await workspaceClient.createCalendarClient(token, true)
clearTimeout(timeout)
} catch (err) {
console.error(`Couldn't create client for ${workspace} ${token.userId} ${token.email}`)
@@ -116,7 +125,8 @@ export class CalendarController {
async push (email: string, mode: 'events' | 'calendar', calendarId?: string): Promise<void> {
const tokens = await this.tokens.find({ email, access_token: { $exists: true } }).toArray()
const token = generateToken(systemAccountEmail, { name: '' })
const workspaces = [...new Set(tokens.map((p) => p.workspace))]
const workspaces = [...new Set(tokens.map((p) => p.workspace).filter((p) => this.syncedWorkspaces.has(p)))]
if (workspaces.length === 0) return
const infos = await getWorkspacesInfo(token, workspaces)
for (const token of tokens) {
const info = infos.find((p) => p.workspaceId === token.workspace)
@@ -142,8 +152,10 @@ export class CalendarController {
}
async pushEvent (workspace: string, event: Event, type: 'create' | 'update' | 'delete'): Promise<void> {
const workspaceController = await this.getWorkspaceClient(workspace)
await workspaceController.pushEvent(event, type)
if (this.syncedWorkspaces.has(workspace)) {
const workspaceController = await this.getWorkspaceClient(workspace)
await workspaceController.pushEvent(event, type)
}
}
async getUserId (email: string, workspace: string): Promise<Ref<Account>> {
@@ -195,6 +207,9 @@ export class CalendarController {
}
try {
const client = WorkspaceClient.create(this.mongo, workspace, this)
if (this.workspaces.has(workspace)) {
console.error('Workspace already exists', workspace)
}
this.workspaces.set(workspace, client)
const res = await client
this.workspaces.set(workspace, res)
@@ -41,7 +41,7 @@ export class GoogleClient {
private refreshTimer: NodeJS.Timeout | undefined = undefined
readonly rateLimiter = new RateLimiter(1000, 500)
readonly rateLimiter = new RateLimiter(100, 30)
constructor (
private readonly user: User,
@@ -214,7 +214,9 @@ export class GoogleClient {
})
if (current != null) {
await this.rateLimiter.take(1)
await this.calendar.channels.stop({ requestBody: { id: current.channelId, resourceId: current.resourceId } })
try {
await this.calendar.channels.stop({ requestBody: { id: current.channelId, resourceId: current.resourceId } })
} catch {}
}
const channelId = generateId()
const me = await this.getMe()
@@ -230,9 +232,11 @@ export class GoogleClient {
calendarId: null
},
{
channelId,
expired: Number.parseInt(res.data.expiration),
resourceId: res.data.resourceId ?? ''
$set: {
channelId,
expired: Number.parseInt(res.data.expiration),
resourceId: res.data.resourceId ?? ''
}
}
)
} else {
@@ -246,8 +250,8 @@ export class GoogleClient {
})
}
}
} catch (err) {
console.error('Calendar watch error', err)
} catch (err: any) {
console.error('Calendar watch error', err.message)
}
}
@@ -260,9 +264,11 @@ export class GoogleClient {
})
if (current != null) {
await this.rateLimiter.take(1)
await this.calendar.channels.stop({
requestBody: { id: current.channelId, resourceId: current.resourceId }
})
try {
await this.calendar.channels.stop({
requestBody: { id: current.channelId, resourceId: current.resourceId }
})
} catch {}
}
const channelId = generateId()
const me = await this.getMe()
@@ -283,9 +289,11 @@ export class GoogleClient {
calendarId
},
{
channelId,
expired: Number.parseInt(res.data.expiration),
resourceId: res.data.resourceId ?? ''
$set: {
channelId,
expired: Number.parseInt(res.data.expiration),
resourceId: res.data.resourceId ?? ''
}
}
)
} else {
@@ -304,7 +312,7 @@ export class GoogleClient {
if (err?.errors?.[0]?.reason === 'pushNotSupportedForRequestedResource') {
return false
} else {
console.error('Watch error', err)
console.error('Watch error', err.message)
await this.checkError(err)
return false
}
+3 -2
View File
@@ -41,9 +41,10 @@ export const main = async (): Promise<void> => {
const db = await getDB()
const calendarController = CalendarController.getCalendarController(db)
await calendarController.startAll()
const watchController = WatchController.get(db)
watchController.startCheck()
void calendarController.startAll().then(() => {
watchController.startCheck()
})
const endpoints: Endpoint[] = [
{
endpoint: '/signin',
@@ -74,7 +74,6 @@ export function parseRecurrenceStrings (recurrenceStrings: string[]): ReccuringD
}
}
}
console.log('parseRecurrenceStrings', recurrenceStrings, JSON.stringify(res))
return res
}
+12 -11
View File
@@ -14,9 +14,12 @@ export class WatchClient {
private readonly calendar: calendar_v3.Calendar
private readonly user: Token
private me: string = ''
readonly rateLimiter = new RateLimiter(1000, 500)
private constructor (mongo: Db, token: Token) {
private constructor (
mongo: Db,
token: Token,
private readonly rateLimiter: RateLimiter
) {
this.user = token
this.watches = mongo.collection<WatchBase>('watch')
const credentials = JSON.parse(config.Credentials)
@@ -25,8 +28,8 @@ export class WatchClient {
this.calendar = google.calendar({ version: 'v3', auth: this.oAuth2Client })
}
static async Create (mongo: Db, token: Token): Promise<WatchClient> {
const watchClient = new WatchClient(mongo, token)
static async Create (mongo: Db, token: Token, rateLimiter: RateLimiter): Promise<WatchClient> {
const watchClient = new WatchClient(mongo, token, rateLimiter)
await watchClient.init(token)
return watchClient
}
@@ -82,8 +85,7 @@ export class WatchClient {
await this.rateLimiter.take(1)
const res = await this.calendar.calendarList.watch({ requestBody: body })
if (res.data.expiration != null && res.data.resourceId !== null) {
// eslint-disable-next-line
this.watches.updateOne(
await this.watches.updateOne(
{
userId: current.userId,
workspace: current.workspace,
@@ -116,8 +118,7 @@ export class WatchClient {
await this.rateLimiter.take(1)
const res = await this.calendar.events.watch({ calendarId: current.calendarId, requestBody: body })
if (res.data.expiration != null && res.data.resourceId != null) {
// eslint-disable-next-line
this.watches.updateOne(
await this.watches.updateOne(
{
userId: current.userId,
workspace: current.workspace,
@@ -142,6 +143,7 @@ export class WatchClient {
export class WatchController {
private readonly watches: Collection<Watch>
private readonly tokens: Collection<Token>
readonly rateLimiter = new RateLimiter(1000, 500)
private timer: NodeJS.Timeout | undefined = undefined
protected static _instance: WatchController
@@ -149,7 +151,6 @@ export class WatchController {
private constructor (private readonly mongo: Db) {
this.watches = mongo.collection<WatchBase>('watch')
this.tokens = mongo.collection<Token>('tokens')
console.log('watch started')
}
static get (mongo: Db): WatchController {
@@ -164,7 +165,7 @@ export class WatchController {
await this.watches.deleteMany({ userId: user.userId, workspae: user.workspace })
const token = this.tokens.findOne({ user: user.userId, workspace: user.workspace })
if (token == null) return
const watchClient = await WatchClient.Create(this.mongo, user)
const watchClient = await WatchClient.Create(this.mongo, user, this.rateLimiter)
await watchClient.unsubscribe(allWatches)
}
@@ -221,7 +222,7 @@ export class WatchController {
await this.watches.deleteMany({ userId, workspace })
continue
}
const watchClient = await WatchClient.Create(this.mongo, token)
const watchClient = await WatchClient.Create(this.mongo, token, this.rateLimiter)
await watchClient.subscribe(group)
} catch {}
}
@@ -50,6 +50,7 @@ export class WorkspaceClient {
private readonly syncHistory: Collection<SyncHistory>
private readonly tokens: Collection<Token>
private closeTimer: NodeJS.Timeout | undefined = undefined
private channels = new Map<Ref<Channel>, Channel>()
private readonly calendarsByEmail = new Map<string, ExternalCalendar[]>()
readonly calendars = {
@@ -84,11 +85,12 @@ export class WorkspaceClient {
static async create (mongo: Db, workspace: string, serviceController: CalendarController): Promise<WorkspaceClient> {
const instance = new WorkspaceClient(mongo, workspace, serviceController)
console.log('create workspace client', workspace)
await instance.initClient(workspace)
return instance
}
async createCalendarClient (user: User): Promise<CalendarClient> {
async createCalendarClient (user: User, stayAlive: boolean = false): Promise<CalendarClient> {
const current = this.getCalendarClient(user.email)
if (current !== undefined) {
if (current instanceof Promise) {
@@ -96,7 +98,10 @@ export class WorkspaceClient {
}
return current
}
const newClient = CalendarClient.create(user, this.mongo, this.client, this)
const newClient = CalendarClient.create(user, this.mongo, this.client, this, stayAlive)
if (this.clients.has(user.email)) {
console.error('Calendar client already exists', user.workspace, user.userId)
}
this.clients.set(user.email, newClient)
const res = await newClient
this.clients.set(user.email, res)
@@ -161,8 +166,12 @@ export class WorkspaceClient {
removeClient (email: string): void {
this.clients.delete(email)
if (this.clients.size > 0) return
void this.close()
this.serviceController.removeWorkspace(this.workspace)
if (this.closeTimer !== undefined) clearTimeout(this.closeTimer)
this.closeTimer = setTimeout(() => {
if (this.clients.size > 0) return
void this.close()
this.serviceController.removeWorkspace(this.workspace)
}, 20000)
}
private getCalendarClient (email: string): CalendarClient | Promise<CalendarClient> | undefined {
@@ -175,7 +184,6 @@ export class WorkspaceClient {
): Promise<CalendarClient | undefined> {
const calendar = this.calendars.byId.get(id)
if (calendar === undefined) {
console.warn("couldn't find calendar by id", id)
return
}
const client = this.clients.get(calendar.externalUser)
@@ -223,14 +231,19 @@ export class WorkspaceClient {
}
async sync (): Promise<void> {
await this.getNewEvents()
try {
await this.getNewEvents()
} catch (err) {
console.error('sync error', err)
}
const limiter = new RateLimiter(config.InitLimit)
for (let client of this.clients.values()) {
void limiter.add(async () => {
await limiter.add(async () => {
if (client instanceof Promise) {
client = await client
}
await client.startSync()
await client.release()
})
}
}
@@ -244,8 +257,8 @@ export class WorkspaceClient {
return res?.timestamp
}
async updateSyncTime (): Promise<void> {
const timestamp = Date.now()
async updateSyncTime (to: number | undefined = undefined): Promise<void> {
const timestamp = to ?? Date.now()
await this.syncHistory.updateOne(
{
workspace: this.workspace
@@ -262,15 +275,17 @@ export class WorkspaceClient {
async pushEvent (event: Event, type: 'create' | 'update' | 'delete'): Promise<void> {
const client = await this.getCalendarClientByCalendar(event.calendar as Ref<ExternalCalendar>, true)
if (client === undefined) {
console.warn('Client not found', event.calendar, this.workspace)
return
}
if (type === 'delete') {
await client.removeEvent(event)
} else {
await client.syncMyEvent(event)
// if client synced our events just call resync
if (event.access === 'owner' || event.access === 'writer') {
if (type === 'delete') {
await client.removeEvent(event)
} else {
await client.syncMyEvent(event)
}
await this.updateSyncTime()
}
await this.updateSyncTime()
}
async getNewEvents (): Promise<void> {
@@ -283,13 +298,12 @@ export class WorkspaceClient {
for (const newEvent of newEvents) {
const client = await this.getCalendarClientByCalendar(newEvent.calendar as Ref<ExternalCalendar>)
if (client === undefined) {
console.warn('Client not found', newEvent.calendar, this.workspace)
return
continue
}
await client.syncMyEvent(newEvent)
await this.updateSyncTime()
await this.updateSyncTime(newEvent.modifiedOn)
}
console.log('all outcoming messages synced', this.workspace)
await this.updateSyncTime()
}
private async txEventHandler (...txes: Tx[]): Promise<void> {
@@ -297,11 +311,11 @@ export class WorkspaceClient {
switch (tx._class) {
case core.class.TxCreateDoc: {
await this.txCreateEvent(tx as TxCreateDoc<Doc>)
return
continue
}
case core.class.TxUpdateDoc: {
await this.txUpdateEvent(tx as TxUpdateDoc<Event>)
return
continue
}
case core.class.TxRemoveDoc: {
await this.txRemoveEvent(tx as TxRemoveDoc<Doc>)