diff --git a/dev/tool/src/github.ts b/dev/tool/src/github.ts index ac4156b0d9..e96c220f5d 100644 --- a/dev/tool/src/github.ts +++ b/dev/tool/src/github.ts @@ -1,17 +1,22 @@ import core, { buildSocialIdString, DOMAIN_MODEL_TX, + isArchivingMode, + isDeletingMode, + MeasureMetricsContext, systemAccountUuid, TxProcessor, - type BackupClient, - type Client, type Doc, + type PersonId, type Ref, + type Tx, type TxCUD, type WorkspaceUuid } from '@hcengineering/core' import { getAccountsFromTxes, getSocialKeyByOldEmail } from '@hcengineering/model-core' -import { createClient, getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client' +import { getAccountClient } from '@hcengineering/server-client' +import { createDummyStorageAdapter, wrapPipeline, type PipelineFactory } from '@hcengineering/server-core' +import { createBackupPipeline } from '@hcengineering/server-pipeline' import { generateToken } from '@hcengineering/server-token' import type { Db } from 'mongodb' @@ -42,7 +47,12 @@ export interface GithubUserRecord { accounts: Record */> } -export async function performGithubAccountMigrations (db: Db, region: string | null): Promise { +export async function performGithubAccountMigrations ( + db: Db, + dbUrl: string, + txes: Tx[], + region: string | null +): Promise { const token = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'admin', admin: 'true' }) const githubToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'github' }) const accountClient = getAccountClient(token) @@ -74,19 +84,30 @@ export async function performGithubAccountMigrations (db: Db, region: string | n } } const processed = new Set() + const metricsContext = new MeasureMetricsContext('github-migrate', {}) + + const factory: PipelineFactory = createBackupPipeline(metricsContext, dbUrl, txes, { + externalStorage: createDummyStorageAdapter(), + usePassedCtx: true + }) const replaces = new Map() + + const failedPersonIds = new Set() for (const it of integrations) { const ws = oldNewIds.get(it.workspace as any) ?? byId.get(it.workspace as any) if (ws != null) { - // Need to connect to workspace to get account mapping + if (isArchivingMode(ws.mode) || isDeletingMode(ws.mode)) { + continue + } + + console.info('processing workspace', ws.uuid, ws.name, ws.url) it.workspace = ws.uuid replaces.set(it.workspace, ws.uuid) - const wsToken = generateToken(systemAccountUuid, ws.uuid, { service: 'github', mode: 'backup' }) - const endpoint = await getTransactorEndpoint(wsToken, 'external') - const client = (await createClient(endpoint, wsToken)) as BackupClient & Client + const pipeline = await factory(metricsContext, ws, (): void => {}, null, null) + const client = wrapPipeline(metricsContext, pipeline, ws, false) const systemAccounts = [core.account.System, core.account.ConfigUser] const accountsTxes: TxCUD[] = [] @@ -100,7 +121,11 @@ export async function performGithubAccountMigrations (db: Db, region: string | n const docs = (await client.loadDocs(DOMAIN_MODEL_TX, ids)).filter((it) => TxProcessor.isExtendsCUD(it._class) ) as TxCUD[] - accountsTxes.push(...docs) + for (const tx of docs) { + if (tx.objectClass === 'core:class:Account' || tx.objectClass === 'contact:class:PersonAccount') { + accountsTxes.push(tx) + } + } if (info.finished && idx !== undefined) { await client.closeChunk(info.idx) break @@ -108,9 +133,6 @@ export async function performGithubAccountMigrations (db: Db, region: string | n } await client.close() - // await client.loadChunk(DOMAIN_MODEL_TX, { - // objectClass: { $in: ['core:class:Account', 'contact:class:PersonAccount'] as Ref>[] } - // }) const accounts: (Doc & { email?: string })[] = getAccountsFromTxes(accountsTxes) const socialKeyByAccount: Record = {} @@ -139,14 +161,19 @@ export async function performGithubAccountMigrations (db: Db, region: string | n }) if (existing == null) { - await githubAccountClient.createIntegration({ - kind: 'github', - workspaceUuid: ws?.uuid, - socialId: person, - data: { - installationId: it.installationId - } - }) + try { + await githubAccountClient.createIntegration({ + kind: 'github', + workspaceUuid: ws?.uuid, + socialId: person, + data: { + installationId: it.installationId + } + }) + } catch (err: any) { + failedPersonIds.add(person) + console.log(err) + } } } @@ -170,26 +197,38 @@ export async function performGithubAccountMigrations (db: Db, region: string | n }) if (existing == null) { - await githubAccountClient.createIntegration({ - kind: 'github-user', - workspaceUuid: null, - socialId: person, - data: { - login: u._id - } - }) - // Check/create integeration in account - await githubAccountClient.addIntegrationSecret({ - kind: 'github-user', - workspaceUuid: null, - socialId: person, - key: u._id, // github login - secret: JSON.stringify(data) - }) + try { + await githubAccountClient.createIntegration({ + kind: 'github-user', + workspaceUuid: null, + socialId: person, + data: { + login: u._id + } + }) + // Check/create integeration in account + await githubAccountClient.addIntegrationSecret({ + kind: 'github-user', + workspaceUuid: null, + socialId: person, + key: u._id, // github login + secret: JSON.stringify(data) + }) + } catch (err: any) { + failedPersonIds.add(person) + console.error(err) + } } } } } } } + console.log('Failed to create integrations for', failedPersonIds.size, 'people') + if (failedPersonIds.size > 0) { + // We need to remove all integrations for failed persons + for (const person of failedPersonIds) { + console.log('Fialed person id', person) + } + } } diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 2f88e5939b..bc4862f155 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -2545,7 +2545,9 @@ export function devTool ( const client = getMongoClient(mongodbUri) const _client = await client.getClient() - await performGithubAccountMigrations(_client.db(cmd.db), cmd.region ?? null) + const { dbUrl, txes } = prepareTools() + + await performGithubAccountMigrations(_client.db(cmd.db), dbUrl, txes, cmd.region ?? null) await _client.close() client.close() }) diff --git a/packages/text-markdown/src/compare.ts b/packages/text-markdown/src/compare.ts index 6dcaf87101..40c2438345 100644 --- a/packages/text-markdown/src/compare.ts +++ b/packages/text-markdown/src/compare.ts @@ -58,10 +58,10 @@ export function isMarkdownsEquals (source1: string, source2: string): boolean { .filter((it) => it.length > 0) .join('\n') - const norm1 = normalizeLineEndings(source1) + const norm1 = normalizeLineEndings(source1 ?? '') const lines1 = excludeBlankLines(norm1) - const norm2 = normalizeLineEndings(source2) + const norm2 = normalizeLineEndings(source2 ?? '') const lines2 = excludeBlankLines(norm2) return lines1 === lines2 diff --git a/plugins/client-resources/src/connection.ts b/plugins/client-resources/src/connection.ts index 47e581d535..e77aab8d3b 100644 --- a/plugins/client-resources/src/connection.ts +++ b/plugins/client-resources/src/connection.ts @@ -301,13 +301,6 @@ class Connection implements ClientConnection { } if (resp.rateLimit !== undefined && resp.rateLimit.remaining < 50) { - console.log( - 'Rate limits:', - resp.rateLimit.remaining, - resp.rateLimit.limit, - resp.rateLimit.reset, - resp.rateLimit.retryAfter - ) this.currentRateLimit = resp.rateLimit if (this.currentRateLimit.remaining < this.currentRateLimit.limit / 3) { if (this.slowDownTimer < 50) { diff --git a/services/github/pod-github/src/config.ts b/services/github/pod-github/src/config.ts index ab3509b1c9..118eca0fc2 100644 --- a/services/github/pod-github/src/config.ts +++ b/services/github/pod-github/src/config.ts @@ -96,7 +96,7 @@ const config: Config = (() => { SentryDSN: process.env[envMap.SentryDSN], BrandingPath: process.env[envMap.BrandingPath] ?? '', - WorkspaceInactivityInterval: parseInt(process.env[envMap.WorkspaceInactivityInterval] ?? '5'), // In days + WorkspaceInactivityInterval: parseInt(process.env[envMap.WorkspaceInactivityInterval] ?? '3'), // In days RateLimit: parseInt(process.env[envMap.RateLimit] ?? '25') } diff --git a/services/github/pod-github/src/platform.ts b/services/github/pod-github/src/platform.ts index c0168ee770..53eea0ea11 100644 --- a/services/github/pod-github/src/platform.ts +++ b/services/github/pod-github/src/platform.ts @@ -20,6 +20,7 @@ import core, { systemAccountUuid, TimeRateLimiter, TxOperations, + versionToString, WorkspaceInfoWithStatus, WorkspaceUuid, type PersonUuid, @@ -866,17 +867,24 @@ export class PlatformWorker { const rateLimiter = new RateLimiter(5) const rechecks: string[] = [] let idx = 0 - const connecting = new Map() + const connecting = new Map< + string, + { + time: number + version: string + } + >() const st = Date.now() const connectingInfo = setInterval(() => { this.ctx.info('****** connecting to workspaces ******', { connecting: connecting.size, time: Date.now() - st, workspaces: workspaces.length, + connected: this.clients.size, queue: rateLimiter.processingQueue.size }) for (const [c, d] of connecting.entries()) { - this.ctx.info('connecting to workspace', { workspace: c, time: Date.now() - d }) + this.ctx.info('connecting to workspace', { workspace: c, time: Date.now() - d.time, version: d.version }) } }, 5000) for (const workspace of workspaces) { @@ -898,10 +906,21 @@ export class PlatformWorker { const branding = Object.values(this.brandingMap).find((b) => b.key === workspaceInfo?.branding) ?? null const workerCtx = this.ctx.newChild('worker', { workspace: workspaceInfo.uuid }, {}) - connecting.set(workspaceInfo.uuid, Date.now()) + connecting.set(workspaceInfo.uuid, { + time: Date.now(), + version: versionToString({ + major: workspaceInfo.versionMajor, + minor: workspaceInfo.versionMinor, + patch: workspaceInfo.versionPatch + }) + }) workerCtx.info('************************* Register worker ************************* ', { workspaceId: workspaceInfo.uuid, workspaceUrl: workspaceInfo.url, + versionMajor: workspaceInfo.versionMajor, + versionMinor: workspaceInfo.versionMinor, + versionPatch: workspaceInfo.versionPatch, + mode: workspaceInfo.mode, index: widx, total: workspaces.length }) @@ -962,6 +981,10 @@ export class PlatformWorker { { workspaceId: workspaceInfo.uuid, workspaceUrl: workspaceInfo.url, + versionMajor: workspaceInfo.versionMajor, + versionMinor: workspaceInfo.versionMinor, + versionPatch: workspaceInfo.versionPatch, + lastVisit: (Date.now() - (workspaceInfo.lastVisit ?? 0)) / (24 * 60 * 60 * 1000), index: widx, total: workspaces.length } diff --git a/services/github/pod-github/src/sync/comments.ts b/services/github/pod-github/src/sync/comments.ts index 24bdfac6fe..f2f8594b87 100644 --- a/services/github/pod-github/src/sync/comments.ts +++ b/services/github/pod-github/src/sync/comments.ts @@ -328,16 +328,19 @@ export class CommentSyncManager implements DocSyncManager { if (Object.keys(platformUpdate).length > 0) { // Check and update body with external const okit = (await this.provider.getOctokit(existing.modifiedBy)) ?? container.container.octokit - await okit?.rest.issues.updateComment({ - owner: repository.owner?.login as string, - repo: repository.name, - issue_number: parent.githubNumber, - comment_id: comment.id, - body: await this.provider.getMarkdown(existingComment.message), - headers: { - 'X-GitHub-Api-Version': '2022-11-28' - } - }) + const mdown = await this.provider.getMarkdown(existingComment.message) + if (mdown.trim().length > 0) { + await okit?.rest.issues.updateComment({ + owner: repository.owner?.login as string, + repo: repository.name, + issue_number: parent.githubNumber, + comment_id: comment.id, + body: mdown, + headers: { + 'X-GitHub-Api-Version': '2022-11-28' + } + }) + } } if (Object.keys(update).length > 0) { await this.client.update(existing, update, false, new Date(comment.updated_at).getTime(), account) @@ -400,24 +403,29 @@ export class CommentSyncManager implements DocSyncManager { // No external version yet, create it. try { - const result = await okit?.rest.issues.createComment({ - owner: repo.owner?.login as string, - repo: repo.name, - issue_number: parent.githubNumber, - body: await this.provider.getMarkdown(chatMessage.message), - headers: { - 'X-GitHub-Api-Version': '2022-11-28' + const mdown = await this.provider.getMarkdown(chatMessage.message) + if (mdown.trim().length > 0) { + const result = await okit?.rest.issues.createComment({ + owner: repo.owner?.login as string, + repo: repo.name, + issue_number: parent.githubNumber, + body: mdown, + headers: { + 'X-GitHub-Api-Version': '2022-11-28' + } + }) + + const upd: DocumentUpdate = { + parent: (result?.data.html_url?.split('#')?.[0] ?? '').toLowerCase(), + url: (result?.data.url ?? '').toLowerCase(), + external: result?.data as CommentExternalData, + current: result?.data, + repository: repo._id } - }) - const upd: DocumentUpdate = { - parent: (result?.data.html_url?.split('#')?.[0] ?? '').toLowerCase(), - url: (result?.data.url ?? '').toLowerCase(), - external: result?.data as CommentExternalData, - current: result?.data, - repository: repo._id + + // We need to update in current promise, to prevent event changes. + await derivedClient.update(info, upd) } - // We need to update in current promise, to prevent event changes. - await derivedClient.update(info, upd) return {} } catch (err: any) { Analytics.handleError(err) diff --git a/services/github/pod-github/src/sync/issueBase.ts b/services/github/pod-github/src/sync/issueBase.ts index 7aabfac777..1e9081f59c 100644 --- a/services/github/pod-github/src/sync/issueBase.ts +++ b/services/github/pod-github/src/sync/issueBase.ts @@ -99,7 +99,7 @@ export abstract class IssueSyncManagerBase { // Find Assignees and reviewers const assignees: PersonId[] = [] - for (const o of issue.assignees.nodes ?? []) { + for (const o of issue.assignees?.nodes ?? []) { const acc = await this.provider.getAccount(o) if (acc !== undefined) { assignees.push(acc) diff --git a/services/github/pod-github/src/worker.ts b/services/github/pod-github/src/worker.ts index 23699c68c6..b7fc5f1433 100644 --- a/services/github/pod-github/src/worker.ts +++ b/services/github/pod-github/src/worker.ts @@ -1717,6 +1717,7 @@ export class GithubWorker implements IntegrationManager { ;({ client, endpoint } = await createPlatformClient(workspace.uuid, 30000, async (event: ClientConnectEvent) => { reconnect(workspace.uuid, event) })) + ctx.info('connected to github', { workspace: workspace.uuid, endpoint }) const githubEnabled = (await client.findOne(core.class.PluginConfiguration, { pluginId: githubId }))?.enabled if (githubEnabled === false) {