// // Copyright © 2023 Hardcore Engineering Inc. // import chunter, { ChatMessage } from '@hcengineering/chunter' import { PersonAccount } from '@hcengineering/contact' import core, { Account, AttachedData, Doc, DocumentUpdate, MeasureContext, Ref, TxOperations } from '@hcengineering/core' import github, { DocSyncInfo, GithubIntegrationRepository, GithubProject } from '@hcengineering/github' import { LiveQuery } from '@hcengineering/query' import { deepEqual } from 'fast-equals' import { ContainerFocus, DocSyncManager, ExternalSyncField, IntegrationContainer, IntegrationManager, githubExternalSyncVersion, githubSyncVersion } from '../types' import { collectUpdate, deleteObjects, errorToObj, getSince, isGHWriteAllowed } from './utils' import { Analytics } from '@hcengineering/analytics' import { IssueComment, IssueCommentCreatedEvent, IssueCommentEvent } from '@octokit/webhooks-types' import config from '../config' import { syncConfig } from './syncConfig' interface MessageData { message: string } type CommentExternalData = Omit export class CommentSyncManager implements DocSyncManager { provider!: IntegrationManager createCommentPromise: Promise> | undefined externalDerivedSync = false constructor ( readonly ctx: MeasureContext, readonly client: TxOperations, readonly lq: LiveQuery ) {} async init (provider: IntegrationManager): Promise { this.provider = provider } eventSync = new Map>() async handleEvent(integration: IntegrationContainer, derivedClient: TxOperations, evt: T): Promise { await this.createCommentPromise const event = evt as IssueCommentEvent this.ctx.info('comments:handleEvent', { action: event.action, login: event.sender.login, workspace: this.provider.getWorkspaceId().name }) if (event.sender.type === 'Bot') { // Ignore events from Bot if it is our bot // No need to handle event from ourself if (event.sender.login.includes(config.BotName)) { return } } await this.eventSync.get(event.issue.url) const promise = this.processEvent(event, derivedClient, integration) this.eventSync.set(event.issue.url, promise) await promise this.eventSync.delete(event.issue.url) } async handleDelete ( existing: Doc | undefined, info: DocSyncInfo, derivedClient: TxOperations, deleteExisting: boolean ): Promise { const container = await this.provider.getContainer(info.space) if (container === undefined) { return false } if ( container?.container === undefined || ((container.project.projectNodeId === undefined || !container.container.projectStructure.has(container.project._id)) && syncConfig.MainProject) ) { return false } const commentExternal = info.external as CommentExternalData | undefined if (commentExternal === undefined) { // No external issue yet, safe delete, since platform document will be deleted a well. return true } const account = existing?.createdBy ?? (await this.provider.getAccountU(commentExternal.user))?._id ?? core.account.System if (commentExternal !== undefined) { try { await this.deleteGithubDocument(container, account, commentExternal.node_id) } catch (err: any) { let cnt = false if (Array.isArray(err.errors)) { for (const e of err.errors) { if (e.type === 'NOT_FOUND') { // Ok issue is already deleted cnt = true break } } } if (!cnt) { Analytics.handleError(err) await derivedClient.update(info, { error: errorToObj(err) }) } } } if (existing !== undefined && deleteExisting) { await deleteObjects(this.ctx, this.client, [existing], account) } return true } async deleteGithubDocument (container: ContainerFocus, account: Ref, id: string): Promise { const okit = (await this.provider.getOctokit(account as Ref)) ?? container.container.octokit const q = `mutation deleteComment($commentID: ID!) { deleteIssueComment( input: {id: $commentID} ) { __typename } }` if (isGHWriteAllowed()) { await okit?.graphql(q, { commentID: id }) } } private async processEvent ( event: IssueCommentEvent, derivedClient: TxOperations, integration: IntegrationContainer ): Promise { const { repository } = await this.provider.getProjectAndRepository(event.repository.node_id) if (repository === undefined) { this.ctx.info('No project for repository', { repository: event.repository, workspace: this.provider.getWorkspaceId().name }) return } const account = (await this.provider.getAccountU(event.sender))?._id ?? core.account.System switch (event.action) { case 'created': { await this.createSyncData(event, derivedClient, repository) break } case 'deleted': { const syncData = await this.client.findOne(github.class.DocSyncInfo, { url: (event.comment.url ?? '').toLowerCase() }) if (syncData !== undefined) { await derivedClient.update(syncData, { deleted: true, needSync: '' }) this.provider.sync() } break } case 'edited': { const commentData = await this.client.findOne(github.class.DocSyncInfo, { url: (event.comment.url ?? '').toLowerCase() }) const messageData: MessageData = { message: await this.provider.getMarkup(integration, event.comment.body) } if (commentData !== undefined) { const chatMessage: ChatMessage | undefined = await this.client.findOne(commentData.objectClass, { _id: commentData._id as unknown as Ref }) if (chatMessage !== undefined) { const lastModified = new Date(event.comment.updated_at).getTime() await derivedClient.diffUpdate( commentData, { external: event.comment, current: messageData, needSync: githubSyncVersion, lastModified }, lastModified ) await this.client.diffUpdate(chatMessage, messageData, lastModified, account) this.provider.sync() } } break } } } private async createSyncData ( createdEvent: IssueCommentCreatedEvent, derivedClient: TxOperations, repo: GithubIntegrationRepository ): Promise { const commentData = await this.client.findOne(github.class.DocSyncInfo, { url: (createdEvent.comment.url ?? '').toLowerCase() }) if (commentData === undefined) { await derivedClient.createDoc(github.class.DocSyncInfo, repo.githubProject as Ref, { url: (createdEvent.comment.url ?? '').toLowerCase(), needSync: '', githubNumber: 0, repository: repo._id, objectClass: chunter.class.ChatMessage, external: createdEvent.comment as CommentExternalData, externalVersion: githubExternalSyncVersion, parent: createdEvent.issue.url, lastModified: new Date(createdEvent.comment.updated_at).getTime() }) this.provider.sync() } } async sync ( existing: Doc | undefined, info: DocSyncInfo, parent: DocSyncInfo | undefined, derivedClient: TxOperations ): Promise | undefined> { const container = await this.provider.getContainer(info.space) if (container?.container === undefined) { return { needSync: githubSyncVersion } } if (info.external === undefined) { // TODO: Use selected repository const repo = await this.provider.getRepositoryById(parent?.repository) if (repo?.nodeId === undefined) { // No need to sync if parent repository is not defined. return { needSync: githubSyncVersion } } // If no external document, we need to create it. this.createCommentPromise = this.createGithubComment(container, existing, info, parent, derivedClient) return await this.createCommentPromise } const comment = info.external as CommentExternalData if (parent === undefined) { // Find parent by issue url parent = await this.client.findOne(github.class.DocSyncInfo, { url: (comment.html_url.split('#')?.[0] ?? '').toLowerCase() }) } if (parent === undefined) { // no Sync until parent is found, parent should trigger all child's refresh. return { needSync: githubSyncVersion } } const account = existing?.modifiedBy ?? (await this.provider.getAccountU(comment.user))?._id ?? core.account.System const messageData: MessageData = { message: await this.provider.getMarkup(container.container, comment.body) } if (existing === undefined) { try { await this.createComment(info, messageData, parent, comment, account) return { needSync: githubSyncVersion, current: messageData } } catch (err: any) { Analytics.handleError(err) this.ctx.error(err) return { needSync: githubSyncVersion, error: errorToObj(err) } } } else { await this.handleDiffUpdate(existing, info, messageData, container, parent, comment, account) } return { current: messageData, needSync: githubSyncVersion } } private async handleDiffUpdate ( existing: Doc, info: DocSyncInfo, messageData: MessageData, container: ContainerFocus, parent: DocSyncInfo, comment: CommentExternalData, account: Ref ): Promise { const repository = await this.provider.getRepositoryById(info.repository) if (repository === undefined) { return } const existingComment = existing as ChatMessage const previousData: MessageData = info.current ?? ({} as unknown as MessageData) const update = collectUpdate(previousData, messageData, Object.keys(messageData)) const platformUpdate = collectUpdate(previousData, existing, Object.keys(messageData)) // We should remove changes we already have from github changed. for (const [k, v] of Object.entries(update)) { if ((platformUpdate as any)[k] !== v) { // eslint-disable-next-line @typescript-eslint/no-dynamic-delete delete (platformUpdate as any)[k] } } // Remove current same values from update for (const [k, v] of Object.entries(existingComment)) { if ((update as any)[k] === v) { // eslint-disable-next-line @typescript-eslint/no-dynamic-delete delete (update as any)[k] } } if (Object.keys(platformUpdate).length > 0) { // Check and update body with external const okit = (await this.provider.getOctokit(existing.modifiedBy as Ref)) ?? container.container.octokit await okit?.rest.issues.updateComment({ owner: repository.owner?.login as string, repo: repository.name, issue_number: parent.githubNumber, comment_id: comment.id, body: await this.provider.getMarkdown(existingComment.message), headers: { 'X-GitHub-Api-Version': '2022-11-28' } }) } if (Object.keys(update).length > 0) { await this.client.update(existing, update, false, new Date(comment.updated_at).getTime(), account) } } private async createComment ( info: DocSyncInfo, messageData: MessageData, parent: DocSyncInfo, comment: CommentExternalData, account: Ref ): Promise { const _id: Ref = info._id as unknown as Ref const value: AttachedData = { ...messageData, attachments: 0 } await this.client.addCollection( chunter.class.ChatMessage, info.space, parent._id, parent.objectClass, 'comments', value, _id, new Date(comment.created_at).getTime(), account ) } async createGithubComment ( container: ContainerFocus, existing: Doc | undefined, info: DocSyncInfo, parent: DocSyncInfo | undefined, derivedClient: TxOperations ): Promise> { // TODO: Use selected repository const repo = await this.provider.getRepositoryById(parent?.repository) if (repo?.nodeId === undefined) { // No need to sync if parent repository is not defined. return { needSync: githubSyncVersion } } if (parent === undefined) { return {} } const chatMessage = existing as ChatMessage const okit = (await this.provider.getOctokit(chatMessage.modifiedBy as Ref)) ?? container.container.octokit // No external version yet, create it. try { const result = await okit?.rest.issues.createComment({ owner: repo.owner?.login as string, repo: repo.name, issue_number: parent.githubNumber, body: await this.provider.getMarkdown(chatMessage.message), headers: { 'X-GitHub-Api-Version': '2022-11-28' } }) const upd: DocumentUpdate = { parent: result?.data.html_url?.split('#')?.[0] ?? '', url: (result?.data.url ?? '').toLowerCase(), external: result?.data as CommentExternalData, current: result?.data, repository: repo._id } // We need to update in current promise, to prevent event changes. await derivedClient.update(info, upd) return {} } catch (err: any) { Analytics.handleError(err) this.ctx.error(err) return { needSync: githubSyncVersion, error: errorToObj(err) } } } async externalSync ( integration: IntegrationContainer, derivedClient: TxOperations, kind: ExternalSyncField, syncDocs: DocSyncInfo[], repository: GithubIntegrationRepository, project: GithubProject ): Promise { // No need to perform external sync for comments, so let's update marks const tx = derivedClient.apply() for (const d of syncDocs) { await tx.update(d, { externalVersion: githubExternalSyncVersion }) } await tx.commit() this.provider.sync() } repositoryDisabled (integration: IntegrationContainer, repo: GithubIntegrationRepository): void { integration.synchronized.delete(`${repo._id}:comment`) } async externalFullSync ( integration: IntegrationContainer, derivedClient: TxOperations, projects: GithubProject[], repositories: GithubIntegrationRepository[] ): Promise { for (const repo of repositories) { const syncKey = `${repo._id}:comment` if (repo.githubProject === undefined || !repo.enabled || integration.synchronized.has(syncKey)) { if (!repo.enabled) { integration.synchronized.delete(syncKey) } continue } const prj = projects.find((it) => repo.githubProject === it._id) if (prj === undefined) { continue } // Wait global project sync await integration.syncLock.get(prj._id) const since = await getSince(this.client, chunter.class.ChatMessage, repo) const i = integration.octokit.paginate.iterator(integration.octokit.rest.issues.listCommentsForRepo, { owner: repo.owner?.login as string, repo: repo.name, state: 'all', sort: 'updated', direction: 'asc', since, headers: { 'X-GitHub-Api-Version': '2022-11-28' } }) try { for await (const data of i) { const comments: CommentExternalData[] = data.data as any this.ctx.info('retrieve comments for', { repo: repo.name, comments: comments.length, used: data.headers['x-ratelimit-used'], limit: data.headers['x-ratelimit-limit'], workspace: this.provider.getWorkspaceId().name }) await this.syncComments(repo, comments, derivedClient) this.provider.sync() } } catch (err: any) { Analytics.handleError(err) this.ctx.error(err) } integration.synchronized.add(syncKey) } } async syncComments ( repo: GithubIntegrationRepository, comments: CommentExternalData[], derivedClient: TxOperations ): Promise { if (repo.githubProject == null) { return } const syncInfo = await this.client.findAll(github.class.DocSyncInfo, { space: repo.githubProject, repository: repo._id, objectClass: chunter.class.ChatMessage, url: { $in: comments.map((it) => (it.url ?? '').toLowerCase()) } }) for (const comment of comments) { try { const existing = syncInfo.find((it) => it.url === comment.url.toLowerCase()) const lastModified = new Date(comment.updated_at).getTime() if (existing === undefined) { await derivedClient.createDoc(github.class.DocSyncInfo, repo.githubProject, { url: comment.url.toLowerCase(), needSync: '', githubNumber: 0, objectClass: chunter.class.ChatMessage, external: comment, externalVersion: githubExternalSyncVersion, parent: comment.html_url.split('#')?.[0], repository: repo._id, lastModified }) } else { if (!deepEqual(existing.external, comment) || existing.externalVersion !== githubExternalSyncVersion) { await derivedClient.diffUpdate( existing, { needSync: '', external: comment, externalVersion: githubExternalSyncVersion, lastModified }, lastModified ) this.provider.sync() } } } catch (err: any) { Analytics.handleError(err) this.ctx.error(err) } } } }