// // Copyright © 2023 Hardcore Engineering Inc. // /* TODO: * Add since to synchronization */ /* eslint-disable @typescript-eslint/no-unused-vars */ import activity from '@hcengineering/activity' import { Analytics } from '@hcengineering/analytics' import { CollaboratorClient } from '@hcengineering/collaborator-client' import contact, { Person } from '@hcengineering/contact' import core, { AttachedDoc, Class, Doc, DocumentUpdate, Markup, MeasureContext, PersonId, Ref, Space, TxOperations, makeDocCollabId, withContext } from '@hcengineering/core' import github, { DocSyncInfo, GithubIntegrationRepository, GithubIssue, GithubProject } from '@hcengineering/github' import { IntlString } from '@hcengineering/platform' import { LiveQuery } from '@hcengineering/query' import { getPublicLink } from '@hcengineering/server-guest-resources' import task, { type Task } from '@hcengineering/task' import { MarkupNode, MarkupNodeType, areEqualMarkups, markupToJSON, traverseNode } from '@hcengineering/text' import time, { type ToDo } from '@hcengineering/time' import tracker, { Issue } from '@hcengineering/tracker' import { deepEqual } from 'fast-equals' import { Octokit } from 'octokit' import { ContainerFocus, IntegrationManager, githubExternalSyncVersion, githubSyncVersion } from '../types' import { IssueExternalData } from './githubTypes' import { stripGuestLink } from './guest' import { collectUpdate, compareMarkdown, deleteObjects, errorToObj, guessStatus } from './utils' /** * @public */ export type WithMarkup = Omit & { description: Markup } /** * @public */ export type GithubIssueData = Omit< WithMarkup, | 'commits' | 'attachments' | 'commits' | 'number' | 'files' | 'space' | 'identifier' | 'rank' | 'status' | 'priority' | 'subIssues' | 'parents' | 'estimation' | 'reportedTime' | 'reports' | 'childInfo' | 'dueDate' | 'kind' | 'reviews' | 'reviewThreads' | 'reviewComments' | 'component' | keyof AttachedDoc > & Record /** * @public */ export type IssueUpdate = DocumentUpdate> export abstract class IssueSyncManagerBase { provider!: IntegrationManager constructor ( readonly client: TxOperations, readonly lq: LiveQuery, readonly collaborator: CollaboratorClient ) {} async init (provider: IntegrationManager): Promise { this.provider = provider } async getAssignees (issue: IssueExternalData): Promise[]> { // Find Assignees and reviewers const assignees: PersonId[] = [] for (const o of issue.assignees?.nodes ?? []) { const acc = await this.provider.getAccount(o) if (acc !== undefined) { assignees.push(acc) } } return await this.getPersonsFromId(assignees) } async getPersonsFromId (assignees: PersonId[]): Promise[]> { const socialIds = await this.client.findAll(contact.class.SocialIdentity, { _id: { $in: assignees as any } }) return socialIds.map((it) => it.attachedTo) } @withContext('issues-handleUpdate') async handleUpdate ( ctx: MeasureContext, external: IssueExternalData, derivedClient: TxOperations, update: IssueUpdate, account: PersonId, prj: GithubProject, needSync: boolean, syncData?: DocSyncInfo, verifyUpdate?: ( state: DocSyncInfo, existing: Issue, external: IssueExternalData, update: IssueUpdate ) => Promise, extraSyncUpdate?: DocumentUpdate ): Promise { if (Object.keys(update).length === 0 && !needSync) { return } syncData = syncData ?? (await ctx.with('findDocInfo', {}, (ctx) => this.client.findOne(github.class.DocSyncInfo, { space: prj._id, url: (external.url ?? '').toLowerCase() }) )) if (syncData !== undefined) { const doc: Issue | undefined = await ctx.with('find pull request', {}, (ctx) => this.client.findOne(syncData.objectClass, { _id: syncData._id as unknown as Ref }) ) // Use now as modified date for events. const lastModified = new Date().getTime() if (doc !== undefined && ((await verifyUpdate?.(syncData, doc, external, update)) ?? true)) { const issueData: DocumentUpdate = { ...update, description: doc.description } if ( update.description !== undefined && !areEqualMarkups(update.description, syncData.current?.description ?? '') ) { try { const collabId = makeDocCollabId(doc, 'description') await this.collaborator.updateMarkup(collabId, update.description) } catch (err: any) { Analytics.handleError(err) ctx.error(err) } } else { delete update.description } // Sync all possibly updated action items states if (update.description !== undefined) { const node = markupToJSON(update.description) const todos: Record = {} traverseNode(node, (n) => { if (n.type === MarkupNodeType.todoItem && n.attrs?.todoid !== undefined) { todos[n.attrs.todoid as string] = (n.attrs.checked as boolean) ?? false } return true }) const updateTodos = this.client.apply() for (const [k, v] of Object.entries(todos)) { await updateTodos.updateDoc(time.class.ToDo, time.space.ToDos, k as Ref, { doneOn: v ? Date.now() : null }) } await updateTodos.commit() } await ctx.with('diffUpdate syncData', {}, (ctx) => derivedClient.diffUpdate( syncData, { external, externalVersion: githubExternalSyncVersion, current: { ...syncData.current, ...update }, needSync: needSync ? '' : githubSyncVersion, // No need to sync after operation. derivedVersion: '', // Check derived changes lastModified, lastGithubUser: account, ...extraSyncUpdate }, lastModified ) ) await ctx.with('diffUpdate-issue', {}, (ctx) => this.client.diffUpdate(doc, issueData, lastModified, account)) this.provider.sync() } else if (doc === undefined) { await ctx.with('diffUpdate-syncData', {}, (ctx) => derivedClient.diffUpdate( syncData, { external, externalVersion: githubExternalSyncVersion, needSync: '', derivedVersion: '', // Check derived changes lastModified, lastGithubUser: account, ...extraSyncUpdate }, lastModified ) ) } } if (needSync) { this.provider.sync() } } abstract fillBackChanges (update: DocumentUpdate, existing: GithubIssue, external: any): Promise async addConnectToMessage ( msg: IntlString, prj: Ref, issueId: Ref, _class: Ref>, issueExternal: { url: string, number: number }, repository: GithubIntegrationRepository ): Promise { await this.client.addCollection(activity.class.ActivityInfoMessage, prj, issueId, _class, 'activity', { message: msg, icon: github.icon.Github, props: { url: issueExternal.url, repository: repository.url?.replace('api.github.com/repos', 'github.com'), repoName: repository.name, number: issueExternal.number } }) } stripGuestLink = async (data: MarkupNode): Promise => { await stripGuestLink(data) } abstract performIssueFieldsUpdate ( ctx: MeasureContext, info: DocSyncInfo, existing: WithMarkup, platformUpdate: DocumentUpdate, issueData: GithubIssueData, container: ContainerFocus, issueExternal: IssueExternalData, okit: Octokit, account: PersonId ): Promise abstract afterSync ( ctx: MeasureContext, existing: Issue, account: PersonId, issueExternal: any, info: DocSyncInfo ): Promise @withContext('issues-handleDiffUpdate') async handleDiffUpdate ( ctx: MeasureContext, container: ContainerFocus, existing: WithMarkup, info: DocSyncInfo, issueData: GithubIssueData, issueExternal: IssueExternalData, account: PersonId, accountGH: PersonId ): Promise> { let needUpdate = false if (!this.client.getHierarchy().hasMixin(existing, github.mixin.GithubIssue)) { await ctx.with( 'create mixin issue: GithubIssue', {}, async (ctx) => { await this.client.createMixin( existing._id as Ref, existing._class, existing.space, github.mixin.GithubIssue, { githubNumber: issueExternal.number, url: issueExternal.url, repository: info.repository as Ref } ) await this.notifyConnected(container, info, existing, issueExternal) }, { identifier: existing.identifier, url: issueExternal.url }, { log: true } ) // Re iterate to have existing value with mixin inside. needUpdate = true } else { const ghIssue = this.client.getHierarchy().as(existing, github.mixin.GithubIssue) await this.client.diffUpdate(ghIssue, { githubNumber: issueExternal.number, url: issueExternal.url, repository: info.repository as Ref }) if (ghIssue.url !== issueExternal.url) { await this.notifyConnected(container, info, existing, issueExternal) } } if (needUpdate) { return { needSync: '' } } const existingIssue = this.client.getHierarchy().as(existing, github.mixin.GithubIssue) const previousData: GithubIssueData = info.current ?? ({} as unknown as GithubIssueData) // const type = await this.provider.getTaskTypeOf(container.project.type, existing._class) // const stst = await this.provider.getStatuses(type?._id) const update = collectUpdate(previousData, issueData, Object.keys(issueData)) const allAttributes = this.client.getHierarchy().getAllAttributes(existingIssue._class) const platformUpdate = collectUpdate(previousData, existingIssue, Array.from(allAttributes.keys())) const okit = (await this.provider.getOctokit(ctx, account)) ?? container.container.octokit // Remove current same values from update for (const [k, v] of Object.entries(update)) { if ((existingIssue as any)[k] === v) { // eslint-disable-next-line @typescript-eslint/no-dynamic-delete delete (update as any)[k] } } if (update.description !== undefined) { if (update.description === existingIssue.description) { delete update.description } } for (const [k, v] of Object.entries(update)) { let pv = (platformUpdate as any)[k] if (k === 'description' && pv != null) { const mdown = await this.provider.getMarkdown(pv) pv = await this.provider.getMarkupSafe(container.container, mdown, this.stripGuestLink) } if (pv != null && pv !== v) { // We have conflict of values, assume platform is more proper one. ctx.error('conflict', { id: existing.identifier, k }) // Assume platform change is more important in case of conflict values. // eslint-disable-next-line @typescript-eslint/no-dynamic-delete delete (update as any)[k] continue } } await this.fillBackChanges(update, existingIssue, issueExternal) let needExternalSync = false if (container !== undefined && okit !== undefined) { // Check and update issue fields. needExternalSync = await this.performIssueFieldsUpdate( ctx, info, existing, platformUpdate, issueData, container, issueExternal, okit, account ) } // We need remove all readonly field values for (const k of Object.keys(update)) { // Skip readonly fields const attr = this.client.getHierarchy().findAttribute(existingIssue._class, k) if (attr?.readonly === true) { // eslint-disable-next-line @typescript-eslint/no-dynamic-delete delete (update as any)[k] continue } } // Update collaborative description if (update.description !== undefined) { ctx.info(`<= perform ${issueExternal.url} update to collaborator`, { workspace: this.provider.getWorkspaceId() }) try { const description = update.description as Markup issueData.description = description const collabId = makeDocCollabId(existingIssue, 'description') await this.collaborator.updateMarkup(collabId, description) } catch (err: any) { Analytics.handleError(err) ctx.error('error during description update', err) } delete update.description } if (Object.keys(update).length > 0) { // We have some fields to update of existing from external ctx.info(`<= perform ${issueExternal.url} update to platform`, { ...update, workspace: this.provider.getWorkspaceId() }) await this.client.update(existingIssue, update, false, new Date().getTime(), accountGH) } await this.afterSync(ctx, existingIssue, accountGH, issueExternal, info) // We need to trigger external version retrieval, via sync or event, to prevent move sync operations from platform before we will be sure all is updated on github. return { current: issueData, needSync: githubSyncVersion, ...(needExternalSync ? { externalVersion: '' } : {}), lastGithubUser: null } } private async notifyConnected ( container: ContainerFocus, info: DocSyncInfo, existing: WithMarkup, issueExternal: IssueExternalData ): Promise { const repo = await this.provider.getRepositoryById(info.repository) if (repo != null) { await this.addConnectToMessage( existing._class === github.class.GithubPullRequest ? github.string.PullRequestConnectedActivityInfo : github.string.IssueConnectedActivityInfo, existing.space, existing._id, existing._class, issueExternal, repo ) } } async collectIssueUpdate ( info: DocSyncInfo, doc: WithMarkup, platformUpdate: DocumentUpdate, issueData: Pick, 'title' | 'description' | 'assignee' | 'status'>, container: ContainerFocus, issueExternal: IssueExternalData, _class: Ref> ): Promise> { const issueUpdate: { title?: string body?: string stateReason?: string assigneeIds?: string[] } & Record = {} if (platformUpdate.title != null) { if (platformUpdate.title !== issueExternal.title) { issueUpdate.title = platformUpdate.title } issueData.title = platformUpdate.title } if (platformUpdate.description != null) { // Need to convert to markdown issueUpdate.body = await this.provider.getMarkdown(platformUpdate.description ?? '') issueData.description = await this.provider.getMarkupSafe( container.container, issueUpdate.body ?? '', this.stripGuestLink ) // Of value is same, not need to update. if (compareMarkdown(issueUpdate.body, issueExternal.body)) { delete issueUpdate.body } } if (platformUpdate.assignee !== undefined) { const info = platformUpdate.assignee !== null ? await this.provider.getGithubLogin(container.container, platformUpdate.assignee) : undefined // Check external const currentAssignees = (issueExternal.assignees.nodes ?? []).map((it) => it.id) currentAssignees.sort((a, b) => a.localeCompare(b)) issueUpdate.assigneeIds = info !== undefined ? [info.id] : [] issueUpdate.assigneeIds.sort((a, b) => a.localeCompare(b)) if (deepEqual(currentAssignees, issueUpdate.assigneeIds)) { // Same ids delete issueUpdate.assigneeIds } issueData.assignee = platformUpdate.assignee } const status = platformUpdate.status ?? issueData.status const type = await this.provider.getTaskTypeOf(container.project.type, _class) const statuses = await this.provider.getStatuses(type?._id) const st = statuses.find((it) => it._id === status) if (st !== undefined) { // Need to convert to two operations. switch (st.category) { case task.statusCategory.UnStarted: case task.statusCategory.ToDo: case task.statusCategory.Active: if (issueExternal.state !== 'OPEN') { issueUpdate.state = 'OPEN' } break case task.statusCategory.Won: if (issueExternal.state !== 'CLOSED' || issueExternal.stateReason !== 'COMPLETED') { issueUpdate.state = 'CLOSED' issueUpdate.stateReason = 'COMPLETED' } break case task.statusCategory.Lost: if (issueExternal.state !== 'CLOSED' || issueExternal.stateReason !== 'NOT_PLANNED') { issueUpdate.state = 'CLOSED' issueUpdate.stateReason = 'not_planed' // Not supported change to github } break } } return issueUpdate } async performDocumentExternalSync ( ctx: MeasureContext, info: DocSyncInfo, previousExternal: IssueExternalData, issueExternal: IssueExternalData, derivedClient: TxOperations ): Promise { // TODO: Since Github integeration need to be re-written to use cards, so this is quick fix to not loose data in case of external sync while service was offline. const update: IssueUpdate = {} const du: DocumentUpdate = {} const account = (await this.provider.getAccount(issueExternal.author)) ?? core.account.System const container = await this.provider.getContainer(info.space) if (container == null) { return } const type = await this.provider.getTaskTypeOf(container.project.type, tracker.class.Issue) const statuses = await this.provider.getStatuses(type?._id) if (previousExternal.state !== issueExternal.state) { update.status = ( await guessStatus({ state: issueExternal.state, stateReason: issueExternal.stateReason }, statuses) )._id } if (previousExternal.title !== issueExternal.title) { update.title = issueExternal.title } if (previousExternal.body !== issueExternal.body) { update.description = await this.provider.getMarkupSafe( container.container, issueExternal.body, this.stripGuestLink ) du.markdown = await this.provider.getMarkdown(update.description) } if (!deepEqual(previousExternal.assignees, issueExternal.assignees)) { const assignees = await this.getAssignees(issueExternal) update.assignee = assignees?.[0] ?? null } if (Object.keys(update).length > 0) { await this.handleUpdate(ctx, issueExternal, derivedClient, update, account, container.project, false) } } async syncIssues ( ctx: MeasureContext, _class: Ref>, repo: GithubIntegrationRepository, issues: IssueExternalData[], derivedClient: TxOperations, syncDocs?: DocSyncInfo[] ): Promise { if (repo.githubProject == null) { return } const syncInfo = syncDocs ?? (await this.client.findAll(github.class.DocSyncInfo, { space: repo.githubProject, // repository: repo._id, // If we skip repository, we will find orphaned issues, so we could connect them on. objectClass: _class, url: { $in: issues.map((it) => (it.url ?? '').toLowerCase()) } })) const ops = derivedClient.apply() for (const issue of issues) { try { if (issue.url === undefined && Object.keys(issue).length === 0) { ctx.info('Retrieve empty document', { repo: repo.name, workspace: this.provider.getWorkspaceId() }) continue } const existing = syncInfo.find((it) => it.url.toLowerCase() === issue.url.toLowerCase()) ?? syncInfo.find((it) => (it.external as IssueExternalData)?.id === issue.id) if (existing === undefined && syncDocs === undefined) { ctx.info('Create sync doc', { url: issue.url, workspace: this.provider.getWorkspaceId() }) await ops.createDoc(github.class.DocSyncInfo, repo.githubProject, { url: issue.url.toLowerCase(), needSync: '', repository: repo._id, githubNumber: issue.number, objectClass: _class, external: issue, externalVersion: githubExternalSyncVersion, derivedVersion: '', externalVersionSince: '', lastModified: new Date(issue.updatedAt).getTime() }) } else if (existing !== undefined) { if (syncDocs !== undefined) { syncDocs = syncDocs.filter((it) => it._id !== existing._id) } const externalEqual = deepEqual(existing.external, issue) && existing.repository === repo._id if (!externalEqual || existing.externalVersion !== githubExternalSyncVersion) { ctx.info('Update sync doc(extarnal changes)', { url: issue.url, workspace: this.provider.getWorkspaceId() }) if (existing.needSync === githubSyncVersion || existing.repository !== repo._id) { // Sync external if and only if no changes from platform or we do resync from github. // We need to apply changes from Github, while service was offline. await this.performDocumentExternalSync(ctx, existing, existing.external, issue, derivedClient) } await ops.diffUpdate( existing, { needSync: externalEqual ? existing.needSync : '', external: issue, externalVersion: githubExternalSyncVersion, derivedVersion: '', // Clear derived state to recalculate it. externalVersionSince: '', repository: repo._id, lastModified: new Date(issue.updatedAt).getTime() }, Date.now() ) } } } catch (err: any) { Analytics.handleError(err) ctx.error(err) } } // if no sync doc, mark it as synchronized for (const sd of syncDocs ?? []) { await ops.update(sd, { needSync: githubSyncVersion, externalVersion: githubExternalSyncVersion, error: 'not found external doc' }) } await ops.commit(true) this.provider.sync() } abstract deleteGithubDocument ( ctx: MeasureContext, container: ContainerFocus, account: PersonId, id: string ): Promise async handleDelete ( ctx: MeasureContext, existing: Doc | undefined, info: DocSyncInfo, derivedClient: TxOperations, deleteExisting: boolean ): Promise { const container = await this.provider.getContainer(info.space) if (container === undefined) { return false } const issueExternal = info.external as IssueExternalData | undefined if (issueExternal === undefined) { // No external issue yet, safe delete, since platform document will be deleted a well. return true } const account = existing?.createdBy ?? (await this.provider.getAccount(issueExternal.author)) ?? core.account.System if (issueExternal !== undefined) { try { await this.deleteGithubDocument(ctx, container, account, issueExternal.id) } catch (err: any) { let cnt = false if (Array.isArray(err.errors)) { for (const e of err.errors) { if (e.type === 'NOT_FOUND') { // Ok issue is already deleted cnt = true break } } } if (!cnt) { Analytics.handleError(err) ctx.error('Error', { err }) await derivedClient.update(info, { error: errorToObj(err), needSync: githubSyncVersion }) return false } } } if (existing !== undefined && deleteExisting) { const childItems = await derivedClient.findAll(github.class.DocSyncInfo, { parent: (issueExternal.url ?? '').toLowerCase() }) for (const u of childItems) { // We need just to clean all of them, since child's for issue are comments for now. await derivedClient.remove(u) } await deleteObjects(ctx, this.client, [existing], account) } return true } protected async createErrorSyncDataByUrl ( url: string, githubNumber: number, date: Date, derivedClient: TxOperations, repo: GithubIntegrationRepository, err: any, _class: Ref> = tracker.class.Issue ): Promise { const syncData = await this.client.findOne(github.class.DocSyncInfo, { space: repo.githubProject as Ref, url: url.toLowerCase() }) if (syncData === undefined) { await derivedClient?.createDoc(github.class.DocSyncInfo, repo.githubProject as Ref, { url, needSync: githubSyncVersion, // We need external sync first. githubNumber, externalVersion: '', repository: repo._id, objectClass: _class, external: {}, error: errorToObj(err), lastModified: date.getTime() }) // We need trigger comments, if their sync data created before const childInfos = await this.client.findAll(github.class.DocSyncInfo, { parent: url.toLowerCase() }) for (const child of childInfos) { await derivedClient?.update(child, { needSync: '' }) } this.provider.sync() } } async addHulyLink ( info: DocSyncInfo, syncResult: DocumentUpdate, object: Doc, external: IssueExternalData, container: ContainerFocus ): Promise { const repository = await this.provider.getRepositoryById(info.repository) if (repository !== undefined) { syncResult.addHulyLink = false const publicLink = await getPublicLink( object, this.client, { uuid: this.provider.getWorkspaceId(), url: this.provider.getWorkspaceUrl() }, false, this.provider.getBranding() ) // We need to create comment on Github about issue is connected. await container.container.octokit.rest.issues.createComment({ owner: repository.owner?.login as string, repo: repository.name, issue_number: external.number, body: `

Connected to Huly®: ${(object as Task).identifier}

`, headers: { 'X-GitHub-Api-Version': '2022-11-28' } }) } } }