diff --git a/.vscode/launch.json b/.vscode/launch.json index 6923fbaaa3..453b5bf8a7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -424,7 +424,7 @@ "name": "Debug tool upgrade", "type": "node", "request": "launch", - "args": ["src/__start.ts", "upgrade-workspace", "mongo-1000-1"], + "args": ["src/__start.ts", "migrate-github-account", "--region", "cockroach", "--db", "%github"], "env": { "SERVER_SECRET": "secret", "MINIO_ACCESS_KEY": "minioadmin", @@ -433,7 +433,7 @@ "TRANSACTOR_URL": "ws://localhost:3333", "MONGO_URL": "mongodb://localhost:27017", "DB_URL": "mongodb://localhost:27017", - "ACCOUNTS_URL": "http://localhost:3000", + "ACCOUNTS_URL": "http://127.0.0.1:3000", "ACCOUNT_DB_URL": "mongodb://localhost:27017", "TELEGRAM_DATABASE": "telegram-service", "REKONI_URL": "http://localhost:4004", @@ -565,21 +565,21 @@ "request": "launch", "args": ["src/index.ts"], "env": { - "MONGO_URL": "mongodb://localhost:27018", + "MONGO_URL": "mongodb://localhost:27017", "SERVER_SECRET": "secret", - "ACCOUNTS_URL": "http://localhost:3003", + "ACCOUNTS_URL": "http://localhost:3000", "APP_ID": "${env:POD_GITHUB_APPID}", "CLIENT_ID": "${env:POD_GITHUB_CLIENTID}", "CLIENT_SECRET": "${env:POD_GITHUB_CLIENT_SECRET}", "PRIVATE_KEY": "${env:POD_GITHUB_PRIVATE_KEY}", - "COLLABORATOR_URL": "ws://huly.local:3079", + "COLLABORATOR_URL": "ws://huly.local:3078", "MINIO_ENDPOINT": "localhost", "MINIO_ACCESS_KEY": "minioadmin", "MINIO_SECRET_KEY": "minioadmin", "PLATFORM_OPERATION_LOGGING": "true", "FRONT_URL": "http://localhost:8080", "PORT": "3500", - "STATS_URL": "http://huly.local:4901" + "STATS_URL": "http://huly.local:4900" }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "sourceMaps": true, diff --git a/dev/tool/src/github.ts b/dev/tool/src/github.ts new file mode 100644 index 0000000000..ac4156b0d9 --- /dev/null +++ b/dev/tool/src/github.ts @@ -0,0 +1,195 @@ +import core, { + buildSocialIdString, + DOMAIN_MODEL_TX, + systemAccountUuid, + TxProcessor, + type BackupClient, + type Client, + type Doc, + type Ref, + type TxCUD, + type WorkspaceUuid +} from '@hcengineering/core' +import { getAccountsFromTxes, getSocialKeyByOldEmail } from '@hcengineering/model-core' +import { createClient, getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client' +import { generateToken } from '@hcengineering/server-token' +import type { Db } from 'mongodb' + +/** + * @public + */ +export interface GithubIntegrationRecord { + installationId: number + workspace: string + accountId: string // Ref +} + +/** + * @public + */ +export interface GithubUserRecord { + _id: string // login + code?: string | null + token?: string + expiresIn?: number | null // seconds + refreshToken?: string | null + refreshTokenExpiresIn?: number | null + authorized?: boolean + state?: string + scope?: string + error?: string | null + + accounts: Record */> +} + +export async function performGithubAccountMigrations (db: Db, region: string | null): Promise { + const token = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'admin', admin: 'true' }) + const githubToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'github' }) + const accountClient = getAccountClient(token) + + const githubAccountClient = getAccountClient(githubToken) + + const usersCollection = db.collection('users') + + const integrationCollection = db.collection('installations') + + const integrations = await integrationCollection.find({}).toArray() + // Check and apply migrations + // We need to update all workspace information accordingly + + const allWorkpaces = await accountClient.listWorkspaces(region) + const byId = new Map(allWorkpaces.map((it) => [it.uuid, it])) + const oldNewIds = new Map(allWorkpaces.map((it) => [it.dataId ?? it.uuid, it])) + + const allAuthorizations = await usersCollection.find({}).toArray() + + const wsToAuth = new Map() + + for (const it of allAuthorizations) { + for (const ws of Object.keys(it.accounts)) { + const wsId = oldNewIds.get(ws as WorkspaceUuid) ?? byId.get(ws as WorkspaceUuid) + if (wsId !== undefined) { + wsToAuth.set(wsId.uuid, (wsToAuth.get(wsId.uuid) ?? []).concat(it)) + } + } + } + const processed = new Set() + + const replaces = new Map() + for (const it of integrations) { + const ws = oldNewIds.get(it.workspace as any) ?? byId.get(it.workspace as any) + if (ws != null) { + // Need to connect to workspace to get account mapping + + it.workspace = ws.uuid + replaces.set(it.workspace, ws.uuid) + + const wsToken = generateToken(systemAccountUuid, ws.uuid, { service: 'github', mode: 'backup' }) + const endpoint = await getTransactorEndpoint(wsToken, 'external') + const client = (await createClient(endpoint, wsToken)) as BackupClient & Client + + const systemAccounts = [core.account.System, core.account.ConfigUser] + const accountsTxes: TxCUD[] = [] + + let idx: number | undefined + + while (true) { + const info = await client.loadChunk(DOMAIN_MODEL_TX, idx) + idx = info.idx + const ids = Array.from(info.docs.map((it) => it.id as Ref)) + const docs = (await client.loadDocs(DOMAIN_MODEL_TX, ids)).filter((it) => + TxProcessor.isExtendsCUD(it._class) + ) as TxCUD[] + accountsTxes.push(...docs) + if (info.finished && idx !== undefined) { + await client.closeChunk(info.idx) + break + } + } + await client.close() + + // await client.loadChunk(DOMAIN_MODEL_TX, { + // objectClass: { $in: ['core:class:Account', 'contact:class:PersonAccount'] as Ref>[] } + // }) + const accounts: (Doc & { email?: string })[] = getAccountsFromTxes(accountsTxes) + + const socialKeyByAccount: Record = {} + for (const account of accounts) { + if (account.email === undefined) { + continue + } + + if (systemAccounts.includes(account._id as any)) { + ;(socialKeyByAccount as any)[account._id] = account._id + } else { + socialKeyByAccount[account._id] = buildSocialIdString(getSocialKeyByOldEmail(account.email)) as any + } + } + + const sid = socialKeyByAccount[it.accountId] + + const person = sid !== undefined ? await accountClient.findSocialIdBySocialKey(sid) : undefined + if (person !== undefined) { + // Check/create integeration in account + + const existing = await githubAccountClient.getIntegration({ + kind: 'github', + workspaceUuid: ws?.uuid, + socialId: person + }) + + if (existing == null) { + await githubAccountClient.createIntegration({ + kind: 'github', + workspaceUuid: ws?.uuid, + socialId: person, + data: { + installationId: it.installationId + } + }) + } + } + + const users = wsToAuth.get(ws.uuid) + for (const u of users ?? []) { + if (processed.has(u._id)) { + continue + } + processed.add(u._id) + + const sid = socialKeyByAccount[u.accounts[ws.dataId ?? ws.uuid]] + if (sid !== undefined) { + const person = await accountClient.findSocialIdBySocialKey(sid) + if (person !== undefined) { + const { _id, accounts, ...data } = u + + const existing = await githubAccountClient.getIntegration({ + kind: 'github-user', + workspaceUuid: null, + socialId: person + }) + + if (existing == null) { + await githubAccountClient.createIntegration({ + kind: 'github-user', + workspaceUuid: null, + socialId: person, + data: { + login: u._id + } + }) + // Check/create integeration in account + await githubAccountClient.addIntegrationSecret({ + kind: 'github-user', + workspaceUuid: null, + socialId: person, + key: u._id, // github login + secret: JSON.stringify(data) + }) + } + } + } + } + } + } +} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 8d9fcdc6f5..96b8b0c3ae 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -71,6 +71,7 @@ import { createMongoAdapter, createMongoDestroyAdapter, createMongoTxAdapter, + getMongoClient, shutdownMongo } from '@hcengineering/mongo' import { backupDownload } from '@hcengineering/server-backup/src/backup' @@ -93,6 +94,7 @@ import { getAccountDBUrl, getMongoDBUrl } from './__start' import { changeConfiguration } from './configuration' import { moveAccountDbFromMongoToPG } from './db' +import { performGithubAccountMigrations } from './github' import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils' const colorConstants = { @@ -2399,6 +2401,20 @@ export function devTool ( // }) // }) + program + .command('migrate-github-account') + .option('--db ', 'Github DB', '%github') + .option('--region ', 'Github DB') + .action(async (cmd: { db: string, region?: string }) => { + const mongodbUri = getMongoDBUrl() + const client = getMongoClient(mongodbUri) + const _client = await client.getClient() + + await performGithubAccountMigrations(_client.db(cmd.db), cmd.region ?? null) + await _client.close() + client.close() + }) + program .command('queue-init-topics') .description('create required kafka topics') diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 3b839e72af..ace83efcd9 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -316,7 +316,7 @@ export async function getSocialKeyByOldAccount (client: MigrationClient): Promis }) const accounts = getAccountsFromTxes(accountsTxes) - const socialKeyByAccount: Record = {} + const socialKeyByAccount: Record = {} for (const account of accounts) { if (account.email === undefined) { continue diff --git a/packages/account-client/src/client.ts b/packages/account-client/src/client.ts index 5f942585ef..91d3a6d427 100644 --- a/packages/account-client/src/client.ts +++ b/packages/account-client/src/client.ts @@ -169,12 +169,12 @@ export interface AccountClient { } /** @public */ -export function getClient (accountsUrl?: string, token?: string): AccountClient { +export function getClient (accountsUrl?: string, token?: string, retryTimeoutMs?: number): AccountClient { if (accountsUrl === undefined) { throw new Error('Accounts url not specified') } - return new AccountClientImpl(accountsUrl, token) + return new AccountClientImpl(accountsUrl, token, retryTimeoutMs) } interface Request { @@ -188,7 +188,8 @@ class AccountClientImpl implements AccountClient { constructor ( private readonly url: string, - private readonly token?: string + private readonly token?: string, + retryTimeoutMs?: number ) { if (url === '') { throw new Error('Accounts url not specified') @@ -207,7 +208,7 @@ class AccountClientImpl implements AccountClient { }, ...(isBrowser ? { credentials: 'include' } : {}) } - this.rpc = withRetryUntilTimeout(this._rpc.bind(this)) + this.rpc = withRetryUntilTimeout(this._rpc.bind(this), retryTimeoutMs ?? 5000) } async getProviders (): Promise { @@ -907,7 +908,7 @@ class AccountClientImpl implements AccountClient { function withRetry Promise> ( f: F, shouldFail: (err: any, attempt: number) => boolean, - intervalMs: number = 1000 + intervalMs: number = 25 ): F { return async function (...params: any[]): Promise { let attempt = 0 @@ -921,6 +922,9 @@ function withRetry Promise> ( attempt++ await new Promise((resolve) => setTimeout(resolve, intervalMs)) + if (intervalMs < 1000) { + intervalMs += 100 + } } } } as F diff --git a/plugins/activity-resources/src/components/activity-message/ActivityMessageTemplate.svelte b/plugins/activity-resources/src/components/activity-message/ActivityMessageTemplate.svelte index 038efbaf28..c326f626ff 100644 --- a/plugins/activity-resources/src/components/activity-message/ActivityMessageTemplate.svelte +++ b/plugins/activity-resources/src/components/activity-message/ActivityMessageTemplate.svelte @@ -14,29 +14,29 @@ --> {#if spaceObj !== undefined} - {#if auth === undefined || auth.login === '' || auth.error != null} + {#if auth === undefined || auth.login === ''} {#if !readonly} = [ 'ClientSecret', 'PrivateKey', - 'MongoURL', - 'ConfigurationDB', - 'CollaboratorURL', 'BotName' @@ -101,9 +92,6 @@ const config: Config = (() => { Port: parseInt(process.env[envMap.Port] ?? '3500'), BotName: process.env[envMap.BotName] ?? 'ao-huly-dev[bot]', - MongoURL: process.env[envMap.MongoURL], - ConfigurationDB: process.env[envMap.ConfigurationDB] ?? '%github', - CollaboratorURL: process.env[envMap.CollaboratorURL], SentryDSN: process.env[envMap.SentryDSN], diff --git a/services/github/pod-github/src/platform.ts b/services/github/pod-github/src/platform.ts index 05d05bbb11..106079d12d 100644 --- a/services/github/pod-github/src/platform.ts +++ b/services/github/pod-github/src/platform.ts @@ -3,37 +3,39 @@ // // /* eslint-disable @typescript-eslint/no-unused-vars */ +import { getClient as getAccountClient } from '@hcengineering/account-client' import chunter from '@hcengineering/chunter' import core, { - PersonId, BrandingMap, + buildSocialIdString, Client, ClientConnectEvent, DocumentUpdate, isActiveMode, isDeletingMode, MeasureContext, + PersonId, RateLimiter, + SocialIdType, + systemAccountUuid, TimeRateLimiter, TxOperations, - systemAccountUuid, + WorkspaceInfoWithStatus, WorkspaceUuid, - WorkspaceInfoWithStatus + type PersonUuid, + type Ref } from '@hcengineering/core' import github, { GithubAuthentication, makeQuery, type GithubIntegration } from '@hcengineering/github' -import { getMongoClient, MongoClientReference } from '@hcengineering/mongo' import { setMetadata } from '@hcengineering/platform' import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import serverToken, { generateToken } from '@hcengineering/server-token' -import { getClient as getAccountClient } from '@hcengineering/account-client' import tracker from '@hcengineering/tracker' import { Installation, type InstallationCreatedEvent, type InstallationUnsuspendEvent } from '@octokit/webhooks-types' -import { Collection } from 'mongodb' import { App, Octokit } from 'octokit' import { Analytics } from '@hcengineering/analytics' import { SplitLogger } from '@hcengineering/analytics-service' -import contact, { Person } from '@hcengineering/contact' +import contact, { type Employee, type SocialIdentityRef } from '@hcengineering/contact' import { type StorageAdapter } from '@hcengineering/server-core' import { join } from 'path' import { createPlatformClient } from './client' @@ -57,7 +59,7 @@ export interface InstallationRecord { } export class PlatformWorker { - private readonly clients: Map = new Map() + private readonly clients = new Map() storageAdapter!: StorageAdapter @@ -65,16 +67,12 @@ export class PlatformWorker { integrations: GithubIntegrationRecord[] = [] - mongoRef!: MongoClientReference - - integrationCollection!: Collection - periodicTimer: any periodicSyncPromise: Promise | undefined canceled = false - userManager!: UserManager + userManager: UserManager = new UserManager() rateLimits = new Map() @@ -98,14 +96,6 @@ export class PlatformWorker { } public async initStorage (): Promise { - this.mongoRef = getMongoClient(config.MongoURL) - const mongoClient = await this.mongoRef.getClient() - - const db = mongoClient.db(config.ConfigurationDB) - this.integrationCollection = db.collection('installations') - - this.userManager = new UserManager(db.collection('users')) - const storageConfig = storageConfigFromEnv() this.storageAdapter = buildStorageFromConfig(storageConfig) } @@ -120,11 +110,30 @@ export class PlatformWorker { ) this.clients.clear() await this.storageAdapter.close() - this.mongoRef.close() } async init (ctx: MeasureContext): Promise { - this.integrations = await this.integrationCollection.find({}).toArray() + const sysToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'github' }) + const accountsClient = getAccountClient(config.AccountsURL, sysToken) + + const allIntegrations = await accountsClient.listIntegrations({ kind: 'github' }) + + this.integrations = [] + + for (const i of allIntegrations) { + if (i.workspaceUuid == null) { + continue + } + const installationId = i.data?.installationId + if (installationId !== undefined) { + this.integrations.push({ + accountId: i.socialId, + workspace: i.workspaceUuid, + installationId + }) + } + } + await this.queryInstallations(ctx) for (const integr of [...this.integrations]) { @@ -134,7 +143,11 @@ export class PlatformWorker { installationId: integr.installationId, workspace: integr.workspace }) - await this.integrationCollection.deleteOne({ installationId: integr.installationId }) + await accountsClient.deleteIntegration({ + kind: 'github', + workspaceUuid: integr.workspace, + socialId: integr.accountId + }) this.integrations = this.integrations.filter((it) => it.installationId !== integr.installationId) } } @@ -153,11 +166,11 @@ export class PlatformWorker { async performPeriodicSync (): Promise { // Sync authorized users information details. - const workspaces = await this.findUsersWorkspaces() - for (const [workspace, users] of workspaces) { + const workspaces = await this.getWorkspaces() + for (const workspace of workspaces) { const worker = this.clients.get(workspace) if (worker !== undefined) { - await this.ctx.with('syncUsers', {}, (ctx) => worker.syncUserData(ctx, users)) + await this.ctx.with('syncUsers', {}, (ctx) => worker.syncUserData(ctx)) } } this.periodicSyncPromise = undefined @@ -190,38 +203,19 @@ export class PlatformWorker { } } - private async findUsersWorkspaces (): Promise> { - const i = this.userManager.getAllUsers() - const workspaces = new Map() - while (await i.hasNext()) { - const userInfo = await i.next() - if (userInfo !== null) { - for (const ws of Object.keys(userInfo.accounts ?? {})) { - if (this.integrations.find((it) => it.workspace === ws) === undefined) { - // No workspace integration found, let's check workspace. - workspaces.set(ws, [...(workspaces.get(ws) ?? []), userInfo]) - } - } - } - } - await i.close() - return workspaces - } - - public async getUsers (workspace: string): Promise { - return await this.userManager.getUsers(workspace) - } - public async getUser (login: string): Promise { return await this.userManager.getAccount(login) } async mapInstallation ( ctx: MeasureContext, - workspace: string, + workspace: WorkspaceUuid, installationId: number, accountId: PersonId ): Promise { + const sysToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'github' }) + const accountsClient = getAccountClient(config.AccountsURL, sysToken) + const oldInstallation = this.integrations.find((it) => it.installationId === installationId) if (oldInstallation != null) { ctx.info('update integration', { workspace, installationId, accountId }) @@ -231,10 +225,19 @@ export class PlatformWorker { // const oldWorkspace = oldInstallation.workspace - await this.integrationCollection.updateOne( - { installationId: oldInstallation.installationId }, - { $set: { workspace } } - ) + await accountsClient.createIntegration({ + kind: 'github', + workspaceUuid: workspace, + socialId: accountId, + data: { installationId: oldInstallation.installationId } + }) + + await accountsClient.deleteIntegration({ + kind: 'github', + workspaceUuid: oldWorkspace, + socialId: accountId + }) + oldInstallation.workspace = workspace const oldWorker = this.clients.get(oldWorkspace) as GithubWorker @@ -244,7 +247,7 @@ export class PlatformWorker { } else { let client: Client | undefined try { - ;({ client } = await createPlatformClient(oldWorkspace as WorkspaceUuid, 30000)) // TODO: FIXME + ;({ client } = await createPlatformClient(oldWorkspace, 30000)) await this.removeInstallationFromWorkspace(oldWorker, installationId) await client.close() } catch (err: any) { @@ -270,7 +273,13 @@ export class PlatformWorker { ctx.info('add integration', { workspace, installationId, accountId }) await ctx.with('add integration', { workspace, installationId, accountId }, async (ctx) => { - await this.integrationCollection.insertOne(record) + await accountsClient.createIntegration({ + kind: 'github', + workspaceUuid: record.workspace, + socialId: record.accountId, + data: { installationId: record.installationId } + }) + this.integrations.push(record) }) // We need to query installations to be sure we have it, in case event is delayed or not received. @@ -313,10 +322,10 @@ export class PlatformWorker { } async requestGithubAccessToken (payload: { - workspace: string + workspace: WorkspaceUuid code: string state: string - accountId: PersonId + accountId: PersonId // Primary social Id }): Promise { try { const uri = @@ -346,6 +355,7 @@ export class PlatformWorker { const user = await okit.rest.users.getAuthenticated() const nowTime = Date.now() / 1000 const dta: GithubUserRecord = { + account: payload.accountId, _id: user.data.login, token: resultJson.access_token, code: null, @@ -363,7 +373,7 @@ export class PlatformWorker { if (existingUser == null) { await this.userManager.insertUser(dta) } else { - dta.accounts = { ...existingUser.accounts, [payload.workspace]: payload.accountId } + dta.accounts = { ...existingUser.accounts, [payload.workspace]: payload.accountId } // Put primary socialId for now. await this.userManager.updateUser(dta) } @@ -388,150 +398,189 @@ export class PlatformWorker { } private async updateAccountAuthRecord ( - payload: { workspace: string, accountId: PersonId }, + payload: { workspace: WorkspaceUuid, accountId: PersonId }, update: DocumentUpdate, dta: GithubUserRecord | undefined, revoke: boolean ): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // try { - // let platformClient: Client | undefined - // let shouldClose = false - // try { - // platformClient = this.clients.get(payload.workspace)?.client - // if (platformClient === undefined) { - // shouldClose = true - // ;({ client: platformClient } = await createPlatformClient(payload.workspace, 30000)) - // } - // const client = new TxOperations(platformClient, payload.accountId) + try { + let platformClient: Client | undefined + let shouldClose = false + try { + platformClient = this.clients.get(payload.workspace)?.client + if (platformClient === undefined) { + shouldClose = true + ;({ client: platformClient } = await createPlatformClient(payload.workspace, 30000)) + } + const client = new TxOperations(platformClient, payload.accountId) - // let personAuths = await client.findAll(github.class.GithubAuthentication, { - // attachedTo: payload.accountId - // }) - // if (personAuths.length > 1) { - // for (const auth of personAuths.slice(1)) { - // await client.remove(auth) - // } - // personAuths.length = 1 - // } + let personAuths = await client.findAll(github.class.GithubAuthentication, { + attachedTo: payload.accountId + }) + if (personAuths.length > 1) { + for (const auth of personAuths.slice(1)) { + await client.remove(auth) + } + personAuths.length = 1 + } - // if (revoke) { - // for (const personAuth of personAuths) { - // await client.remove(personAuth, Date.now(), payload.accountId) - // } - // } else { - // if (personAuths.length > 0) { - // await client.update(personAuths[0], update, false, Date.now(), payload.accountId) - // } else if (dta !== undefined) { - // const authId = await client.createDoc( - // github.class.GithubAuthentication, - // core.space.Workspace, - // { - // error: null, - // authRequestTime: Date.now(), - // createdAt: new Date(), - // followers: 0, - // following: 0, - // nodeId: '', - // updatedAt: new Date(), - // url: '', - // repositories: 0, - // organizations: { totalCount: 0, nodes: [] }, - // closedIssues: 0, - // openIssues: 0, - // mergedPRs: 0, - // openPRs: 0, - // closedPRs: 0, - // repositoryDiscussions: 0, - // starredRepositories: 0, - // ...update, - // attachedTo: payload.accountId, - // login: dta._id - // }, - // undefined, - // undefined, - // payload.accountId - // ) + if (revoke) { + for (const personAuth of personAuths) { + await client.remove(personAuth, Date.now(), payload.accountId) + } - // personAuths = await client.findAll(github.class.GithubAuthentication, { - // _id: authId - // }) - // } - // } + // TODO: Do we need to remove social ids? + } else { + if (personAuths.length > 0) { + await client.update(personAuths[0], update, false, Date.now(), payload.accountId) + } else if (dta !== undefined) { + const authId = await client.createDoc( + github.class.GithubAuthentication, + core.space.Workspace, + { + error: null, + authRequestTime: Date.now(), + createdAt: new Date(), + followers: 0, + following: 0, + nodeId: '', + updatedAt: new Date(), + url: '', + repositories: 0, + organizations: { totalCount: 0, nodes: [] }, + closedIssues: 0, + openIssues: 0, + mergedPRs: 0, + openPRs: 0, + closedPRs: 0, + repositoryDiscussions: 0, + starredRepositories: 0, + ...update, + attachedTo: payload.accountId, + login: dta._id + }, + undefined, + undefined, + payload.accountId + ) - // // We need to re-bind previously created github:login account to a proper person. - // const account = client.getModel().getObject(payload.accountId) as PersonAccount - // const person = (await client.findOne(contact.class.Person, { _id: account.person })) as Person - // if (person !== undefined) { - // if (!revoke) { - // const personSpace = await client.findOne(contact.class.PersonSpace, { person: person._id }) - // if (personSpace !== undefined) { - // await createNotification(client, person, { - // user: account._id, - // space: personSpace._id, - // message: github.string.AuthenticatedWithGithub, - // props: { - // login: update.login - // } - // }) - // } + personAuths = await client.findAll(github.class.GithubAuthentication, { + _id: authId + }) + } + } - // const githubAccount = client.getModel().getAccountByEmail('github:' + update.login) as PersonAccount - // if (githubAccount !== undefined && githubAccount.person !== account.person) { - // const dummyPerson = githubAccount.person - // // To add activity entry to dummy person. - // await client.update(githubAccount, { person: account.person }, false, Date.now(), payload.accountId) + const account = await client.findOne(contact.class.SocialIdentity, { + _id: payload.accountId as SocialIdentityRef + }) + const person = + account !== undefined + ? await client.findOne(contact.mixin.Employee, { _id: account?.attachedTo as Ref }) + : undefined + if (person !== undefined) { + if (!revoke) { + const personSpace = await client.findOne(contact.class.PersonSpace, { person: person._id }) + if (personSpace !== undefined && person.personUuid !== undefined) { + await createNotification(client, person, { + user: person.personUuid, + space: personSpace._id, + message: github.string.AuthenticatedWithGithub, + props: { + login: update.login + } + }) + } - // const dPerson = (await client.findOne(contact.class.Person, { _id: dummyPerson })) as Person - // if (person !== undefined && dPerson !== undefined) { - // const personSpace = await client.findOne(contact.class.PersonSpace, { person: person._id }) - // if (personSpace !== undefined) { - // await createNotification(client, dPerson, { - // user: githubAccount._id, - // space: personSpace._id, - // message: github.string.AuthenticatedWithGithubEmployee, - // props: { - // login: update.login - // } - // }) - // } - // } - // } - // } else { - // const personSpace = await client.findOne(contact.class.PersonSpace, { person: person._id }) - // if (personSpace !== undefined) { - // await createNotification(client, person, { - // user: account._id, - // space: personSpace._id, - // message: github.string.AuthenticationRevokedGithub, - // props: { - // login: update.login - // } - // }) - // } - // } - // } + if (dta?._id !== undefined) { + const sysToken = generateToken(systemAccountUuid, payload.workspace, { + service: 'github' + }) + const userToken = generateToken(person.personUuid as PersonUuid, payload.workspace, { + service: 'github' + }) + const sysAccountClient = getAccountClient(config.AccountsURL, sysToken) + const userAccountClient = getAccountClient(config.AccountsURL, userToken) - // if (dta !== undefined && personAuths.length === 1) { - // try { - // await syncUser(this.ctx, dta, personAuths[0], client, payload.accountId) - // } catch (err: any) { - // if (err.response?.data?.message === 'Bad credentials') { - // await this.revokeUserAuth(dta) - // } else { - // this.ctx.error(`Failed to sync user ${dta._id}`, { error: errorToObj(err) }) - // } - // } - // } - // } finally { - // if (shouldClose) { - // await platformClient?.close() - // } - // } - // } catch (err: any) { - // Analytics.handleError(err) - // } + const ids = await userAccountClient.getSocialIds() + + let githubSocialId: PersonId | undefined = ids.find( + (it) => it.type === SocialIdType.GITHUB && it.value === dta?._id + )?._id + // We need to assign socialId to person in global account if missing and get it to match if exists. + + if (githubSocialId === undefined) { + // We need to create a new social id for this account. + githubSocialId = await sysAccountClient.addSocialIdToPerson( + person.personUuid as PersonUuid, + SocialIdType.GITHUB, + dta?._id ?? '', + true + ) + } + + const socialIdentity = await client.findOne(contact.class.SocialIdentity, { + _id: githubSocialId as SocialIdentityRef + }) + if (socialIdentity === undefined) { + // We need to create a new social id for this account. + + // We need to create social id github account + await client.addCollection( + contact.class.SocialIdentity, + contact.space.Contacts, + person._id, + contact.class.Person, + 'socialIds', + { + type: SocialIdType.GITHUB, + value: dta._id, + key: buildSocialIdString({ + type: SocialIdType.GITHUB, + value: dta._id + }), + verifiedOn: Date.now() + }, + githubSocialId as SocialIdentityRef + ) + } + } + } else { + const personSpace = await client.findOne(contact.class.PersonSpace, { person: person._id }) + if (personSpace !== undefined && person.personUuid !== undefined) { + await createNotification(client, person, { + user: person.personUuid, + space: personSpace._id, + message: github.string.AuthenticationRevokedGithub, + props: { + login: update.login + } + }) + } + } + } + + if (dta !== undefined && personAuths.length === 1) { + try { + await syncUser(this.ctx, dta, personAuths[0], client, payload.accountId) + } catch (err: any) { + if (err.response?.data?.message === 'Bad credentials') { + await this.revokeUserAuth(dta) + } else { + this.ctx.error(`Failed to sync user ${dta._id}`, { error: errorToObj(err) }) + } + } + } + } catch (err: any) { + this.ctx.error('error workspace update', { err }) + Analytics.handleError(err) + } finally { + if (shouldClose) { + await platformClient?.close() + } + } + } catch (err: any) { + Analytics.handleError(err) + } } async checkRefreshToken (auth: GithubUserRecord, force: boolean = false): Promise { @@ -584,7 +633,7 @@ export class PlatformWorker { return await this.userManager.getAccount(login) } - async getAccountByRef (workspace: string, ref: PersonId): Promise { + async getAccountByRef (workspace: WorkspaceUuid, ref: PersonId): Promise { return await this.userManager.getAccountByRef(workspace, ref) } @@ -668,7 +717,7 @@ export class PlatformWorker { integeration.enabled = enabled } - await worker.syncUserData(this.ctx, await this.getUsers(worker.workspace.uuid)) + await worker.syncUserData(this.ctx) await worker.reloadRepositories(install.id) worker.triggerUpdate() @@ -705,23 +754,29 @@ export class PlatformWorker { // No worker } this.integrations = this.integrations.filter((it) => it.installationId !== installId) - await this.integrationCollection.deleteOne({ installationId: installId }) + if (interg !== undefined) { + const sysToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'github' }) + const sysAccountClient = getAccountClient(config.AccountsURL, sysToken) + await sysAccountClient.deleteIntegration({ + kind: 'github', + workspaceUuid: interg.workspace, + socialId: interg.accountId + }) + } this.triggerCheckWorkspaces() } async getWorkspaces (): Promise { - const workspaces = new Set(this.integrations.map((it) => it.workspace as WorkspaceUuid)) // TODO: FIXME - - return Array.from(workspaces) + return this.integrations.map((it) => it.workspace) } async checkWorkspaceIsActive ( token: string, - workspace: string + workspace: WorkspaceUuid ): Promise<{ workspaceInfo: WorkspaceInfoWithStatus | undefined, needRecheck: boolean }> { let workspaceInfo: WorkspaceInfoWithStatus | undefined try { - workspaceInfo = await getAccountClient(token).getWorkspaceInfo(true) + workspaceInfo = await getAccountClient(config.AccountsURL, token).getWorkspaceInfo(false) } catch (err: any) { this.ctx.error('Workspace not found:', { workspace }) return { workspaceInfo: undefined, needRecheck: false } @@ -752,11 +807,8 @@ export class PlatformWorker { this.ctx.info('************************* Check workspaces ************************* ', { workspaces: this.clients.size }) - let workspaces = await this.getWorkspaces() - if (process.env.GITHUB_USE_WS !== undefined) { - workspaces = [process.env.GITHUB_USE_WS as WorkspaceUuid] - } - const toDelete = new Set(this.clients.keys()) + const workspaces = await this.getWorkspaces() + const toDelete = new Set(this.clients.keys()) const rateLimiter = new RateLimiter(5) let errors = 0 @@ -1094,8 +1146,7 @@ export class PlatformWorker { payload.installation.html_url ) const doSyncUsers = async (worker: GithubWorker): Promise => { - const users = await this.getUsers(worker.workspace.uuid) - await worker.syncUserData(this.ctx, users) + await worker.syncUserData(this.ctx) } catchEventError(doSyncUsers(worker), payload.action, name, id, payload.installation.html_url) }) @@ -1141,7 +1192,12 @@ export class PlatformWorker { public async revokeUserAuth (record: GithubUserRecord): Promise { for (const [ws, acc] of Object.entries(record.accounts)) { - await this.updateAccountAuthRecord({ workspace: ws, accountId: acc }, { login: record._id }, undefined, true) + await this.updateAccountAuthRecord( + { workspace: ws as WorkspaceUuid, accountId: acc }, + { login: record._id }, + undefined, + true + ) } } diff --git a/services/github/pod-github/src/server.ts b/services/github/pod-github/src/server.ts index 1a3a3e9fa6..883c1dfc97 100644 --- a/services/github/pod-github/src/server.ts +++ b/services/github/pod-github/src/server.ts @@ -73,37 +73,35 @@ export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Pro // eslint-disable-next-line @typescript-eslint/no-misused-promises app.post('/api/v1/installation', async (req, res) => { - // TODO: FIXME - throw new Error('Not implemented') - // const payloadData: { - // installationId: number - // accountId: PersonId - // token: string - // } = req.body - // try { - // const decodedToken = decodeToken(payloadData.token) - // ctx.info('/api/v1/installation', { - // email: decodedToken.email, - // workspaceName: decodedToken.workspace, - // body: req.body - // }) + const payloadData: { + installationId: number + accountId: PersonId + token: string + } = req.body + try { + const decodedToken = decodeToken(payloadData.token) + ctx.info('/api/v1/installation', { + email: decodedToken.account, + workspaceName: decodedToken.workspace, + body: req.body + }) - // await ctx.with('map-installation', {}, (ctx) => - // worker.mapInstallation(ctx, decodedToken.workspace, payloadData.installationId, payloadData.accountId) - // ) - // res.status(200) - // res.json({}) - // } catch (err: any) { - // Analytics.handleError(err) - // const tok = decodeToken(payloadData.token, false) - // ctx.error('failed to map-installation', { - // workspace: tok.workspace, - // installationid: payloadData.installationId, - // email: tok?.email - // }) - // res.status(401) - // res.json({ error: err.message }) - // } + await ctx.with('map-installation', {}, (ctx) => + worker.mapInstallation(ctx, decodedToken.workspace, payloadData.installationId, payloadData.accountId) + ) + res.status(200) + res.json({}) + } catch (err: any) { + Analytics.handleError(err) + const tok = decodeToken(payloadData.token, false) + ctx.error('failed to map-installation', { + workspace: tok.workspace, + installationid: payloadData.installationId, + email: tok?.account + }) + res.status(401) + res.json({ error: err.message }) + } }) // eslint-disable-next-line @typescript-eslint/no-misused-promises @@ -148,35 +146,33 @@ export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Pro // eslint-disable-next-line @typescript-eslint/no-misused-promises app.post('/api/v1/installation-remove', async (req, res) => { - // TODO: FIXME - throw new Error('Not implemented') - // try { - // const payloadData: { - // installationId: number - // token: string - // } = req.body + try { + const payloadData: { + installationId: number + token: string + } = req.body - // const decodedToken = decodeToken(payloadData.token) - // ctx.info('/api/v1/installation-remove', { - // email: decodedToken.email, - // workspaceName: decodedToken.workspace, - // body: req.body - // }) + const decodedToken = decodeToken(payloadData.token) + ctx.info('/api/v1/installation-remove', { + email: decodedToken.account, + workspaceName: decodedToken.workspace, + body: req.body + }) - // ctx.info('remove-installation', { - // workspace: decodedToken.workspace, - // installationId: payloadData.installationId - // }) - // await ctx.with('remove-installation', {}, (ctx) => - // worker.removeInstallation(ctx, decodedToken.workspace, payloadData.installationId) - // ) - // res.status(200) - // res.json({}) - // } catch (err: any) { - // Analytics.handleError(err) - // res.status(401) - // res.json({ error: err.message }) - // } + ctx.info('remove-installation', { + workspace: decodedToken.workspace, + installationId: payloadData.installationId + }) + await ctx.with('remove-installation', {}, (ctx) => + worker.removeInstallation(ctx, decodedToken.workspace, payloadData.installationId) + ) + res.status(200) + res.json({}) + } catch (err: any) { + Analytics.handleError(err) + res.status(401) + res.json({ error: err.message }) + } }) const server = app.listen(port, () => { diff --git a/services/github/pod-github/src/sync/comments.ts b/services/github/pod-github/src/sync/comments.ts index 322f7bb0d5..695e54facc 100644 --- a/services/github/pod-github/src/sync/comments.ts +++ b/services/github/pod-github/src/sync/comments.ts @@ -104,7 +104,7 @@ export class CommentSyncManager implements DocSyncManager { return true } const account = - existing?.createdBy ?? (await this.provider.getAccountU(commentExternal.user))?._id ?? core.account.System + existing?.createdBy ?? (await this.provider.getAccountU(commentExternal.user)) ?? core.account.System if (commentExternal !== undefined) { try { @@ -164,7 +164,7 @@ export class CommentSyncManager implements DocSyncManager { return } - const account = (await this.provider.getAccountU(event.sender))?._id ?? core.account.System + const account = (await this.provider.getAccountU(event.sender)) ?? core.account.System switch (event.action) { case 'created': { await this.createSyncData(event, derivedClient, repo) @@ -277,7 +277,7 @@ export class CommentSyncManager implements DocSyncManager { return { needSync: githubSyncVersion } } - const account = existing?.modifiedBy ?? (await this.provider.getAccountU(comment.user))?._id ?? core.account.System + const account = existing?.modifiedBy ?? (await this.provider.getAccountU(comment.user)) ?? core.account.System const messageData: MessageData = { message: await this.provider.getMarkupSafe(container.container, comment.body) diff --git a/services/github/pod-github/src/sync/issueBase.ts b/services/github/pod-github/src/sync/issueBase.ts index 28e0e67d60..afe0f82d63 100644 --- a/services/github/pod-github/src/sync/issueBase.ts +++ b/services/github/pod-github/src/sync/issueBase.ts @@ -10,14 +10,15 @@ import activity from '@hcengineering/activity' import { Analytics } from '@hcengineering/analytics' import { CollaboratorClient } from '@hcengineering/collaborator-client' +import contact, { Person } from '@hcengineering/contact' import core, { - PersonId, AttachedDoc, Class, Doc, DocumentUpdate, Markup, MeasureContext, + PersonId, Ref, Space, Status, @@ -29,7 +30,6 @@ import github, { GithubFieldMapping, GithubIntegrationRepository, GithubIssue, - GithubIssue as GithubIssueP, GithubMilestone, GithubProject } from '@hcengineering/github' @@ -138,19 +138,22 @@ export abstract class IssueSyncManagerBase { this.provider = provider } - async getAssignees (issue: IssueExternalData): Promise { - // TODO: FIXME - throw new Error('Not implemented') + async getAssignees (issue: IssueExternalData): Promise[]> { // Find Assignees and reviewers - // const assignees: PersonAccount[] = [] + const assignees: PersonId[] = [] - // for (const o of issue.assignees.nodes) { - // const acc = await this.provider.getAccount(o) - // if (acc !== undefined) { - // assignees.push(acc) - // } - // } - // return assignees + for (const o of issue.assignees.nodes) { + const acc = await this.provider.getAccount(o) + if (acc !== undefined) { + assignees.push(acc) + } + } + return await this.getPersonsFromId(assignees) + } + + async getPersonsFromId (assignees: PersonId[]): Promise[]> { + const socialIds = await this.client.findAll(contact.class.SocialIdentity, { _id: { $in: assignees as any } }) + return socialIds.map((it) => it.attachedTo) } async processProjectV2Event ( @@ -159,7 +162,7 @@ export abstract class IssueSyncManagerBase { derivedClient: TxOperations, prj: GithubProject ): Promise { - const account = (await this.provider.getAccountU(event.sender))?._id ?? core.account.System + const account = (await this.provider.getAccountU(event.sender)) ?? core.account.System switch (event.action) { case 'edited': { const itemId = event.projects_v2_item.node_id @@ -708,263 +711,261 @@ export abstract class IssueSyncManagerBase { accountGH: PersonId, syncToProject: boolean ): Promise> { - // TODO: FIXME - throw new Error('Not implemented') - // let needUpdate = false - // if (!this.client.getHierarchy().hasMixin(existing, github.mixin.GithubIssue)) { - // await this.ctx.withLog( - // 'create mixin issue: GithubIssue', - // {}, - // async () => { - // await this.client.createMixin( - // existing._id as Ref, - // existing._class, - // existing.space, - // github.mixin.GithubIssue, - // { - // githubNumber: issueExternal.number, - // url: issueExternal.url, - // repository: info.repository as Ref - // } - // ) - // await this.notifyConnected(container, info, existing, issueExternal) - // }, - // { identifier: existing.identifier, url: issueExternal.url } - // ) - // // Re iterate to have existing value with mixin inside. - // needUpdate = true - // } else { - // const ghIssue = this.client.getHierarchy().as(existing, github.mixin.GithubIssue) - // await this.client.diffUpdate(ghIssue, { - // githubNumber: issueExternal.number, - // url: issueExternal.url, - // repository: info.repository as Ref - // }) - // if (ghIssue.url !== issueExternal.url) { - // await this.notifyConnected(container, info, existing, issueExternal) - // } - // } - // if (!this.client.getHierarchy().hasMixin(existing, container.project.mixinClass)) { - // await this.ctx.withLog( - // 'create mixin issue', - // {}, - // () => - // this.client.createMixin( - // existing._id as Ref, - // existing._class, - // existing.space, - // container.project.mixinClass, - // {} - // ), - // { identifier: existing.identifier, url: issueExternal.url } - // ) - // // Re iterate to have existing value with mixin inside. - // needUpdate = true - // } - // if (needUpdate) { - // return { needSync: '' } - // } + let needUpdate = false + if (!this.client.getHierarchy().hasMixin(existing, github.mixin.GithubIssue)) { + await this.ctx.withLog( + 'create mixin issue: GithubIssue', + {}, + async () => { + await this.client.createMixin( + existing._id as Ref, + existing._class, + existing.space, + github.mixin.GithubIssue, + { + githubNumber: issueExternal.number, + url: issueExternal.url, + repository: info.repository as Ref + } + ) + await this.notifyConnected(container, info, existing, issueExternal) + }, + { identifier: existing.identifier, url: issueExternal.url } + ) + // Re iterate to have existing value with mixin inside. + needUpdate = true + } else { + const ghIssue = this.client.getHierarchy().as(existing, github.mixin.GithubIssue) + await this.client.diffUpdate(ghIssue, { + githubNumber: issueExternal.number, + url: issueExternal.url, + repository: info.repository as Ref + }) + if (ghIssue.url !== issueExternal.url) { + await this.notifyConnected(container, info, existing, issueExternal) + } + } + if (!this.client.getHierarchy().hasMixin(existing, container.project.mixinClass)) { + await this.ctx.withLog( + 'create mixin issue', + {}, + () => + this.client.createMixin( + existing._id as Ref, + existing._class, + existing.space, + container.project.mixinClass, + {} + ), + { identifier: existing.identifier, url: issueExternal.url } + ) + // Re iterate to have existing value with mixin inside. + needUpdate = true + } + if (needUpdate) { + return { needSync: '' } + } - // const existingIssue = this.client.getHierarchy().as(existing, container.project.mixinClass) - // const previousData: GithubIssueData = info.current ?? ({} as unknown as GithubIssueData) - // const type = await this.provider.getTaskTypeOf(container.project.type, existing._class) - // const stst = await this.provider.getStatuses(type?._id) + const existingIssue = this.client.getHierarchy().as(existing, container.project.mixinClass) + const previousData: GithubIssueData = info.current ?? ({} as unknown as GithubIssueData) + const type = await this.provider.getTaskTypeOf(container.project.type, existing._class) + const stst = await this.provider.getStatuses(type?._id) - // const update = collectUpdate(previousData, issueData, Object.keys(issueData)) + const update = collectUpdate(previousData, issueData, Object.keys(issueData)) - // const allAttributes = this.client.getHierarchy().getAllAttributes(container.project.mixinClass) - // const platformUpdate = collectUpdate(previousData, existingIssue, Array.from(allAttributes.keys())) + const allAttributes = this.client.getHierarchy().getAllAttributes(container.project.mixinClass) + const platformUpdate = collectUpdate(previousData, existingIssue, Array.from(allAttributes.keys())) - // const okit = (await this.provider.getOctokit(account as PersonId)) ?? container.container.octokit + const okit = (await this.provider.getOctokit(account)) ?? container.container.octokit - // // Remove current same values from update - // for (const [k, v] of Object.entries(update)) { - // if ((existingIssue as any)[k] === v) { - // // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - // delete (update as any)[k] - // } - // } + // Remove current same values from update + for (const [k, v] of Object.entries(update)) { + if ((existingIssue as any)[k] === v) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete (update as any)[k] + } + } - // if (update.description !== undefined) { - // if (update.description === existingIssue.description) { - // delete update.description - // } - // } + if (update.description !== undefined) { + if (update.description === existingIssue.description) { + delete update.description + } + } - // for (const [k, v] of Object.entries(update)) { - // let pv = (platformUpdate as any)[k] + for (const [k, v] of Object.entries(update)) { + let pv = (platformUpdate as any)[k] - // if (k === 'description' && pv != null) { - // const mdown = await this.provider.getMarkdown(pv) - // pv = await this.provider.getMarkupSafe(container.container, mdown, this.stripGuestLink) - // } - // if (pv != null && pv !== v) { - // // We have conflict of values, assume platform is more proper one. - // this.ctx.error('conflict', { id: existing.identifier, k }) - // // Assume platform change is more important in case of conflict values. - // // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - // delete (update as any)[k] - // continue - // } - // } + if (k === 'description' && pv != null) { + const mdown = await this.provider.getMarkdown(pv) + pv = await this.provider.getMarkupSafe(container.container, mdown, this.stripGuestLink) + } + if (pv != null && pv !== v) { + // We have conflict of values, assume platform is more proper one. + this.ctx.error('conflict', { id: existing.identifier, k }) + // Assume platform change is more important in case of conflict values. + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete (update as any)[k] + continue + } + } - // await this.fillBackChanges(update, existingIssue, issueExternal) + await this.fillBackChanges(update, existingIssue, issueExternal) - // let needExternalSync = false + let needExternalSync = false - // if (container !== undefined && okit !== undefined) { - // // Check and update issue fields. - // needExternalSync = await this.performIssueFieldsUpdate( - // info, - // existing, - // platformUpdate, - // issueData, - // container, - // issueExternal, - // okit, - // account - // ) + if (container !== undefined && okit !== undefined) { + // Check and update issue fields. + needExternalSync = await this.performIssueFieldsUpdate( + info, + existing, + platformUpdate, + issueData, + container, + issueExternal, + okit, + account + ) - // const fieldsUpdate: { id: string, value: any, dataType: GithubDataType }[] = [] + const fieldsUpdate: { id: string, value: any, dataType: GithubDataType }[] = [] - // // Collect field update. - // for (const [k, v] of Object.entries(platformUpdate)) { - // const mapping = target.mappings.filter((it) => it != null).find((it) => it.name === k) - // if (mapping === undefined) { - // continue - // } - // const attr = this.client.getHierarchy().getAttribute(mapping._class, mapping.name) + // Collect field update. + for (const [k, v] of Object.entries(platformUpdate)) { + const mapping = target.mappings.filter((it) => it != null).find((it) => it.name === k) + if (mapping === undefined) { + continue + } + const attr = this.client.getHierarchy().getAttribute(mapping._class, mapping.name) - // if (attr.name === 'status') { - // // Handle status field - // const status = stst.find((it) => it._id === v) as Status - // const optionId = this.findOptionId(container, mapping.githubId, status.name, target) - // if (optionId !== undefined) { - // fieldsUpdate.push({ - // id: mapping.githubId, - // dataType: 'SINGLE_SELECT', - // value: optionId - // }) - // this.ctx.info(' => prepare issue status update', { - // url: issueExternal.url, - // name: status.name, - // workspace: this.provider.getWorkspaceId() - // }) - // continue - // } - // } - // if (attr.name === 'priority') { - // const values: Record = { - // [IssuePriority.NoPriority]: '', - // [IssuePriority.High]: 'High', - // [IssuePriority.Medium]: 'Medium', - // [IssuePriority.Low]: 'Low', - // [IssuePriority.Urgent]: 'Urgent' - // } - // // Handle priority field TODO: Add clear of field - // const priorityName = values[v as IssuePriority] - // const optionId = this.findOptionId(container, mapping.githubId, priorityName, target) - // if (optionId !== undefined) { - // fieldsUpdate.push({ - // id: mapping.githubId, - // dataType: 'SINGLE_SELECT', - // value: optionId - // }) - // this.ctx.info(' => prepare issue priority update', { - // url: issueExternal.url, - // priority: priorityName, - // workspace: this.provider.getWorkspaceId() - // }) - // continue - // } - // } + if (attr.name === 'status') { + // Handle status field + const status = stst.find((it) => it._id === v) as Status + const optionId = this.findOptionId(container, mapping.githubId, status.name, target) + if (optionId !== undefined) { + fieldsUpdate.push({ + id: mapping.githubId, + dataType: 'SINGLE_SELECT', + value: optionId + }) + this.ctx.info(' => prepare issue status update', { + url: issueExternal.url, + name: status.name, + workspace: this.provider.getWorkspaceId() + }) + continue + } + } + if (attr.name === 'priority') { + const values: Record = { + [IssuePriority.NoPriority]: '', + [IssuePriority.High]: 'High', + [IssuePriority.Medium]: 'Medium', + [IssuePriority.Low]: 'Low', + [IssuePriority.Urgent]: 'Urgent' + } + // Handle priority field TODO: Add clear of field + const priorityName = values[v as IssuePriority] + const optionId = this.findOptionId(container, mapping.githubId, priorityName, target) + if (optionId !== undefined) { + fieldsUpdate.push({ + id: mapping.githubId, + dataType: 'SINGLE_SELECT', + value: optionId + }) + this.ctx.info(' => prepare issue priority update', { + url: issueExternal.url, + priority: priorityName, + workspace: this.provider.getWorkspaceId() + }) + continue + } + } - // const dataType = getType(attr) - // if (dataType === 'SINGLE_SELECT') { - // // Handle status field - // const optionId = this.findOptionId(container, mapping.githubId, v, target) - // if (optionId !== undefined) { - // fieldsUpdate.push({ - // id: mapping.githubId, - // dataType: 'SINGLE_SELECT', - // value: optionId - // }) - // this.ctx.info(` => prepare issue field ${attr.label} update`, { - // url: issueExternal.url, - // value: v, - // workspace: this.provider.getWorkspaceId() - // }) - // continue - // } - // } + const dataType = getType(attr) + if (dataType === 'SINGLE_SELECT') { + // Handle status field + const optionId = this.findOptionId(container, mapping.githubId, v, target) + if (optionId !== undefined) { + fieldsUpdate.push({ + id: mapping.githubId, + dataType: 'SINGLE_SELECT', + value: optionId + }) + this.ctx.info(` => prepare issue field ${attr.label} update`, { + url: issueExternal.url, + value: v, + workspace: this.provider.getWorkspaceId() + }) + continue + } + } - // if (dataType === undefined) { - // continue - // } - // fieldsUpdate.push({ - // id: mapping.githubId, - // dataType, - // value: v - // }) - // this.ctx.info(`=> prepare issue field ${attr.label} update`, { - // url: issueExternal.url, - // value: v, - // workspace: this.provider.getWorkspaceId() - // }) - // } - // if (fieldsUpdate.length > 0 && syncToProject && target.prjData !== undefined) { - // const errors = await this.updateIssueValues(target, okit, fieldsUpdate) - // if (errors.length === 0) { - // needExternalSync = true - // } - // } - // // TODO: Add support for labels, milestone, assignees - // } + if (dataType === undefined) { + continue + } + fieldsUpdate.push({ + id: mapping.githubId, + dataType, + value: v + }) + this.ctx.info(`=> prepare issue field ${attr.label} update`, { + url: issueExternal.url, + value: v, + workspace: this.provider.getWorkspaceId() + }) + } + if (fieldsUpdate.length > 0 && syncToProject && target.prjData !== undefined) { + const errors = await this.updateIssueValues(target, okit, fieldsUpdate) + if (errors.length === 0) { + needExternalSync = true + } + } + // TODO: Add support for labels, milestone, assignees + } - // // We need remove all readonly field values - // for (const k of Object.keys(update)) { - // // Skip readonly fields - // const attr = this.client.getHierarchy().findAttribute(target.project.mixinClass, k) - // if (attr?.readonly === true) { - // // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - // delete (update as any)[k] - // continue - // } - // } + // We need remove all readonly field values + for (const k of Object.keys(update)) { + // Skip readonly fields + const attr = this.client.getHierarchy().findAttribute(target.project.mixinClass, k) + if (attr?.readonly === true) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete (update as any)[k] + continue + } + } - // // Update collaborative description - // if (update.description !== undefined) { - // this.ctx.info(`<= perform ${issueExternal.url} update to collaborator`, { - // workspace: this.provider.getWorkspaceId() - // }) - // try { - // const description = update.description as Markup - // issueData.description = description - // const collabId = makeDocCollabId(existingIssue, 'description') - // await this.collaborator.updateMarkup(collabId, description) - // } catch (err: any) { - // Analytics.handleError(err) - // this.ctx.error('error during description update', err) - // } - // } + // Update collaborative description + if (update.description !== undefined) { + this.ctx.info(`<= perform ${issueExternal.url} update to collaborator`, { + workspace: this.provider.getWorkspaceId() + }) + try { + const description = update.description as Markup + issueData.description = description + const collabId = makeDocCollabId(existingIssue, 'description') + await this.collaborator.updateMarkup(collabId, description) + } catch (err: any) { + Analytics.handleError(err) + this.ctx.error('error during description update', err) + } + } - // if (Object.keys(update).length > 0) { - // // We have some fields to update of existing from external - // this.ctx.info(`<= perform ${issueExternal.url} update to platform`, { - // ...update, - // workspace: this.provider.getWorkspaceId() - // }) - // await this.client.update(existingIssue, update, false, new Date().getTime(), accountGH) - // } + if (Object.keys(update).length > 0) { + // We have some fields to update of existing from external + this.ctx.info(`<= perform ${issueExternal.url} update to platform`, { + ...update, + workspace: this.provider.getWorkspaceId() + }) + await this.client.update(existingIssue, update, false, new Date().getTime(), accountGH) + } - // await this.afterSync(existingIssue, accountGH, issueExternal, info) - // // We need to trigger external version retrieval, via sync or event, to prevent move sync operations from platform before we will be sure all is updated on github. - // return { - // current: issueData, - // needSync: githubSyncVersion, - // ...(needExternalSync ? { externalVersion: '' } : {}), - // lastGithubUser: null - // } + await this.afterSync(existingIssue, accountGH, issueExternal, info) + // We need to trigger external version retrieval, via sync or event, to prevent move sync operations from platform before we will be sure all is updated on github. + return { + current: issueData, + needSync: githubSyncVersion, + ...(needExternalSync ? { externalVersion: '' } : {}), + lastGithubUser: null + } } private async notifyConnected ( @@ -997,83 +998,81 @@ export abstract class IssueSyncManagerBase { issueExternal: IssueExternalData, _class: Ref> ): Promise> { - // TODO: FIXME - throw new Error('Not implemented') - // const issueUpdate: { - // title?: string - // body?: string - // stateReason?: string - // assigneeIds?: string[] - // } & Record = {} - // if (platformUpdate.title != null) { - // if (platformUpdate.title !== issueExternal.title) { - // issueUpdate.title = platformUpdate.title - // } - // issueData.title = platformUpdate.title - // } - // if (platformUpdate.description != null) { - // // Need to convert to markdown - // issueUpdate.body = await this.provider.getMarkdown(platformUpdate.description ?? '') - // issueData.description = await this.provider.getMarkupSafe( - // container.container, - // issueUpdate.body ?? '', - // this.stripGuestLink - // ) + const issueUpdate: { + title?: string + body?: string + stateReason?: string + assigneeIds?: string[] + } & Record = {} + if (platformUpdate.title != null) { + if (platformUpdate.title !== issueExternal.title) { + issueUpdate.title = platformUpdate.title + } + issueData.title = platformUpdate.title + } + if (platformUpdate.description != null) { + // Need to convert to markdown + issueUpdate.body = await this.provider.getMarkdown(platformUpdate.description ?? '') + issueData.description = await this.provider.getMarkupSafe( + container.container, + issueUpdate.body ?? '', + this.stripGuestLink + ) - // // Of value is same, not need to update. - // if (compareMarkdown(issueUpdate.body, issueExternal.body)) { - // delete issueUpdate.body - // } - // } - // if (platformUpdate.assignee !== undefined) { - // const info = - // platformUpdate.assignee !== null - // ? await this.provider.getGithubLogin(container.container, platformUpdate.assignee) - // : undefined - // // Check external + // Of value is same, not need to update. + if (compareMarkdown(issueUpdate.body, issueExternal.body)) { + delete issueUpdate.body + } + } + if (platformUpdate.assignee !== undefined) { + const info = + platformUpdate.assignee !== null + ? await this.provider.getGithubLogin(container.container, platformUpdate.assignee) + : undefined + // Check external - // const currentAssignees = issueExternal.assignees.nodes.map((it) => it.id) - // currentAssignees.sort((a, b) => a.localeCompare(b)) + const currentAssignees = issueExternal.assignees.nodes.map((it) => it.id) + currentAssignees.sort((a, b) => a.localeCompare(b)) - // issueUpdate.assigneeIds = info !== undefined ? [info.id] : [] - // issueUpdate.assigneeIds.sort((a, b) => a.localeCompare(b)) + issueUpdate.assigneeIds = info !== undefined ? [info.id] : [] + issueUpdate.assigneeIds.sort((a, b) => a.localeCompare(b)) - // if (deepEqual(currentAssignees, issueUpdate.assigneeIds)) { - // // Same ids - // delete issueUpdate.assigneeIds - // } - // issueData.assignee = platformUpdate.assignee - // } + if (deepEqual(currentAssignees, issueUpdate.assigneeIds)) { + // Same ids + delete issueUpdate.assigneeIds + } + issueData.assignee = platformUpdate.assignee + } - // const status = platformUpdate.status ?? issueData.status - // const type = await this.provider.getTaskTypeOf(container.project.type, _class) - // const statuses = await this.provider.getStatuses(type?._id) - // const st = statuses.find((it) => it._id === status) - // if (st !== undefined) { - // // Need to convert to two operations. - // switch (st.category) { - // case task.statusCategory.UnStarted: - // case task.statusCategory.ToDo: - // case task.statusCategory.Active: - // if (issueExternal.state !== 'OPEN') { - // issueUpdate.state = 'OPEN' - // } - // break - // case task.statusCategory.Won: - // if (issueExternal.state !== 'CLOSED' || issueExternal.stateReason !== 'COMPLETED') { - // issueUpdate.state = 'CLOSED' - // issueUpdate.stateReason = 'COMPLETED' - // } - // break - // case task.statusCategory.Lost: - // if (issueExternal.state !== 'CLOSED' || issueExternal.stateReason !== 'NOT_PLANNED') { - // issueUpdate.state = 'CLOSED' - // issueUpdate.stateReason = 'not_planed' // Not supported change to github - // } - // break - // } - // } - // return issueUpdate + const status = platformUpdate.status ?? issueData.status + const type = await this.provider.getTaskTypeOf(container.project.type, _class) + const statuses = await this.provider.getStatuses(type?._id) + const st = statuses.find((it) => it._id === status) + if (st !== undefined) { + // Need to convert to two operations. + switch (st.category) { + case task.statusCategory.UnStarted: + case task.statusCategory.ToDo: + case task.statusCategory.Active: + if (issueExternal.state !== 'OPEN') { + issueUpdate.state = 'OPEN' + } + break + case task.statusCategory.Won: + if (issueExternal.state !== 'CLOSED' || issueExternal.stateReason !== 'COMPLETED') { + issueUpdate.state = 'CLOSED' + issueUpdate.stateReason = 'COMPLETED' + } + break + case task.statusCategory.Lost: + if (issueExternal.state !== 'CLOSED' || issueExternal.stateReason !== 'NOT_PLANNED') { + issueUpdate.state = 'CLOSED' + issueUpdate.stateReason = 'not_planed' // Not supported change to github + } + break + } + } + return issueUpdate } async syncIssues ( @@ -1219,9 +1218,8 @@ export abstract class IssueSyncManagerBase { // No external issue yet, safe delete, since platform document will be deleted a well. return true } - const account = - existing?.createdBy ?? (await this.provider.getAccount(issueExternal.author))?._id ?? core.account.System - const okit = (await this.provider.getOctokit(account as PersonId)) ?? container.container.octokit + const account = existing?.createdBy ?? (await this.provider.getAccount(issueExternal.author)) ?? core.account.System + const okit = (await this.provider.getOctokit(account)) ?? container.container.octokit if (existing !== undefined && issueExternal !== undefined) { let target = await this.getMilestoneIssueTarget( diff --git a/services/github/pod-github/src/sync/issues.ts b/services/github/pod-github/src/sync/issues.ts index bf06e30dea..f99f84184e 100644 --- a/services/github/pod-github/src/sync/issues.ts +++ b/services/github/pod-github/src/sync/issues.ts @@ -52,19 +52,17 @@ import { getSince, gqlp, guessStatus, isGHWriteAllowed, syncRunner } from './uti export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncManager { createPromise: Promise | undefined externalDerivedSync = false - async getAssigneesI (issue: GithubIssue): Promise { - // TODO: FIXME - throw new Error('Not implemented') + async getAssigneesI (issue: GithubIssue): Promise { // Find Assignees and reviewers - // const assignees: PersonAccount[] = [] + const assignees: PersonId[] = [] - // for (const o of issue.assignees) { - // const acc = await this.provider.getAccountU(o) - // if (acc !== undefined) { - // assignees.push(acc) - // } - // } - // return assignees + for (const o of issue.assignees) { + const acc = await this.provider.getAccountU(o) + if (acc !== undefined) { + assignees.push(acc) + } + } + return assignees } async handleEvent( @@ -150,7 +148,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan integration: IntegrationContainer, prj: GithubProject ): Promise { - const account = (await this.provider.getAccountU(event.sender))?._id ?? core.account.System + const account = (await this.provider.getAccountU(event.sender)) ?? core.account.System let externalData: IssueExternalData | undefined if (event.action !== 'deleted') { @@ -243,8 +241,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan case 'assigned': case 'unassigned': { const assignees = await this.getAssigneesI(event.issue) + const persons = await this.getPersonsFromId(assignees) const update: IssueUpdate = { - assignee: assignees?.[0]?.person ?? null + assignee: persons?.[0] ?? null } await this.handleUpdate(externalData as IssueExternalData, derivedClient, update, account, prj, false) break @@ -481,9 +480,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan info: DocSyncInfo ): Promise> { const account = - existing?.modifiedBy ?? (await this.provider.getAccount(issueExternal.author))?._id ?? core.account.System + existing?.modifiedBy ?? (await this.provider.getAccount(issueExternal.author)) ?? core.account.System const accountGH = - info.lastGithubUser ?? (await this.provider.getAccount(issueExternal.author))?._id ?? core.account.System + info.lastGithubUser ?? (await this.provider.getAccount(issueExternal.author)) ?? core.account.System const isProjectProjectTarget = target.target.projectNodeId === target.project.projectNodeId const supportProjects = @@ -492,7 +491,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan // A target node id const targetNodeId: string | undefined = info.targetNodeId as string - const okit = (await this.provider.getOctokit(account as PersonId)) ?? container.container.octokit + const okit = (await this.provider.getOctokit(account)) ?? container.container.octokit const type = await this.provider.getTaskTypeOf(container.project.type, tracker.class.Issue) const statuses = await this.provider.getStatuses(type?._id) @@ -502,7 +501,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan const issueData = { title: issueExternal.title, description: await this.provider.getMarkupSafe(container.container, issueExternal.body, this.stripGuestLink), - assignee: assignees[0]?.person, + assignee: assignees[0], repository: info.repository, remainingTime: 0 } diff --git a/services/github/pod-github/src/sync/pullrequests.ts b/services/github/pod-github/src/sync/pullrequests.ts index 2391429894..b0b15b4c45 100644 --- a/services/github/pod-github/src/sync/pullrequests.ts +++ b/services/github/pod-github/src/sync/pullrequests.ts @@ -2,10 +2,10 @@ import { Analytics } from '@hcengineering/analytics' import { Person } from '@hcengineering/contact' import core, { - PersonId, AttachedData, Doc, DocumentUpdate, + PersonId, Ref, SortingOrder, Status, @@ -41,10 +41,10 @@ import { DocSyncManager, ExternalSyncField, IntegrationContainer, - UserInfo, githubDerivedSyncVersion, githubExternalSyncVersion, - githubSyncVersion + githubSyncVersion, + type UserInfo } from '../types' import { IssueExternalData, @@ -154,7 +154,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS integration: IntegrationContainer, prj: GithubProject ): Promise { - const account = (await this.provider.getAccountU(event.sender))?._id ?? core.account.System + const account = (await this.provider.getAccountU(event.sender)) ?? core.account.System let externalData: PullRequestExternalData try { @@ -243,7 +243,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS case 'unassigned': { const assignees = await this.getAssignees(externalData) const update: GithubPullRequestUpdate = { - assignee: assignees?.[0]?.person ?? null + assignee: assignees?.[0] ?? null } await this.handleUpdate(externalData, derivedClient, update, account, prj, true) break @@ -326,27 +326,27 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS } } - // async getReviewers (issue: PullRequestExternalData): Promise { - // // Find Assignees and reviewers - // const ids: UserInfo[] = issue.reviewRequests.nodes.map((it: any) => it.requestedReviewer) + async getReviewers (issue: PullRequestExternalData): Promise { + // Find Assignees and reviewers + const ids: UserInfo[] = issue.reviewRequests.nodes.map((it: any) => it.requestedReviewer) - // const values: PersonAccount[] = [] + const values: PersonId[] = [] - // for (const o of ids) { - // const acc = await this.provider.getAccount(o) - // if (acc !== undefined) { - // values.push(acc) - // } - // } + for (const o of ids) { + const acc = await this.provider.getAccount(o) + if (acc !== undefined) { + values.push(acc) + } + } - // for (const n of issue.latestReviews.nodes) { - // const acc = await this.provider.getAccount(n.author) - // if (acc !== undefined) { - // values.push(acc) - // } - // } - // return values - // } + for (const n of issue.latestReviews.nodes) { + const acc = await this.provider.getAccount(n.author) + if (acc !== undefined) { + values.push(acc) + } + } + return values + } private async createSyncData ( pullRequestExternal: PullRequestExternalData, @@ -387,14 +387,14 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS info: DocSyncInfo ): Promise> { const account = - existing?.modifiedBy ?? (await this.provider.getAccount(pullRequestExternal.author))?._id ?? core.account.System + existing?.modifiedBy ?? (await this.provider.getAccount(pullRequestExternal.author)) ?? core.account.System const accountGH = - info.lastGithubUser ?? (await this.provider.getAccount(pullRequestExternal.author))?._id ?? core.account.System + info.lastGithubUser ?? (await this.provider.getAccount(pullRequestExternal.author)) ?? core.account.System // A target node id const targetNodeId: string | undefined = info.targetNodeId as string - const okit = (await this.provider.getOctokit(account as PersonId)) ?? container.container.octokit + const okit = (await this.provider.getOctokit(account)) ?? container.container.octokit const isProjectProjectTarget = target.target.projectNodeId === target.project.projectNodeId const supportProjects = @@ -452,13 +452,12 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS } const assignees = await this.getAssignees(pullRequestExternal) - // TODO: FIXME - const reviewers: any = [] // await this.getReviewers(pullRequestExternal) + const reviewers: PersonId[] = await this.getReviewers(pullRequestExternal) const latestReviews: LastReviewState[] = [] for (const d of pullRequestExternal.latestReviews?.nodes ?? []) { - const author = (await this.provider.getAccount(d.author))?._id + const author = await this.provider.getAccount(d.author) if (author !== undefined) { latestReviews.push({ state: toReviewState(d.state), @@ -473,7 +472,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS pullRequestExternal.body, this.stripGuestLink ), - assignee: assignees[0]?.person ?? null, + assignee: assignees[0] ?? null, reviewers: reviewers.map((it: any) => it.person), draft: pullRequestExternal.isDraft, head: pullRequestExternal.headRef, @@ -706,10 +705,10 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS } } - const pendingOrDismissed = new Map, PullRequestReviewState>() + const pendingOrDismissedIds = new Map() - const approvedOrChangesRequested = new Map, PullRequestReviewState>() - const reviewStates = new Map, PullRequestReviewState[]>() + const approvedOrChangesRequested = new Map() + const reviewStates = new Map() const sortedReviews: (Review & { date: number })[] = external.reviews.nodes .filter((it) => it != null) @@ -734,14 +733,18 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS continue } if (r.state === 'PENDING' || r.state === 'DISMISSED') { - pendingOrDismissed.set(rp.person, r.state) + pendingOrDismissedIds.set(rp, r.state) } if (r.state === 'APPROVED' || r.state === 'CHANGES_REQUESTED') { - approvedOrChangesRequested.set(rp.person, r.state) + approvedOrChangesRequested.set(rp, r.state) } - reviewStates.set(rp.person, [...(reviewStates.get(rp.person) ?? []), r.state]) + reviewStates.set(rp, [...(reviewStates.get(rp) ?? []), r.state]) } + const pendingOrDismissed = new Set( + await this.getPersonsFromId(Array.from(pendingOrDismissedIds.entries()).map((it) => it[0])) + ) + for (const r of pullRequest.reviewers ?? []) { // Find all related todos's const todos = [...allTodos, ...removedTodos].filter((it) => it.user === r && it.purpose === 'review') @@ -750,10 +753,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS const hasPending = todos.some((it) => it.doneOn !== null) // Create review Todo, if missing. - if ( - pullRequest.state === GithubPullRequestState.open || - (!hasPending && pendingOrDismissed.get(r) !== undefined) - ) { + if (pullRequest.state === GithubPullRequestState.open || (!hasPending && pendingOrDismissed.has(r))) { if (todos.length === 0) { await this.requestReview(client, pullRequest, external, r, account) } @@ -763,26 +763,28 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS // Handle change requests. // If we have change requests pending, we need to create Todo to resolve them to author or assigned person, to resolve them. - const changeRequestPersons = new Set>() + const changeRequestPersonsIds = new Set() const author = await this.provider.getAccount(external.author) if (author !== undefined) { - changeRequestPersons.add(author.person) + changeRequestPersonsIds.add(author) } for (const au of external.assignees.nodes ?? []) { const u = await this.provider.getAccount(au) if (u !== undefined) { - changeRequestPersons.add(u.person) + changeRequestPersonsIds.add(u) } } // Check review threads and create todo to resolve them. const requestedIds: Ref[] = [] + const changeRequestPersons = await this.getPersonsFromId(Array.from(changeRequestPersonsIds)) + let allResolved = true for (const r of external.reviewThreads.nodes) { if (!r.isResolved) { allResolved = false - for (const c of Array.from(changeRequestPersons)) { + for (const c of changeRequestPersons) { // We need to add Todo to resolve PR. const todos = [...allTodos, ...removedTodos].filter((it) => it.user === c && it.purpose === 'fix') if (todos.length === 0) { @@ -801,7 +803,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS for (const [, sst] of approvedOrChangesRequested.entries()) { if (sst === 'CHANGES_REQUESTED') { // We have changes requested and not resolved yet. - for (const c of Array.from(changeRequestPersons)) { + for (const c of changeRequestPersons) { const todos = [...allTodos, ...removedTodos].filter((it) => it.user === c && it.purpose === 'fix') if (todos.length === 0 && !requestedIds.includes(c)) { requestedIds.push(c) diff --git a/services/github/pod-github/src/sync/repository.ts b/services/github/pod-github/src/sync/repository.ts index 3e7d3d97c8..4b338cee1d 100644 --- a/services/github/pod-github/src/sync/repository.ts +++ b/services/github/pod-github/src/sync/repository.ts @@ -105,7 +105,7 @@ export class RepositorySyncMapper implements DocSyncManager { async handleEvent(integration: IntegrationContainer, derivedClient: TxOperations, evt: T): Promise { const event = evt as RepositoryEvent - const account = (await this.provider.getAccountU(event.sender))?._id ?? core.account.System + const account = (await this.provider.getAccountU(event.sender)) ?? core.account.System switch (event.action) { case 'created': { await this.client.addCollection( diff --git a/services/github/pod-github/src/sync/reviewComments.ts b/services/github/pod-github/src/sync/reviewComments.ts index f907a73676..38250710f6 100644 --- a/services/github/pod-github/src/sync/reviewComments.ts +++ b/services/github/pod-github/src/sync/reviewComments.ts @@ -113,7 +113,7 @@ export class ReviewCommentSyncManager implements DocSyncManager { return true } const account = - existing?.createdBy ?? (await this.provider.getAccountU(commentExternal.user))?._id ?? core.account.System + existing?.createdBy ?? (await this.provider.getAccountU(commentExternal.user)) ?? core.account.System if (commentExternal !== undefined) { try { @@ -177,7 +177,7 @@ export class ReviewCommentSyncManager implements DocSyncManager { repo: GithubIntegrationRepository, integration: IntegrationContainer ): Promise { - const account = (await this.provider.getAccountU(event.sender))?._id ?? core.account.System + const account = (await this.provider.getAccountU(event.sender)) ?? core.account.System let externalData: ReviewCommentExternalData try { @@ -329,7 +329,7 @@ export class ReviewCommentSyncManager implements DocSyncManager { const reviewComment = info.external as ReviewCommentExternalData const account = - existing?.modifiedBy ?? (await this.provider.getAccount(reviewComment.author))?._id ?? core.account.System + existing?.modifiedBy ?? (await this.provider.getAccount(reviewComment.author)) ?? core.account.System if (info.reviewThreadId === undefined && reviewComment.replyTo?.url !== undefined) { const rthread = await derivedClient.findOne(github.class.GithubReviewComment, { diff --git a/services/github/pod-github/src/sync/reviewThreads.ts b/services/github/pod-github/src/sync/reviewThreads.ts index a868a09b87..439595853a 100644 --- a/services/github/pod-github/src/sync/reviewThreads.ts +++ b/services/github/pod-github/src/sync/reviewThreads.ts @@ -132,7 +132,7 @@ export class ReviewThreadSyncManager implements DocSyncManager { return true } const account = - existing?.createdBy ?? (await this.provider.getAccountU(commentExternal.user))?._id ?? core.account.System + existing?.createdBy ?? (await this.provider.getAccountU(commentExternal.user)) ?? core.account.System if (commentExternal !== undefined) { try { @@ -172,7 +172,7 @@ export class ReviewThreadSyncManager implements DocSyncManager { repo: GithubIntegrationRepository, integration: IntegrationContainer ): Promise { - const account = (await this.provider.getAccountU(event.sender))?._id ?? core.account.System + const account = (await this.provider.getAccountU(event.sender)) ?? core.account.System let externalData: ReviewThreadExternalData try { @@ -287,7 +287,7 @@ export class ReviewThreadSyncManager implements DocSyncManager { // Use first comment as author, since github doesn't provide one. const account = existing?.modifiedBy ?? - (await this.provider.getAccount(review.comments.nodes[0].author ?? null))?._id ?? + (await this.provider.getAccount(review.comments.nodes[0].author ?? null)) ?? core.account.System const messageData: ReviewThreadData = { @@ -301,7 +301,7 @@ export class ReviewThreadSyncManager implements DocSyncManager { originalLine: review.originalLine, originalStartLine: review.originalStartLine, path: review.path, - resolvedBy: (await this.provider.getAccount(review.resolvedBy))?._id ?? core.account.System, + resolvedBy: (await this.provider.getAccount(review.resolvedBy)) ?? core.account.System, startDiffSide: review.startDiffSide } if (existing === undefined) { diff --git a/services/github/pod-github/src/sync/reviews.ts b/services/github/pod-github/src/sync/reviews.ts index 17d215d771..31e7a3eea7 100644 --- a/services/github/pod-github/src/sync/reviews.ts +++ b/services/github/pod-github/src/sync/reviews.ts @@ -110,7 +110,7 @@ export class ReviewSyncManager implements DocSyncManager { return true } const account = - existing?.createdBy ?? (await this.provider.getAccountU(commentExternal.user))?._id ?? core.account.System + existing?.createdBy ?? (await this.provider.getAccountU(commentExternal.user)) ?? core.account.System if (commentExternal !== undefined) { try { @@ -164,7 +164,7 @@ export class ReviewSyncManager implements DocSyncManager { repo: GithubIntegrationRepository, integration: IntegrationContainer ): Promise { - const account = (await this.provider.getAccountU(event.sender))?._id ?? core.account.System + const account = (await this.provider.getAccountU(event.sender)) ?? core.account.System let externalData: ReviewExternalData try { @@ -302,7 +302,7 @@ export class ReviewSyncManager implements DocSyncManager { } const review = info.external as ReviewExternalData - const account = existing?.modifiedBy ?? (await this.provider.getAccount(review.author))?._id ?? core.account.System + const account = existing?.modifiedBy ?? (await this.provider.getAccount(review.author)) ?? core.account.System const messageData: ReviewData = { body: await this.provider.getMarkupSafe(container.container, review.body), diff --git a/services/github/pod-github/src/types.ts b/services/github/pod-github/src/types.ts index b863d32dc0..eb2326132d 100644 --- a/services/github/pod-github/src/types.ts +++ b/services/github/pod-github/src/types.ts @@ -1,11 +1,11 @@ import { Person } from '@hcengineering/contact' import { - PersonId, Branding, Class, Data, Doc, DocumentUpdate, + PersonId, Ref, Space, Status, @@ -14,10 +14,6 @@ import { WorkspaceUuid, type Blob } from '@hcengineering/core' -import { LiveQuery } from '@hcengineering/query' -import { ProjectType, TaskType } from '@hcengineering/task' -import { MarkupNode } from '@hcengineering/text' -import { User } from '@octokit/webhooks-types' import { DocSyncInfo, GithubIntegration, @@ -26,6 +22,10 @@ import { GithubProject, GithubUserInfo } from '@hcengineering/github' +import { LiveQuery } from '@hcengineering/query' +import { ProjectType, TaskType } from '@hcengineering/task' +import { MarkupNode } from '@hcengineering/text' +import { User } from '@octokit/webhooks-types' import { Octokit } from 'octokit' import { GithubProjectV2 } from './sync/githubTypes' @@ -83,9 +83,8 @@ export interface ContainerFocus { export interface IntegrationManager { liveQuery: LiveQuery getContainer: (space: Ref) => Promise - // TODO: FIXME - getAccount: (user?: UserInfo | null) => Promise - getAccountU: (user: User) => Promise + getAccount: (user?: UserInfo | null) => Promise + getAccountU: (user: User) => Promise getOctokit: (account: PersonId) => Promise getMarkupSafe: ( container: IntegrationContainer, @@ -194,7 +193,7 @@ export interface DocSyncManager { */ export interface GithubIntegrationRecord { installationId: number - workspace: string + workspace: WorkspaceUuid accountId: PersonId } @@ -202,6 +201,7 @@ export interface GithubIntegrationRecord { * @public */ export interface GithubUserRecord { + account: PersonId _id: string // login code?: string | null token?: string @@ -212,6 +212,5 @@ export interface GithubUserRecord { state?: string scope?: string error?: string | null - - accounts: Record + accounts: Record } diff --git a/services/github/pod-github/src/users.ts b/services/github/pod-github/src/users.ts index 13f9f02653..0207f15d7b 100644 --- a/services/github/pod-github/src/users.ts +++ b/services/github/pod-github/src/users.ts @@ -1,19 +1,18 @@ -import type { PersonId } from '@hcengineering/core' -import type { Collection, FindCursor } from 'mongodb' +import type { AccountClient, IntegrationSecret } from '@hcengineering/account-client' +import { systemAccountUuid, type PersonId, type WorkspaceUuid } from '@hcengineering/core' +import { getAccountClient } from '@hcengineering/server-client' +import { generateToken } from '@hcengineering/server-token' import type { GithubUserRecord } from './types' export class UserManager { userCache = new Map() refUserCache = new Map() - constructor (readonly usersCollection: Collection) {} + accountClient: AccountClient - public async getUsers (workspace: string): Promise { - return await this.usersCollection - .find({ - [`accounts.${workspace}`]: { $exists: true } - }) - .toArray() + constructor () { + const sysToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'github' }) + this.accountClient = getAccountClient(sysToken, 30000) } async getAccount (login: string): Promise { @@ -21,7 +20,11 @@ export class UserManager { if (res !== undefined) { return res } - res = (await this.usersCollection.findOne({ _id: login })) ?? undefined + const secrets = await this.accountClient.listIntegrationsSecrets({ kind: 'github-user', key: login }) + if (secrets.length === 0) { + return + } + res = this.secretToUserRecord(secrets[0], login) if (res !== undefined) { if (this.userCache.size > 1000) { this.userCache.clear() @@ -31,13 +34,28 @@ export class UserManager { return res } - async getAccountByRef (workspace: string, ref: PersonId): Promise { + private secretToUserRecord (secret: IntegrationSecret, login: string): GithubUserRecord | undefined { + return { + ...(JSON.parse(secret.secret) ?? {}), // TODO: Add security + account: secret.socialId, + _id: login, + accounts: {} + } + } + + async getAccountByRef (workspace: WorkspaceUuid, ref: PersonId): Promise { const key = `${workspace}.${ref}` let rec = this.refUserCache.get(key) if (rec !== undefined) { return rec } - rec = (await this.usersCollection.findOne({ [`accounts.${workspace}`]: ref })) ?? undefined + + const secrets = await this.accountClient.listIntegrationsSecrets({ kind: 'github-user', socialId: ref }) + if (secrets.length === 0) { + return + } + + rec = this.secretToUserRecord(secrets[0], secrets[0].key) if (rec !== undefined) { if (this.refUserCache.size > 1000) { this.refUserCache.clear() @@ -47,23 +65,67 @@ export class UserManager { return rec } - async updateUser (dta: GithubUserRecord): Promise { - this.userCache.clear() - this.refUserCache.clear() - await this.usersCollection.updateOne({ _id: dta._id }, { $set: dta } as any) + async updateUser (dta: GithubUserRecord, clear: boolean = true): Promise { + if (clear) { + this.userCache.clear() + this.refUserCache.clear() + } + + // Need to check if user integeration exists. + const existing = await this.accountClient.getIntegration({ + kind: 'github-user', + workspaceUuid: null, + socialId: dta.account + }) + + if (existing == null) { + await this.accountClient.createIntegration({ + kind: 'github-user', + workspaceUuid: null, + socialId: dta.account, + data: { + login: dta._id + } + }) + } + + const exists = await this.accountClient.getIntegrationSecret({ + key: dta._id, + kind: 'github-user', + socialId: dta.account, + workspaceUuid: null + }) + + if (exists !== null) { + await this.accountClient.updateIntegrationSecret({ + key: dta._id, + kind: 'github-user', + socialId: dta.account, + secret: JSON.stringify(dta), + workspaceUuid: null + }) + } else { + await this.accountClient.addIntegrationSecret({ + key: dta._id, + kind: 'github-user', + socialId: dta.account, + secret: JSON.stringify(dta), + workspaceUuid: null + }) + } } async insertUser (dta: GithubUserRecord): Promise { - await this.usersCollection.insertOne(dta) + await this.updateUser(dta, false) } async removeUser (login: string): Promise { this.userCache.clear() this.refUserCache.clear() - await this.usersCollection.deleteOne({ _id: login }) - } - getAllUsers (): FindCursor { - return this.usersCollection.find({}) + const secerts = await this.accountClient.listIntegrationsSecrets({ kind: 'github-user', key: login }) + for (const s of secerts) { + await this.accountClient.deleteIntegrationSecret(s) + } } } diff --git a/services/github/pod-github/src/worker.ts b/services/github/pod-github/src/worker.ts index b31627d491..36f54b613e 100644 --- a/services/github/pod-github/src/worker.ts +++ b/services/github/pod-github/src/worker.ts @@ -1,23 +1,29 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ +import type { AccountClient } from '@hcengineering/account-client' import { Analytics } from '@hcengineering/analytics' import chunter from '@hcengineering/chunter' import { CollaboratorClient } from '@hcengineering/collaborator-client' -import contact, { AvatarType, Person } from '@hcengineering/contact' +import contact, { + AvatarType, + Person, + type Employee, + type SocialIdentity, + type SocialIdentityRef +} from '@hcengineering/contact' import core, { - PersonId, - AccountRole, AttachedDoc, Branding, Class, Client, ClientConnectEvent, - Data, Doc, DocumentQuery, DocumentUpdate, FindResult, MeasureContext, + PersonId, Ref, + SocialIdType, SortingOrder, Space, Status, @@ -30,16 +36,19 @@ import core, { WithLookup, WorkspaceEvent, WorkspaceUuid, + buildSocialIdString, concatLink, generateId, groupByArray, reduceCalls, + systemAccountUuid, toIdMap, type Blob, + type Data, type MigrationState, + type PersonUuid, type TimeRateLimiter, - type WorkspaceIds, - type WorkspaceDataId + type WorkspaceIds } from '@hcengineering/core' import github, { DocSyncInfo, @@ -48,12 +57,14 @@ import github, { GithubIntegrationRepository, GithubIssue, GithubProject, - GithubUserInfo, - githubId + githubId, + type GithubUserInfo } from '@hcengineering/github' import { LiveQuery } from '@hcengineering/query' +import { getAccountClient } from '@hcengineering/server-client' import { StorageAdapter } from '@hcengineering/server-core' import { getPublicLinkUrl } from '@hcengineering/server-guest-resources' +import { generateToken } from '@hcengineering/server-token' import task, { ProjectType, TaskType } from '@hcengineering/task' import { MarkupNode, MarkupNodeType, jsonToMarkup } from '@hcengineering/text' import { isMarkdownsEquals } from '@hcengineering/text-markdown' @@ -91,9 +102,6 @@ import { } from './types' import { equalExceptKeys } from './utils' -// TODO: FIXME -type PersonAccount = any - /** * @public */ @@ -105,8 +113,7 @@ export class GithubWorker implements IntegrationManager { triggerRequests: number = 0 - // TODO: FIXME - authRequestSend = new Set() + authRequestSend = new Set() triggerSync: () => void = () => { this.triggerRequests++ @@ -271,7 +278,7 @@ export class GithubWorker implements IntegrationManager { } } - async getAccountU (user?: User): Promise { + async getAccountU (user?: User): Promise { if (user == null) { return undefined } @@ -284,98 +291,96 @@ export class GithubWorker implements IntegrationManager { }) } - accountMap = new Map>() - async getAccount (userInfo?: UserInfo | null): Promise { + accountMap = new Map>() + async getAccount (userInfo?: UserInfo | null): Promise { if (userInfo?.login == null) { return } const info = this.accountMap.get(userInfo?.login ?? '') if (info !== undefined) { - return await info + if (info instanceof Promise) { + const p = await info + this.accountMap.set(userInfo?.login, p) + return p + } + return info } const p = this._getAccountRaw(userInfo) this.accountMap.set(userInfo?.login ?? '', p) return await p } - async _getAccountRaw (userInfo?: UserInfo | null): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // // We need to sync by userInfo id to prevent parallel requests. - // if (userInfo === null) { - // // Ghost author. - // return await this.getAccount({ - // id: 'ghost', - // login: 'ghost', - // avatarUrl: 'https://avatars.githubusercontent.com/u/10137?v=4', - // email: '', - // name: 'Ghost' - // }) - // } - // if (userInfo?.login == null) { - // return - // } - // const userName = (userInfo.name ?? userInfo.login) - // .split(' ') - // .map((it) => it.trim()) - // .reverse() - // .join(',') // TODO: Convert first, last name + async _getAccountRaw (userInfo?: UserInfo | null): Promise { + // We need to sync by userInfo id to prevent parallel requests. + if (userInfo === null) { + // Ghost author. + return await this.getAccount({ + id: 'ghost', + login: 'ghost', + avatarUrl: 'https://avatars.githubusercontent.com/u/10137?v=4', + email: '', + name: 'Ghost' + }) + } + if (userInfo?.login == null) { + return + } + const userName = (userInfo.name ?? userInfo.login) + .split(' ') + .map((it) => it.trim()) + .reverse() + .join(',') // TODO: Convert first, last name - // const infos = await this.liveQuery.findOne(github.class.GithubUserInfo, { login: userInfo.login }) - // if (infos === undefined) { - // await this._client.createDoc(github.class.GithubUserInfo, contact.space.Contacts, { - // ...userInfo - // }) - // } + const infos = await this.liveQuery.findOne(github.class.GithubUserInfo, { login: userInfo.login }) + if (infos === undefined) { + await this._client.createDoc(github.class.GithubUserInfo, contact.space.Contacts, { + ...userInfo + }) + } - // const account = await this.client - // .getModel() - // .findOne(contact.class.PersonAccount, { email: `github:${userInfo.login}` }) - // if (account !== undefined) { - // const person = await this.liveQuery.findOne(contact.class.Person, { _id: account.person }) - // // We need to be sure employee are exists. - // if (person === undefined) { - // const person: Ref = await this.findPerson(userInfo, userName) - // if (account.person !== person) { - // await this._client.update(account, { person }) - // } - // } - // return account - // } else { - // // Check authorized users - // const accountRecord = await this.platform.getAccount(userInfo.login) - // if (accountRecord !== undefined) { - // const authorizedId = accountRecord.accounts[this.workspace.name] - // if (authorizedId !== undefined) { - // const emp = await this._client.findOne(contact.class.PersonAccount, { - // _id: authorizedId as Ref - // }) - // if (emp !== undefined) { - // // We need to create github account - // const gid = await this._client.createDoc(contact.class.PersonAccount, core.space.Model, { - // email: `github:${userInfo.login}`, - // person: emp.person, - // role: AccountRole.User - // }) - // const acc = await this._client.findOne(contact.class.PersonAccount, { _id: gid }) - // return acc - // } - // } - // } + // Find a local social id already existing + const existingSocialId = await this._client.findOne(contact.class.SocialIdentity, { + type: SocialIdType.GITHUB, + value: userInfo.login + }) - // const person: Ref | undefined = await this.findPerson(userInfo, userName) + if (existingSocialId !== undefined) { + return existingSocialId?._id + } - // // We need to create email account - // const id = await this._client.createDoc(contact.class.PersonAccount, core.space.Model, { - // email: `github:${userInfo.login}`, - // person, - // role: AccountRole.User - // }) - // const acc = await this.client.getModel().findOne(contact.class.PersonAccount, { _id: id }) - // return acc - // } + const { uuid, socialId } = await this.accountClient.ensurePerson( + SocialIdType.GITHUB, + userInfo.login, + userInfo.name ?? userInfo.login, + '' + ) + const person: Ref | undefined = await this.findPerson(userInfo, userName, uuid) + // We need to find or create a local person for uuid if missing. + + // We need to create social id github account + await this._client.addCollection( + contact.class.SocialIdentity, + contact.space.Contacts, + person, + contact.class.Person, + 'socialIds', + { + type: SocialIdType.GITHUB, + value: userInfo.login, + key: buildSocialIdString({ + type: SocialIdType.GITHUB, + value: userInfo.login + }), + verifiedOn: Date.now() + }, + socialId as SocialIdentityRef + ) + + return socialId } + accountClient: AccountClient + private constructor ( readonly ctx: MeasureContext, readonly limiter: TimeRateLimiter, @@ -388,6 +393,9 @@ export class GithubWorker implements IntegrationManager { readonly branding: Branding | null, readonly periodicSyncInterval = 60 * 60 * 1000 ) { + const token = generateToken(systemAccountUuid, this.workspace.uuid, { service: 'github', mode: 'github' }) + this.accountClient = getAccountClient(token, 30000) + this._client = new TxOperations(this.client, core.account.System) this.liveQuery = new LiveQuery(client) @@ -420,7 +428,6 @@ export class GithubWorker implements IntegrationManager { _class: [chunter.class.ChatMessage], mapper: new CommentSyncManager(this.ctx.newChild('comment', {}), this._client, this.liveQuery) }, - // TODO: FIXME // { // _class: [contact.class.PersonAccount], // mapper: this.personMapper @@ -458,221 +465,235 @@ export class GithubWorker implements IntegrationManager { this.periodicSyncPromise = undefined } - // private async findPerson (userInfo: UserInfo, userName: string): Promise> { - // let person: Ref | undefined - // // try to find by account. - // if (userInfo.email != null && userInfo.email.trim().length > 0) { - // const personAccount = await this.client.getModel().findOne(contact.class.PersonAccount, { email: userInfo.email }) - // person = personAccount?.person - // } + private async findPerson (userInfo: UserInfo, userName: string, uuid: PersonUuid): Promise> { + let person: Ref | undefined + // try to find by account. + if (userInfo.email != null && userInfo.email.trim().length > 0) { + const personAccount = await this.client.findOne(contact.class.SocialIdentity, { + type: SocialIdType.EMAIL, + value: userInfo.email + }) + person = personAccount?.attachedTo + } - // if (person === undefined) { - // const channel = await this.liveQuery.findOne(contact.class.Channel, { - // provider: contact.channelProvider.GitHub, - // value: userInfo.login - // }) - // person = channel?.attachedTo as Ref - // } - - // if (person === undefined) { - // // We need to create some person to identify this account. - // person = await this._client.createDoc(contact.class.Person, contact.space.Contacts, { - // name: userName, - // avatarType: AvatarType.EXTERNAL, - // avatarProps: { url: userInfo.avatarUrl }, - // city: '', - // comments: 0, - // channels: 0, - // attachments: 0 - // }) - // await this._client.addCollection( - // contact.class.Channel, - // contact.space.Contacts, - // person, - // contact.class.Person, - // 'channels', - // { - // provider: contact.channelProvider.GitHub, - // value: userInfo.login - // } - // ) - // if (userInfo.email != null && userInfo.email.trim() !== '') { - // await this._client.addCollection( - // contact.class.Channel, - // contact.space.Contacts, - // person, - // contact.class.Person, - // 'channels', - // { - // provider: contact.channelProvider.Email, - // value: userInfo.email - // } - // ) - // } - // } - // return person - // } - - async getGithubLogin (container: IntegrationContainer, person: Ref): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // const accounts = this.client.getModel().findAllSync(contact.class.PersonAccount, {}) - // const acc = accounts.find((it) => it.person === person && it.email.startsWith('github:')) - // if (acc === undefined) { - // return // Nobody, will use system account. - // } - // const login = acc.email.substring(7) - // let info = await this.liveQuery.findOne(github.class.GithubUserInfo, { login }) - // if (info === undefined) { - // // We need to retrieve info for login - // const response: any = await container.octokit?.graphql( - // `query($login: String!) { - // user(login: $login) { - // id - // email - // login - // name - // avatarUrl - // } - // }`, - // { - // login - // } - // ) - // } - // info = response.user - // await this._client.createDoc(github.class.GithubUserInfo, contact.space.Contacts, info as Data) - // } - // return info + if (person === undefined) { + // We need to create some person to identify this account. + person = await this._client.createDoc(contact.class.Person, contact.space.Contacts, { + name: userName, + avatarType: AvatarType.EXTERNAL, + avatarProps: { url: userInfo.avatarUrl }, + city: '', + comments: 0, + channels: 0, + attachments: 0, + personUuid: uuid + }) + await this._client.addCollection( + contact.class.Channel, + contact.space.Contacts, + person, + contact.class.Person, + 'channels', + { + provider: contact.channelProvider.GitHub, + value: userInfo.login + } + ) + if (userInfo.email != null && userInfo.email.trim() !== '') { + await this._client.addCollection( + contact.class.Channel, + contact.space.Contacts, + person, + contact.class.Person, + 'channels', + { + provider: contact.channelProvider.Email, + value: userInfo.email + } + ) + } + } + return person } - async syncUserData (ctx: MeasureContext, users: GithubUserRecord[]): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // Let's sync information about users and send some details - // const accounts = await this._client.findAll(contact.class.PersonAccount, { - // email: { $in: users.map((it) => `github:${it._id}`) } - // }) - // const userAuths = await this._client.findAll(github.class.GithubAuthentication, {}) - // const persons = await this._client.findAll(contact.class.Person, { _id: { $in: accounts.map((it) => it.person) } }) - // for (const record of users) { - // if (record.error !== undefined) { - // // Skip accounts with error - // continue - // } - // const account = accounts.find((it) => it.email === `github:${record._id}`) - // const userAuth = userAuths.find((it) => it.login === record._id) - // const person = persons.find((it) => account?.person) - // if (account === undefined || userAuth === undefined || person === undefined) { - // continue - // } - // const accountRef = record.accounts[this.workspace.name] - // try { - // await this.platform.checkRefreshToken(record, true) + async getGithubLogin (container: IntegrationContainer, person: Ref): Promise { + const personRef = await this.client.findOne(contact.class.Person, { _id: person }) + if (personRef === undefined) { + return + } + const accounts = await this.client.findAll(contact.class.SocialIdentity, { + type: SocialIdType.GITHUB, + attachedTo: personRef._id + }) + if (accounts.length === 0) { + return // Nobody, will use system account. + } + const info = await this.client.findOne(github.class.GithubUserInfo, { + login: { $in: accounts.map((it) => it.value) } + }) + if (info?.id === undefined) { + // We need to retrieve info for login + const response: any = await container.octokit?.graphql( + `query($login: String!) { + user(login: $login) { + id + email + login + name + avatarUrl + } + }`, + { + login: accounts[0].value + } + ) + const infoData = response.user + if (info == null) { + await this._client.createDoc( + github.class.GithubUserInfo, + contact.space.Contacts, + infoData as Data + ) + } else { + await this._client.diffUpdate(info, { + ...infoData + }) + } + } + return info + } - // const ops = new TxOperations(this.client, accountRef) - // await syncUser(ctx, record, userAuth, ops, accountRef) - // } catch (err: any) { - // try { - // await this.platform.revokeUserAuth(record) - // } catch (err: any) { - // ctx.error(`Failed to revoke user ${record._id}`, err) - // } - // if (err.response?.data?.message !== 'Bad credentials') { - // ctx.error(`Failed to sync user ${record._id}`, err) - // Analytics.handleError(err) - // } - // if (userAuth !== undefined) { - // await this._client.update( - // userAuth, - // { - // error: errorToObj(err) - // }, - // undefined, - // Date.now(), - // accountRef - // ) - // } - // } - // } + async syncUserData (ctx: MeasureContext): Promise { + // Let's sync information about users and send some details + const accounts = await this._client.findAll(contact.class.SocialIdentity, { + type: SocialIdType.GITHUB + }) + const userAuths = await this._client.findAll(github.class.GithubAuthentication, {}) + const persons = await this._client.findAll(contact.class.Person, { + _id: { $in: accounts.map((it) => it.attachedTo) } + }) + for (const account of accounts) { + const userAuth = userAuths.find((it) => it.login === account.value) + const person = persons.find((it) => account?.attachedTo) + if (account === undefined || userAuth === undefined || person === undefined) { + continue + } + const record = await this.platform.getUser(account.value) + if (record === undefined) { + continue + } + try { + await this.platform.checkRefreshToken(record, true) + + const ops = new TxOperations(this.client, account._id) + await syncUser(ctx, record, userAuth, ops, account._id) + } catch (err: any) { + try { + await this.platform.revokeUserAuth(record) + } catch (err: any) { + ctx.error(`Failed to revoke user ${record._id}`, err) + } + if (err.response?.data?.message !== 'Bad credentials') { + ctx.error(`Failed to sync user ${record._id}`, err) + Analytics.handleError(err) + } + if (userAuth !== undefined) { + await this._client.update( + userAuth, + { + error: errorToObj(err) + }, + undefined, + Date.now(), + account._id + ) + } + } + } } async getOctokit (account: PersonId): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // let record = await this.platform.getAccountByRef(this.workspace.name, account) + let record = await this.platform.getAccountByRef(this.workspace.uuid, account) - // const accountRef = this.accounts.find((it) => it._id === account) - // const [accountRef] = this.client.getModel().findAllSync(contact.class.PersonAccount, { _id: account }) - // if (record === undefined) { - // if (accountRef !== undefined) { - // const accounts = this._client.getModel().getAccountByPersonId(accountRef.person) - // for (const aa of accounts) { - // record = await this.platform.getAccountByRef(this.workspace.name, aa._id) - // if (record !== undefined) { - // break - // } - // } - // } - // } - // // Check and refresh token if required. - // if (record !== undefined) { - // this.ctx.info('get octokit', { account, recordId: record._id, workspace: this.workspace.name }) - // await this.platform.checkRefreshToken(record) - // return new Octokit({ - // auth: record.token, - // client_id: config.ClientID, - // client_secret: config.ClientSecret - // }) - // } + const accountRef = await this._client.findOne(contact.class.SocialIdentity, { _id: account as any }) + if (record === undefined) { + if (accountRef !== undefined) { + const accounts = await this._client.findAll(contact.class.SocialIdentity, { attachedTo: accountRef.attachedTo }) + for (const aa of accounts) { + record = await this.platform.getAccountByRef(this.workspace.uuid, aa._id) + if (record !== undefined) { + break + } + } + } + } + // Check and refresh token if required. + if (record !== undefined) { + this.ctx.info('get octokit', { account, recordId: record._id, workspace: this.workspace.uuid }) + await this.platform.checkRefreshToken(record) + return new Octokit({ + auth: record.token, + client_id: config.ClientID, + client_secret: config.ClientSecret + }) + } - // // We need to inform user, he need to authorize this account with github. - // if (accountRef !== undefined && !this.authRequestSend.has(accountRef._id)) { - // this.authRequestSend.add(accountRef._id) - // const person = await this.liveQuery.findOne(contact.class.Person, { _id: accountRef.person }) - // if (person !== undefined) { - // const personSpace = await this.liveQuery.findOne(contact.class.PersonSpace, { person: person._id }) - // if (personSpace !== undefined) { - // // We need to remove if user has authentication in workspace but doesn't have a record. + // We need to inform user, he need to authorize this account with github. + // TODO: Inform user it need authenticsion + if (!this.authRequestSend.has(account)) { + this.authRequestSend.add(account) + const socialId = await this._client.findOne(contact.class.SocialIdentity, { _id: account as any }) + if (socialId !== undefined) { + const personSpace = await this.liveQuery.findOne(contact.class.PersonSpace, { person: socialId.attachedTo }) + const person = await this._client.findOne(contact.mixin.Employee, { _id: socialId.attachedTo as Ref }) + if (personSpace !== undefined && person !== undefined) { + // We need to remove if user has authentication in workspace but doesn't have a record. - // const accounts = this._client.getModel().getAccountByPersonId(accountRef.person) - // const authentications = await this.liveQuery.findAll(github.class.GithubAuthentication, { - // createdBy: { $in: accounts.map((it) => it._id) } - // }) - // for (const auth of authentications) { - // await this._client.remove(auth) - // } + const allSocialId = await this._client.findAll(contact.class.SocialIdentity, { + attachedTo: personSpace.person + }) - // await createNotification(this._client, person, { - // user: account, - // space: personSpace._id, - // message: github.string.AuthenticatedWithGithubRequired, - // props: {} - // }) - // } - // } - // } - // this.ctx.info('get octokit: return bot', { account, workspace: this.workspace.name }) + const authentications = await this.liveQuery.findAll(github.class.GithubAuthentication, { + createdBy: { $in: allSocialId.map((it) => it._id) } + }) + for (const auth of authentications) { + await this._client.remove(auth) + } + + if (person.personUuid !== undefined) { + await createNotification(this._client, person, { + user: person.personUuid, + space: personSpace._id, + message: github.string.AuthenticatedWithGithubRequired, + props: {} + }) + } + } + } + } + this.ctx.info('get octokit: return bot', { account, workspace: this.workspace.uuid }) } async isPlatformUser (account: PersonId): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // let record = await this.platform.getAccountByRef(this.workspace.name, account) - // const accountRef = await this.liveQuery.findOne(contact.class.PersonAccount, { _id: account }) - // if (record === undefined) { - // if (accountRef !== undefined) { - // const accounts = this._client.getModel().getAccountByPersonId(accountRef.person) - // for (const aa of accounts) { - // record = await this.platform.getAccountByRef(this.workspace.name, aa._id) - // if (record !== undefined) { - // break - // } - // } - // } - // } - // // Check and refresh token if required. - // return record !== undefined && accountRef !== undefined + let record = await this.platform.getAccountByRef(this.workspace.uuid, account) + let accountRef: Employee | undefined + if (record === undefined) { + const socialId = await this._client.findOne(contact.class.SocialIdentity, { _id: account as any }) + if (socialId !== undefined) { + accountRef = await this._client.findOne(contact.mixin.Employee, { _id: socialId?.attachedTo as Ref }) + if (accountRef !== undefined) { + const socialIds = await this._client.findAll(contact.class.SocialIdentity, { attachedTo: accountRef._id }) + for (const aa of socialIds) { + record = await this.platform.getAccountByRef(this.workspace.uuid, aa._id) + if (record !== undefined) { + break + } + } + } + } + } + // Check and refresh token if required. + return record !== undefined && accountRef !== undefined } async uploadFile (patch: string, file?: string, contentType?: string): Promise { @@ -787,10 +808,8 @@ export class GithubWorker implements IntegrationManager { this.triggerRequests = 1 this.updateRequests = 1 this.syncPromise = this.syncAndWait() - - const userRecords = await this.platform.getUsers(this.workspace.uuid) try { - await this.syncUserData(this.ctx, userRecords) + await this.syncUserData(this.ctx) } catch (err: any) { Analytics.handleError(err) } @@ -909,59 +928,55 @@ export class GithubWorker implements IntegrationManager { } private async queryAccounts (): Promise { - // TODO: FIXME - throw new Error('Not implemented') - // const updateAccounts = async (accounts: PersonAccount[]): Promise => { - // const persons = await this.liveQuery.findAll(contact.class.Person, { - // _id: { $in: accounts.map((it) => it.person) } - // }) - // const h = this.client.getHierarchy() - // for (const a of accounts) { - // if (a.email.startsWith('github:')) { - // const login = a.email.substring(7) - // const person = persons.find((it) => it._id === a.person) - // if (person !== undefined) { - // // #1 check if person has GithubUser mixin. - // if (!h.hasMixin(person, github.mixin.GithubUser)) { - // await this._client.createMixin(person._id, person._class, person.space, github.mixin.GithubUser, { - // url: `https://github.com/${login}` - // }) - // } else { - // const ghu = h.as(person, github.mixin.GithubUser) - // if (ghu.url !== `https://github.com/${login}`) { - // await this._client.updateMixin(person._id, person._class, person.space, github.mixin.GithubUser, { - // url: `https://github.com/${login}` - // }) - // } - // } - // // #2 check if person has contact github and if not add it. - // const channel = await this._client.findOne(contact.class.Channel, { - // provider: contact.channelProvider.GitHub, - // value: login, - // attachedTo: person._id - // }) - // if (channel === undefined) { - // await this._client.addCollection( - // contact.class.Channel, - // person.space, - // person._id, - // contact.class.Person, - // 'channels', - // { - // provider: contact.channelProvider.GitHub, - // value: login - // } - // ) - // } - // } - // } - // } - // } - // await new Promise((resolve, reject) => { - // this.liveQuery.query(contact.class.PersonAccount, {}, (res) => { - // void updateAccounts(res).then(resolve).catch(reject) - // }) - // }) + const updateAccounts = async (accounts: SocialIdentity[]): Promise => { + const persons = await this.liveQuery.findAll(contact.class.Person, { + _id: { $in: accounts.map((it) => it.attachedTo) } + }) + const h = this.client.getHierarchy() + for (const a of accounts) { + const login = a.value + const person = persons.find((it) => it._id === a.attachedTo) + if (person !== undefined) { + // #1 check if person has GithubUser mixin. + if (!h.hasMixin(person, github.mixin.GithubUser)) { + await this._client.createMixin(person._id, person._class, person.space, github.mixin.GithubUser, { + url: `https://github.com/${login}` + }) + } else { + const ghu = h.as(person, github.mixin.GithubUser) + if (ghu.url !== `https://github.com/${login}`) { + await this._client.updateMixin(person._id, person._class, person.space, github.mixin.GithubUser, { + url: `https://github.com/${login}` + }) + } + } + // #2 check if person has contact github and if not add it. + const channel = await this._client.findOne(contact.class.Channel, { + provider: contact.channelProvider.GitHub, + value: login, + attachedTo: person._id + }) + if (channel === undefined) { + await this._client.addCollection( + contact.class.Channel, + person.space, + person._id, + contact.class.Person, + 'channels', + { + provider: contact.channelProvider.GitHub, + value: login + } + ) + } + } + } + } + await new Promise((resolve, reject) => { + this.liveQuery.query(contact.class.SocialIdentity, { type: SocialIdType.GITHUB }, (res) => { + void updateAccounts(res).then(resolve).catch(reject) + }) + }) } async performExternalSync ( @@ -1642,7 +1657,7 @@ export class GithubWorker implements IntegrationManager { branding: Branding | null, app: App, storageAdapter: StorageAdapter, - reconnect: (workspaceId: string, event: ClientConnectEvent) => void + reconnect: (workspaceId: WorkspaceUuid, event: ClientConnectEvent) => void ): Promise { ctx.info('Connecting to', { workspace }) let client: Client | undefined @@ -1729,7 +1744,8 @@ export async function syncUser ( repositoryDiscussions: details.viewer.repositoryDiscussions.totalCount, organizations: details.viewer.organizations, nodeId: details.viewer.id, - ...dta + ...dta, + error: null }, undefined, account diff --git a/services/github/server-github-resources/src/index.ts b/services/github/server-github-resources/src/index.ts index bbe39fc7da..8e787a862e 100644 --- a/services/github/server-github-resources/src/index.ts +++ b/services/github/server-github-resources/src/index.ts @@ -14,6 +14,7 @@ import core, { TxCUD, TxProcessor, TxUpdateDoc, + systemAccount, systemAccountUuid, type Class, type TxMixin @@ -176,50 +177,45 @@ async function updateDocSyncInfo ( cache: Map, toApply: Tx[] ): Promise { - // TODO: FIXME - // throw new Error('Not implemented') - // const checkTx = (tx: Tx): boolean => - // control.hierarchy.isDerived(tx._class, core.class.TxCUD) && - // (tx as TxCUD).objectClass === github.class.DocSyncInfo && - // (tx as TxCUD).objectId === cud.objectId - // const txes = [...control.txes, ...control.ctx.contextData.broadcast.txes, ...toApply] - // // Check already captured Txes - // for (const i of txes) { - // if (checkTx(i)) { - // // We have sync doc create request already. - // return - // } - // } - // const [account] = control.modelDb.findAllSync(contact.class.PersonAccount, { - // _id: tx.modifiedBy as PersonId - // }) - // // Do not modify state if is modified by github service. - // if (account === undefined) { - // return - // } - // const projects = - // (cache.get('projects') as GithubProject[]) ?? - // (await control.queryFind(control.ctx, github.mixin.GithubProject, {}, { projection: { _id: 1 } })) - // cache.set('projects', projects) - // if (projects.some((it) => it._id === (space as Ref))) { - // const sdoc = - // (cache.get(cud.objectId) as DocSyncInfo) ?? - // ( - // await control.findAll(control.ctx, github.class.DocSyncInfo, { - // _id: cud.objectId as Ref - // }) - // ).shift() - // // We need to check if sync doc is already exists. - // if (sdoc === undefined) { - // // Created by non github integration - // // We need to create the doc sync info - // createSyncDoc(control, cud, tx, space, toApply) - // } else { - // cache.set(cud.objectId, sdoc) - // // We need to create the doc sync info - // updateSyncDoc(control, cud, space, sdoc, toApply) - // } - // } + const checkTx = (tx: Tx): boolean => + control.hierarchy.isDerived(tx._class, core.class.TxCUD) && + (tx as TxCUD).objectClass === github.class.DocSyncInfo && + (tx as TxCUD).objectId === cud.objectId + const txes = [...control.txes, ...control.ctx.contextData.broadcast.txes, ...toApply] + // Check already captured Txes + for (const i of txes) { + if (checkTx(i)) { + // We have sync doc create request already. + return + } + } + // Do not modify state if is modified by github service. + if (tx.modifiedBy === systemAccount.primarySocialId) { + return + } + const projects = + (cache.get('projects') as GithubProject[]) ?? + (await control.queryFind(control.ctx, github.mixin.GithubProject, {}, { projection: { _id: 1 } })) + cache.set('projects', projects) + if (projects.some((it) => it._id === (space as Ref))) { + const sdoc = + (cache.get(cud.objectId) as DocSyncInfo) ?? + ( + await control.findAll(control.ctx, github.class.DocSyncInfo, { + _id: cud.objectId as Ref + }) + ).shift() + // We need to check if sync doc is already exists. + if (sdoc === undefined) { + // Created by non github integration + // We need to create the doc sync info + createSyncDoc(control, cud, tx, space, toApply) + } else { + cache.set(cud.objectId, sdoc) + // We need to create the doc sync info + updateSyncDoc(control, cud, space, sdoc, toApply) + } + } } function isDocSyncUpdateRequired (h: Hierarchy, coll: TxCUD): boolean {