UBERF-9724: Use updated accounts (#8452)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev
2025-04-14 09:36:20 +07:00
committed by GitHub
parent 143f0bc7b6
commit 5ca2a73fba
28 changed files with 1506 additions and 1153 deletions
+264 -208
View File
@@ -3,37 +3,39 @@
//
//
/* eslint-disable @typescript-eslint/no-unused-vars */
import { getClient as getAccountClient } from '@hcengineering/account-client'
import chunter from '@hcengineering/chunter'
import core, {
PersonId,
BrandingMap,
buildSocialIdString,
Client,
ClientConnectEvent,
DocumentUpdate,
isActiveMode,
isDeletingMode,
MeasureContext,
PersonId,
RateLimiter,
SocialIdType,
systemAccountUuid,
TimeRateLimiter,
TxOperations,
systemAccountUuid,
WorkspaceInfoWithStatus,
WorkspaceUuid,
WorkspaceInfoWithStatus
type PersonUuid,
type Ref
} from '@hcengineering/core'
import github, { GithubAuthentication, makeQuery, type GithubIntegration } from '@hcengineering/github'
import { getMongoClient, MongoClientReference } from '@hcengineering/mongo'
import { setMetadata } from '@hcengineering/platform'
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import serverToken, { generateToken } from '@hcengineering/server-token'
import { getClient as getAccountClient } from '@hcengineering/account-client'
import tracker from '@hcengineering/tracker'
import { Installation, type InstallationCreatedEvent, type InstallationUnsuspendEvent } from '@octokit/webhooks-types'
import { Collection } from 'mongodb'
import { App, Octokit } from 'octokit'
import { Analytics } from '@hcengineering/analytics'
import { SplitLogger } from '@hcengineering/analytics-service'
import contact, { Person } from '@hcengineering/contact'
import contact, { type Employee, type SocialIdentityRef } from '@hcengineering/contact'
import { type StorageAdapter } from '@hcengineering/server-core'
import { join } from 'path'
import { createPlatformClient } from './client'
@@ -57,7 +59,7 @@ export interface InstallationRecord {
}
export class PlatformWorker {
private readonly clients: Map<string, GithubWorker> = new Map<string, GithubWorker>()
private readonly clients = new Map<WorkspaceUuid, GithubWorker>()
storageAdapter!: StorageAdapter
@@ -65,16 +67,12 @@ export class PlatformWorker {
integrations: GithubIntegrationRecord[] = []
mongoRef!: MongoClientReference
integrationCollection!: Collection<GithubIntegrationRecord>
periodicTimer: any
periodicSyncPromise: Promise<void> | undefined
canceled = false
userManager!: UserManager
userManager: UserManager = new UserManager()
rateLimits = new Map<string, TimeRateLimiter>()
@@ -98,14 +96,6 @@ export class PlatformWorker {
}
public async initStorage (): Promise<void> {
this.mongoRef = getMongoClient(config.MongoURL)
const mongoClient = await this.mongoRef.getClient()
const db = mongoClient.db(config.ConfigurationDB)
this.integrationCollection = db.collection<GithubIntegrationRecord>('installations')
this.userManager = new UserManager(db.collection<GithubUserRecord>('users'))
const storageConfig = storageConfigFromEnv()
this.storageAdapter = buildStorageFromConfig(storageConfig)
}
@@ -120,11 +110,30 @@ export class PlatformWorker {
)
this.clients.clear()
await this.storageAdapter.close()
this.mongoRef.close()
}
async init (ctx: MeasureContext): Promise<void> {
this.integrations = await this.integrationCollection.find({}).toArray()
const sysToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'github' })
const accountsClient = getAccountClient(config.AccountsURL, sysToken)
const allIntegrations = await accountsClient.listIntegrations({ kind: 'github' })
this.integrations = []
for (const i of allIntegrations) {
if (i.workspaceUuid == null) {
continue
}
const installationId = i.data?.installationId
if (installationId !== undefined) {
this.integrations.push({
accountId: i.socialId,
workspace: i.workspaceUuid,
installationId
})
}
}
await this.queryInstallations(ctx)
for (const integr of [...this.integrations]) {
@@ -134,7 +143,11 @@ export class PlatformWorker {
installationId: integr.installationId,
workspace: integr.workspace
})
await this.integrationCollection.deleteOne({ installationId: integr.installationId })
await accountsClient.deleteIntegration({
kind: 'github',
workspaceUuid: integr.workspace,
socialId: integr.accountId
})
this.integrations = this.integrations.filter((it) => it.installationId !== integr.installationId)
}
}
@@ -153,11 +166,11 @@ export class PlatformWorker {
async performPeriodicSync (): Promise<void> {
// Sync authorized users information details.
const workspaces = await this.findUsersWorkspaces()
for (const [workspace, users] of workspaces) {
const workspaces = await this.getWorkspaces()
for (const workspace of workspaces) {
const worker = this.clients.get(workspace)
if (worker !== undefined) {
await this.ctx.with('syncUsers', {}, (ctx) => worker.syncUserData(ctx, users))
await this.ctx.with('syncUsers', {}, (ctx) => worker.syncUserData(ctx))
}
}
this.periodicSyncPromise = undefined
@@ -190,38 +203,19 @@ export class PlatformWorker {
}
}
private async findUsersWorkspaces (): Promise<Map<string, GithubUserRecord[]>> {
const i = this.userManager.getAllUsers()
const workspaces = new Map<string, GithubUserRecord[]>()
while (await i.hasNext()) {
const userInfo = await i.next()
if (userInfo !== null) {
for (const ws of Object.keys(userInfo.accounts ?? {})) {
if (this.integrations.find((it) => it.workspace === ws) === undefined) {
// No workspace integration found, let's check workspace.
workspaces.set(ws, [...(workspaces.get(ws) ?? []), userInfo])
}
}
}
}
await i.close()
return workspaces
}
public async getUsers (workspace: string): Promise<GithubUserRecord[]> {
return await this.userManager.getUsers(workspace)
}
public async getUser (login: string): Promise<GithubUserRecord | undefined> {
return await this.userManager.getAccount(login)
}
async mapInstallation (
ctx: MeasureContext,
workspace: string,
workspace: WorkspaceUuid,
installationId: number,
accountId: PersonId
): Promise<void> {
const sysToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'github' })
const accountsClient = getAccountClient(config.AccountsURL, sysToken)
const oldInstallation = this.integrations.find((it) => it.installationId === installationId)
if (oldInstallation != null) {
ctx.info('update integration', { workspace, installationId, accountId })
@@ -231,10 +225,19 @@ export class PlatformWorker {
//
const oldWorkspace = oldInstallation.workspace
await this.integrationCollection.updateOne(
{ installationId: oldInstallation.installationId },
{ $set: { workspace } }
)
await accountsClient.createIntegration({
kind: 'github',
workspaceUuid: workspace,
socialId: accountId,
data: { installationId: oldInstallation.installationId }
})
await accountsClient.deleteIntegration({
kind: 'github',
workspaceUuid: oldWorkspace,
socialId: accountId
})
oldInstallation.workspace = workspace
const oldWorker = this.clients.get(oldWorkspace) as GithubWorker
@@ -244,7 +247,7 @@ export class PlatformWorker {
} else {
let client: Client | undefined
try {
;({ client } = await createPlatformClient(oldWorkspace as WorkspaceUuid, 30000)) // TODO: FIXME
;({ client } = await createPlatformClient(oldWorkspace, 30000))
await this.removeInstallationFromWorkspace(oldWorker, installationId)
await client.close()
} catch (err: any) {
@@ -270,7 +273,13 @@ export class PlatformWorker {
ctx.info('add integration', { workspace, installationId, accountId })
await ctx.with('add integration', { workspace, installationId, accountId }, async (ctx) => {
await this.integrationCollection.insertOne(record)
await accountsClient.createIntegration({
kind: 'github',
workspaceUuid: record.workspace,
socialId: record.accountId,
data: { installationId: record.installationId }
})
this.integrations.push(record)
})
// We need to query installations to be sure we have it, in case event is delayed or not received.
@@ -313,10 +322,10 @@ export class PlatformWorker {
}
async requestGithubAccessToken (payload: {
workspace: string
workspace: WorkspaceUuid
code: string
state: string
accountId: PersonId
accountId: PersonId // Primary social Id
}): Promise<void> {
try {
const uri =
@@ -346,6 +355,7 @@ export class PlatformWorker {
const user = await okit.rest.users.getAuthenticated()
const nowTime = Date.now() / 1000
const dta: GithubUserRecord = {
account: payload.accountId,
_id: user.data.login,
token: resultJson.access_token,
code: null,
@@ -363,7 +373,7 @@ export class PlatformWorker {
if (existingUser == null) {
await this.userManager.insertUser(dta)
} else {
dta.accounts = { ...existingUser.accounts, [payload.workspace]: payload.accountId }
dta.accounts = { ...existingUser.accounts, [payload.workspace]: payload.accountId } // Put primary socialId for now.
await this.userManager.updateUser(dta)
}
@@ -388,150 +398,189 @@ export class PlatformWorker {
}
private async updateAccountAuthRecord (
payload: { workspace: string, accountId: PersonId },
payload: { workspace: WorkspaceUuid, accountId: PersonId },
update: DocumentUpdate<GithubAuthentication>,
dta: GithubUserRecord | undefined,
revoke: boolean
): Promise<void> {
// TODO: FIXME
throw new Error('Not implemented')
// try {
// let platformClient: Client | undefined
// let shouldClose = false
// try {
// platformClient = this.clients.get(payload.workspace)?.client
// if (platformClient === undefined) {
// shouldClose = true
// ;({ client: platformClient } = await createPlatformClient(payload.workspace, 30000))
// }
// const client = new TxOperations(platformClient, payload.accountId)
try {
let platformClient: Client | undefined
let shouldClose = false
try {
platformClient = this.clients.get(payload.workspace)?.client
if (platformClient === undefined) {
shouldClose = true
;({ client: platformClient } = await createPlatformClient(payload.workspace, 30000))
}
const client = new TxOperations(platformClient, payload.accountId)
// let personAuths = await client.findAll(github.class.GithubAuthentication, {
// attachedTo: payload.accountId
// })
// if (personAuths.length > 1) {
// for (const auth of personAuths.slice(1)) {
// await client.remove(auth)
// }
// personAuths.length = 1
// }
let personAuths = await client.findAll(github.class.GithubAuthentication, {
attachedTo: payload.accountId
})
if (personAuths.length > 1) {
for (const auth of personAuths.slice(1)) {
await client.remove(auth)
}
personAuths.length = 1
}
// if (revoke) {
// for (const personAuth of personAuths) {
// await client.remove(personAuth, Date.now(), payload.accountId)
// }
// } else {
// if (personAuths.length > 0) {
// await client.update<GithubAuthentication>(personAuths[0], update, false, Date.now(), payload.accountId)
// } else if (dta !== undefined) {
// const authId = await client.createDoc<GithubAuthentication>(
// github.class.GithubAuthentication,
// core.space.Workspace,
// {
// error: null,
// authRequestTime: Date.now(),
// createdAt: new Date(),
// followers: 0,
// following: 0,
// nodeId: '',
// updatedAt: new Date(),
// url: '',
// repositories: 0,
// organizations: { totalCount: 0, nodes: [] },
// closedIssues: 0,
// openIssues: 0,
// mergedPRs: 0,
// openPRs: 0,
// closedPRs: 0,
// repositoryDiscussions: 0,
// starredRepositories: 0,
// ...update,
// attachedTo: payload.accountId,
// login: dta._id
// },
// undefined,
// undefined,
// payload.accountId
// )
if (revoke) {
for (const personAuth of personAuths) {
await client.remove(personAuth, Date.now(), payload.accountId)
}
// personAuths = await client.findAll(github.class.GithubAuthentication, {
// _id: authId
// })
// }
// }
// TODO: Do we need to remove social ids?
} else {
if (personAuths.length > 0) {
await client.update<GithubAuthentication>(personAuths[0], update, false, Date.now(), payload.accountId)
} else if (dta !== undefined) {
const authId = await client.createDoc<GithubAuthentication>(
github.class.GithubAuthentication,
core.space.Workspace,
{
error: null,
authRequestTime: Date.now(),
createdAt: new Date(),
followers: 0,
following: 0,
nodeId: '',
updatedAt: new Date(),
url: '',
repositories: 0,
organizations: { totalCount: 0, nodes: [] },
closedIssues: 0,
openIssues: 0,
mergedPRs: 0,
openPRs: 0,
closedPRs: 0,
repositoryDiscussions: 0,
starredRepositories: 0,
...update,
attachedTo: payload.accountId,
login: dta._id
},
undefined,
undefined,
payload.accountId
)
// // We need to re-bind previously created github:login account to a proper person.
// const account = client.getModel().getObject(payload.accountId) as PersonAccount
// const person = (await client.findOne(contact.class.Person, { _id: account.person })) as Person
// if (person !== undefined) {
// if (!revoke) {
// const personSpace = await client.findOne(contact.class.PersonSpace, { person: person._id })
// if (personSpace !== undefined) {
// await createNotification(client, person, {
// user: account._id,
// space: personSpace._id,
// message: github.string.AuthenticatedWithGithub,
// props: {
// login: update.login
// }
// })
// }
personAuths = await client.findAll(github.class.GithubAuthentication, {
_id: authId
})
}
}
// const githubAccount = client.getModel().getAccountByEmail('github:' + update.login) as PersonAccount
// if (githubAccount !== undefined && githubAccount.person !== account.person) {
// const dummyPerson = githubAccount.person
// // To add activity entry to dummy person.
// await client.update(githubAccount, { person: account.person }, false, Date.now(), payload.accountId)
const account = await client.findOne(contact.class.SocialIdentity, {
_id: payload.accountId as SocialIdentityRef
})
const person =
account !== undefined
? await client.findOne(contact.mixin.Employee, { _id: account?.attachedTo as Ref<Employee> })
: undefined
if (person !== undefined) {
if (!revoke) {
const personSpace = await client.findOne(contact.class.PersonSpace, { person: person._id })
if (personSpace !== undefined && person.personUuid !== undefined) {
await createNotification(client, person, {
user: person.personUuid,
space: personSpace._id,
message: github.string.AuthenticatedWithGithub,
props: {
login: update.login
}
})
}
// const dPerson = (await client.findOne(contact.class.Person, { _id: dummyPerson })) as Person
// if (person !== undefined && dPerson !== undefined) {
// const personSpace = await client.findOne(contact.class.PersonSpace, { person: person._id })
// if (personSpace !== undefined) {
// await createNotification(client, dPerson, {
// user: githubAccount._id,
// space: personSpace._id,
// message: github.string.AuthenticatedWithGithubEmployee,
// props: {
// login: update.login
// }
// })
// }
// }
// }
// } else {
// const personSpace = await client.findOne(contact.class.PersonSpace, { person: person._id })
// if (personSpace !== undefined) {
// await createNotification(client, person, {
// user: account._id,
// space: personSpace._id,
// message: github.string.AuthenticationRevokedGithub,
// props: {
// login: update.login
// }
// })
// }
// }
// }
if (dta?._id !== undefined) {
const sysToken = generateToken(systemAccountUuid, payload.workspace, {
service: 'github'
})
const userToken = generateToken(person.personUuid as PersonUuid, payload.workspace, {
service: 'github'
})
const sysAccountClient = getAccountClient(config.AccountsURL, sysToken)
const userAccountClient = getAccountClient(config.AccountsURL, userToken)
// if (dta !== undefined && personAuths.length === 1) {
// try {
// await syncUser(this.ctx, dta, personAuths[0], client, payload.accountId)
// } catch (err: any) {
// if (err.response?.data?.message === 'Bad credentials') {
// await this.revokeUserAuth(dta)
// } else {
// this.ctx.error(`Failed to sync user ${dta._id}`, { error: errorToObj(err) })
// }
// }
// }
// } finally {
// if (shouldClose) {
// await platformClient?.close()
// }
// }
// } catch (err: any) {
// Analytics.handleError(err)
// }
const ids = await userAccountClient.getSocialIds()
let githubSocialId: PersonId | undefined = ids.find(
(it) => it.type === SocialIdType.GITHUB && it.value === dta?._id
)?._id
// We need to assign socialId to person in global account if missing and get it to match if exists.
if (githubSocialId === undefined) {
// We need to create a new social id for this account.
githubSocialId = await sysAccountClient.addSocialIdToPerson(
person.personUuid as PersonUuid,
SocialIdType.GITHUB,
dta?._id ?? '',
true
)
}
const socialIdentity = await client.findOne(contact.class.SocialIdentity, {
_id: githubSocialId as SocialIdentityRef
})
if (socialIdentity === undefined) {
// We need to create a new social id for this account.
// We need to create social id github account
await client.addCollection(
contact.class.SocialIdentity,
contact.space.Contacts,
person._id,
contact.class.Person,
'socialIds',
{
type: SocialIdType.GITHUB,
value: dta._id,
key: buildSocialIdString({
type: SocialIdType.GITHUB,
value: dta._id
}),
verifiedOn: Date.now()
},
githubSocialId as SocialIdentityRef
)
}
}
} else {
const personSpace = await client.findOne(contact.class.PersonSpace, { person: person._id })
if (personSpace !== undefined && person.personUuid !== undefined) {
await createNotification(client, person, {
user: person.personUuid,
space: personSpace._id,
message: github.string.AuthenticationRevokedGithub,
props: {
login: update.login
}
})
}
}
}
if (dta !== undefined && personAuths.length === 1) {
try {
await syncUser(this.ctx, dta, personAuths[0], client, payload.accountId)
} catch (err: any) {
if (err.response?.data?.message === 'Bad credentials') {
await this.revokeUserAuth(dta)
} else {
this.ctx.error(`Failed to sync user ${dta._id}`, { error: errorToObj(err) })
}
}
}
} catch (err: any) {
this.ctx.error('error workspace update', { err })
Analytics.handleError(err)
} finally {
if (shouldClose) {
await platformClient?.close()
}
}
} catch (err: any) {
Analytics.handleError(err)
}
}
async checkRefreshToken (auth: GithubUserRecord, force: boolean = false): Promise<void> {
@@ -584,7 +633,7 @@ export class PlatformWorker {
return await this.userManager.getAccount(login)
}
async getAccountByRef (workspace: string, ref: PersonId): Promise<GithubUserRecord | undefined> {
async getAccountByRef (workspace: WorkspaceUuid, ref: PersonId): Promise<GithubUserRecord | undefined> {
return await this.userManager.getAccountByRef(workspace, ref)
}
@@ -668,7 +717,7 @@ export class PlatformWorker {
integeration.enabled = enabled
}
await worker.syncUserData(this.ctx, await this.getUsers(worker.workspace.uuid))
await worker.syncUserData(this.ctx)
await worker.reloadRepositories(install.id)
worker.triggerUpdate()
@@ -705,23 +754,29 @@ export class PlatformWorker {
// No worker
}
this.integrations = this.integrations.filter((it) => it.installationId !== installId)
await this.integrationCollection.deleteOne({ installationId: installId })
if (interg !== undefined) {
const sysToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'github' })
const sysAccountClient = getAccountClient(config.AccountsURL, sysToken)
await sysAccountClient.deleteIntegration({
kind: 'github',
workspaceUuid: interg.workspace,
socialId: interg.accountId
})
}
this.triggerCheckWorkspaces()
}
async getWorkspaces (): Promise<WorkspaceUuid[]> {
const workspaces = new Set(this.integrations.map((it) => it.workspace as WorkspaceUuid)) // TODO: FIXME
return Array.from(workspaces)
return this.integrations.map((it) => it.workspace)
}
async checkWorkspaceIsActive (
token: string,
workspace: string
workspace: WorkspaceUuid
): Promise<{ workspaceInfo: WorkspaceInfoWithStatus | undefined, needRecheck: boolean }> {
let workspaceInfo: WorkspaceInfoWithStatus | undefined
try {
workspaceInfo = await getAccountClient(token).getWorkspaceInfo(true)
workspaceInfo = await getAccountClient(config.AccountsURL, token).getWorkspaceInfo(false)
} catch (err: any) {
this.ctx.error('Workspace not found:', { workspace })
return { workspaceInfo: undefined, needRecheck: false }
@@ -752,11 +807,8 @@ export class PlatformWorker {
this.ctx.info('************************* Check workspaces ************************* ', {
workspaces: this.clients.size
})
let workspaces = await this.getWorkspaces()
if (process.env.GITHUB_USE_WS !== undefined) {
workspaces = [process.env.GITHUB_USE_WS as WorkspaceUuid]
}
const toDelete = new Set<string>(this.clients.keys())
const workspaces = await this.getWorkspaces()
const toDelete = new Set<WorkspaceUuid>(this.clients.keys())
const rateLimiter = new RateLimiter(5)
let errors = 0
@@ -1094,8 +1146,7 @@ export class PlatformWorker {
payload.installation.html_url
)
const doSyncUsers = async (worker: GithubWorker): Promise<void> => {
const users = await this.getUsers(worker.workspace.uuid)
await worker.syncUserData(this.ctx, users)
await worker.syncUserData(this.ctx)
}
catchEventError(doSyncUsers(worker), payload.action, name, id, payload.installation.html_url)
})
@@ -1141,7 +1192,12 @@ export class PlatformWorker {
public async revokeUserAuth (record: GithubUserRecord): Promise<void> {
for (const [ws, acc] of Object.entries(record.accounts)) {
await this.updateAccountAuthRecord({ workspace: ws, accountId: acc }, { login: record._id }, undefined, true)
await this.updateAccountAuthRecord(
{ workspace: ws as WorkspaceUuid, accountId: acc },
{ login: record._id },
undefined,
true
)
}
}