UBERF-9137: Fix Support for suspended installations (#7667)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev
2025-01-15 23:23:23 +07:00
committed by GitHub
parent 0c6feb646e
commit c8dde57123
24 changed files with 435 additions and 336 deletions
+2 -1
View File
@@ -44,7 +44,8 @@ export async function createPlatformClient (
const connection = await (
await clientResources()
).function.GetClient(token, endpoint, {
onConnect: reconnect
onConnect: reconnect,
useGlobalRPCHandler: true
})
return { client: connection, endpoint }
+14 -6
View File
@@ -31,15 +31,23 @@ Analytics.setTag('application', 'github-service')
let doOnClose: () => Promise<void> = async () => {}
void start(metricsContext, loadBrandingMap(config.BrandingPath)).then((r) => {
doOnClose = r
})
void start(metricsContext, loadBrandingMap(config.BrandingPath))
.then((r) => {
doOnClose = r
})
.catch((err) => {
metricsContext.error('Error', { error: err })
})
const onClose = (): void => {
metricsContext.info('Closed')
void doOnClose().then((r) => {
process.exit(0)
})
void doOnClose()
.then((r) => {
process.exit(0)
})
.catch((err) => {
metricsContext.error('Error', { error: err })
})
}
process.on('uncaughtException', (e) => {
+35 -19
View File
@@ -52,6 +52,7 @@ export interface InstallationRecord {
repositories?: InstallationCreatedEvent['repositories'] | InstallationUnsuspendEvent['repositories']
type: 'Bot' | 'User' | 'Organization'
octokit: Octokit
suspended: boolean
}
export class PlatformWorker {
@@ -593,7 +594,8 @@ export class PlatformWorker {
login: tinst.account.login,
loginNodeId: tinst.account.node_id,
type: tinst.account?.type ?? 'User',
installationName: `${tinst.account?.html_url ?? ''}`
installationName: `${tinst.account?.html_url ?? ''}`,
suspended: install.data.suspended_at != null
}
this.updateInstallationRecord(installationId, val)
}
@@ -609,6 +611,7 @@ export class PlatformWorker {
current.loginNodeId = val.loginNodeId
current.type = val.type
current.installationName = val.installationName
current.suspended = val.suspended
if (val.repositories !== undefined) {
current.repositories = val.repositories
}
@@ -625,7 +628,8 @@ export class PlatformWorker {
login: tinst.account.login,
loginNodeId: tinst.account.node_id,
type: tinst.account?.type ?? 'User',
installationName: `${tinst.account?.html_url ?? ''}`
installationName: `${tinst.account?.html_url ?? ''}`,
suspended: install.installation.suspended_at != null
}
this.updateInstallationRecord(install.installation.id, val)
ctx.info('Found installation', {
@@ -650,11 +654,17 @@ export class PlatformWorker {
type: install.account?.type ?? 'User',
loginNodeId: install.account.node_id,
installationName: iName,
repositories
repositories,
suspended: !enabled
})
const worker = this.getWorker(install.id)
if (worker !== undefined) {
const integeration = worker.integrations.get(install.id)
if (integeration !== undefined) {
integeration.enabled = enabled
}
await worker.syncUserData(this.ctx, await this.getUsers(worker.workspace.name))
await worker.reloadRepositories(install.id)
@@ -662,13 +672,6 @@ export class PlatformWorker {
worker.triggerSync()
}
// Need to inform workspace
const integeration = this.integrations.find((it) => it.installationId === install.id)
if (integeration !== undefined) {
const worker = this.clients.get(integeration.workspace) as GithubWorker
worker?.triggerUpdate()
}
// Check if no workspace was available
this.triggerCheckWorkspaces()
}
@@ -810,17 +813,28 @@ export class PlatformWorker {
this.storageAdapter,
(workspace, event) => {
if (event === ClientConnectEvent.Refresh || event === ClientConnectEvent.Upgraded) {
void this.clients.get(workspace)?.refreshClient(event === ClientConnectEvent.Upgraded)
void this.clients
.get(workspace)
?.refreshClient(event === ClientConnectEvent.Upgraded)
?.catch((err) => {
workerCtx.error('Failed to refresh', { error: err })
})
}
if (initialized) {
// We need to check if workspace is inactive
void this.checkWorkspaceIsActive(token, workspace).then((res) => {
if (res === undefined) {
this.ctx.warn('Workspace is inactive, removing from clients list.', { workspace })
this.clients.delete(workspace)
void worker?.close()
}
})
void this.checkWorkspaceIsActive(token, workspace)
.then((res) => {
if (res === undefined) {
this.ctx.warn('Workspace is inactive, removing from clients list.', { workspace })
this.clients.delete(workspace)
void worker?.close().catch((err) => {
this.ctx.error('Failed to close workspace', { workspace, error: err })
})
}
})
.catch((err) => {
this.ctx.error('Failed to check workspace is active', { workspace, error: err })
})
}
}
)
@@ -879,7 +893,9 @@ export class PlatformWorker {
try {
this.ctx.info('workspace removed from tracking list', { workspace: deleted })
this.clients.delete(deleted)
void ws.close()
void ws.close().catch((err) => {
this.ctx.error('Error', { error: err })
})
} catch (err: any) {
Analytics.handleError(err)
errors++
@@ -207,9 +207,9 @@ export abstract class IssueSyncManagerBase {
})
if (syncData !== undefined) {
const milestone = (
await this.provider.liveQuery.queryFind<GithubMilestone>(github.mixin.GithubMilestone, {})
).find((it) => it.projectNodeId === projectId)
const milestone = await this.client.findOne<GithubMilestone>(github.mixin.GithubMilestone, {
projectNodeId: projectId
})
const target: IssueSyncTarget | undefined =
milestone !== undefined
@@ -264,11 +264,6 @@ export abstract class IssueSyncManagerBase {
let structure = integration.projectStructure.get(target.target._id)
const repositories = await this.provider.liveQuery.queryFind<GithubIntegrationRepository>(
github.class.GithubIntegrationRepository,
{}
)
for (const f of target.prjData.fieldValues?.nodes ?? []) {
if (!('id' in f)) {
continue
@@ -281,8 +276,13 @@ export abstract class IssueSyncManagerBase {
needProjectRefresh = true
}
}
if (needProjectRefresh) {
const repo = repositories.find((it) => it._id === syncData.repository)
if (needProjectRefresh && syncData.repository != null) {
const repo = await this.provider.liveQuery.findOne<GithubIntegrationRepository>(
github.class.GithubIntegrationRepository,
{
_id: syncData.repository
}
)
if (repo !== undefined) {
await this.provider.handleEvent(github.class.GithubIntegration, integration.installationId, repo, {})
@@ -1153,9 +1153,9 @@ export abstract class IssueSyncManagerBase {
if (existingIssue !== undefined) {
// Select a milestone project
if (existingIssue.milestone != null) {
const milestone = (
await this.provider.liveQuery.queryFind<GithubMilestone>(github.mixin.GithubMilestone, {})
).find((it) => it._id === existingIssue.milestone)
const milestone = await this.provider.liveQuery.findOne<GithubMilestone>(github.mixin.GithubMilestone, {
_id: existingIssue.milestone as Ref<GithubMilestone>
})
if (milestone === undefined) {
return
}
+8 -14
View File
@@ -95,13 +95,15 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
if (projectV2Event) {
const projectV2Event = event as ProjectsV2ItemEvent
const githubProjects = await this.provider.liveQuery.queryFind(github.mixin.GithubProject, {})
const githubProjects = await this.provider.liveQuery.findAll(github.mixin.GithubProject, {
archived: false
})
let prj = githubProjects.find((it) => it.projectNodeId === projectV2Event.projects_v2_item.project_node_id)
if (prj === undefined) {
// Checking for milestones
const m = (await this.provider.liveQuery.queryFind(github.mixin.GithubMilestone, {})).find(
(it) => it.projectNodeId === projectV2Event.projects_v2_item.project_node_id
)
const m = await this.provider.liveQuery.findOne(github.mixin.GithubMilestone, {
projectNodeId: projectV2Event.projects_v2_item.project_node_id
})
if (m !== undefined) {
prj = githubProjects.find((it) => it._id === m.space)
}
@@ -353,13 +355,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
}
if (info.repository == null) {
// No need to sync if component it not yet set
const repos = (await this.provider.getProjectRepositories(container.project._id))
.map((it) => it.name)
.join(', ')
this.ctx.error('Not syncing repository === null', {
url: info.url,
identifier: (existing as Issue).identifier,
repos
identifier: (existing as Issue).identifier
})
return { needSync: githubSyncVersion }
}
@@ -372,13 +370,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
if (info.external === undefined && existing !== undefined) {
const repository = await this.provider.getRepositoryById(info.repository)
if (repository === undefined) {
const repos = (await this.provider.getProjectRepositories(container.project._id))
.map((it) => it.name)
.join(', ')
this.ctx.error('Not syncing repository === undefined', {
url: info.url,
identifier: (existing as Issue).identifier,
repos
identifier: (existing as Issue).identifier
})
return { needSync: githubSyncVersion }
}
@@ -446,9 +446,9 @@ export class ProjectsSyncManager implements DocSyncManager {
if (syncConfig.SupportMilestones && integration.type === 'Organization') {
// Check project milestones and sync their structure as well.
const milestones = (await this.provider.liveQuery.queryFind(github.mixin.GithubMilestone, {})).filter(
(it) => it.space === prj._id
)
const milestones = await this.provider.liveQuery.findAll(github.mixin.GithubMilestone, {
space: prj._id
})
for (const m of milestones) {
if (this.provider.isClosing()) {
break
@@ -100,13 +100,15 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
if (projectV2Event) {
const projectV2Event = _event as ProjectsV2ItemEvent
const githubProjects = await this.provider.liveQuery.queryFind(github.mixin.GithubProject, {})
const githubProjects = await this.provider.liveQuery.findAll(github.mixin.GithubProject, {
archived: false
})
let prj = githubProjects.find((it) => it.projectNodeId === projectV2Event.projects_v2_item.project_node_id)
if (prj === undefined) {
// Checking for milestones
const m = (await this.provider.liveQuery.queryFind(github.mixin.GithubMilestone, {})).find(
(it) => it.projectNodeId === projectV2Event.projects_v2_item.project_node_id
)
const m = await this.provider.liveQuery.findOne(github.mixin.GithubMilestone, {
projectNodeId: projectV2Event.projects_v2_item.project_node_id
})
if (m !== undefined) {
prj = githubProjects.find((it) => it._id === m.space)
}
@@ -48,9 +48,9 @@ export class RepositorySyncMapper implements DocSyncManager {
if (repositories !== undefined) {
// We have a list of repositories, so we could create them if they are missing.
// Need to find all repositories, not only active, so passed repositories are not work.
const allRepositories = (
await this.provider.liveQuery.queryFind(github.class.GithubIntegrationRepository, {})
).filter((it) => it.attachedTo === integration.integration._id)
const allRepositories = await this.provider.liveQuery.findAll(github.class.GithubIntegrationRepository, {
attachedTo: integration.integration._id
})
const allRepos: GithubIntegrationRepository[] = [...allRepositories]
for (const repository of repositories) {
@@ -259,9 +259,9 @@ export class RepositorySyncMapper implements DocSyncManager {
const iterable = this.app.eachRepository.iterator({ installationId: integration.installationId })
// Need to find all repositories, not only active, so passed repositories are not work.
const allRepositories = (
await this.provider.liveQuery.queryFind(github.class.GithubIntegrationRepository, {})
).filter((it) => it.attachedTo === integration.integration._id)
const allRepositories = await this.provider.liveQuery.findAll(github.class.GithubIntegrationRepository, {
attachedTo: integration.integration._id
})
let allRepos: GithubIntegrationRepository[] = [...allRepositories]
-2
View File
@@ -126,8 +126,6 @@ export interface IntegrationManager {
isPlatformUser: (account: Ref<PersonAccount>) => Promise<boolean>
getProjectRepositories: (space: Ref<Space>) => Promise<GithubIntegrationRepository[]>
getRepositoryById: (ref?: Ref<GithubIntegrationRepository> | null) => Promise<GithubIntegrationRepository | undefined>
isClosing: () => boolean
+77 -69
View File
@@ -44,7 +44,6 @@ import github, {
GithubIntegration,
GithubIntegrationRepository,
GithubIssue,
GithubMilestone,
GithubProject,
GithubUserInfo,
githubId
@@ -198,11 +197,9 @@ export class GithubWorker implements IntegrationManager {
}
async getContainer (space: Ref<Space>): Promise<ContainerFocus | undefined> {
const project = (
await this.liveQuery.queryFind<GithubProject>(github.mixin.GithubProject, {
_id: space as Ref<GithubProject>
})
).shift()
const project = await this.liveQuery.findOne<GithubProject>(github.mixin.GithubProject, {
_id: space as Ref<GithubProject>
})
if (project !== undefined) {
for (const v of this.integrations.values()) {
if (v.octokit === undefined) {
@@ -219,21 +216,13 @@ export class GithubWorker implements IntegrationManager {
}
}
async getProjectRepositories (space: Ref<Space>): Promise<GithubIntegrationRepository[]> {
const repositories = await this.liveQuery.queryFind<GithubIntegrationRepository>(
github.class.GithubIntegrationRepository,
{}
)
return repositories.filter((it) => it.githubProject === space)
}
async getRepositoryById (
_id?: Ref<GithubIntegrationRepository> | null
): Promise<GithubIntegrationRepository | undefined> {
if (_id != null) {
return (
await this.liveQuery.queryFind<GithubIntegrationRepository>(github.class.GithubIntegrationRepository, { _id })
).shift()
return await this.liveQuery.findOne<GithubIntegrationRepository>(github.class.GithubIntegrationRepository, {
_id
})
}
}
@@ -289,7 +278,9 @@ export class GithubWorker implements IntegrationManager {
})
}
const account = await this.liveQuery.findOne(contact.class.PersonAccount, { email: `github:${userInfo.login}` })
const account = await this.client
.getModel()
.findOne(contact.class.PersonAccount, { email: `github:${userInfo.login}` })
if (account !== undefined) {
const person = await this.liveQuery.findOne(contact.class.Person, { _id: account.person })
// We need to be sure employee are exists.
@@ -330,7 +321,7 @@ export class GithubWorker implements IntegrationManager {
person,
role: AccountRole.User
})
const acc = await this.liveQuery.findOne(contact.class.PersonAccount, { _id: id })
const acc = await this.client.getModel().findOne(contact.class.PersonAccount, { _id: id })
return acc
}
}
@@ -420,7 +411,7 @@ export class GithubWorker implements IntegrationManager {
let person: Ref<Person> | undefined
// try to find by account.
if (userInfo.email != null && userInfo.email.trim().length > 0) {
const personAccount = await this.liveQuery.findOne(contact.class.PersonAccount, { email: userInfo.email })
const personAccount = await this.client.getModel().findOne(contact.class.PersonAccount, { email: userInfo.email })
person = personAccount?.person
}
@@ -472,7 +463,7 @@ export class GithubWorker implements IntegrationManager {
}
async getGithubLogin (container: IntegrationContainer, person: Ref<Person>): Promise<UserInfo | undefined> {
const accounts = await this.liveQuery.queryFind(contact.class.PersonAccount, {})
const accounts = this.client.getModel().findAllSync(contact.class.PersonAccount, {})
const acc = accounts.find((it) => it.person === person && it.email.startsWith('github:'))
if (acc === undefined) {
return // Nobody, will use system account.
@@ -527,7 +518,11 @@ export class GithubWorker implements IntegrationManager {
const ops = new TxOperations(this.client, accountRef)
await syncUser(ctx, record, userAuth, ops, accountRef)
} catch (err: any) {
await this.platform.revokeUserAuth(record)
try {
await this.platform.revokeUserAuth(record)
} catch (err: any) {
ctx.error(`Failed to revoke user ${record._id}`, err)
}
if (err.response?.data?.message !== 'Bad credentials') {
ctx.error(`Failed to sync user ${record._id}`, err)
Analytics.handleError(err)
@@ -551,7 +546,7 @@ export class GithubWorker implements IntegrationManager {
let record = await this.platform.getAccountByRef(this.workspace.name, account)
// const accountRef = this.accounts.find((it) => it._id === account)
const [accountRef] = await this.liveQuery.queryFind(contact.class.PersonAccount, { _id: account })
const [accountRef] = this.client.getModel().findAllSync(contact.class.PersonAccount, { _id: account })
if (record === undefined) {
if (accountRef !== undefined) {
const accounts = this._client.getModel().getAccountByPersonId(accountRef.person)
@@ -651,7 +646,7 @@ export class GithubWorker implements IntegrationManager {
async getProjectStatuses (type: Ref<ProjectType> | undefined): Promise<Status[]> {
if (type === undefined) return []
const statuses = await this.liveQuery.queryFind(core.class.Status, {})
const statuses = this.client.getModel().findAllSync(core.class.Status, {})
const projectType = await this.getProjectType(type)
@@ -663,7 +658,7 @@ export class GithubWorker implements IntegrationManager {
if (type === undefined) return []
const taskType = await this.getTaskType(type)
const statuses = await this.liveQuery.queryFind(core.class.Status, {})
const statuses = this.client.getModel().findAllSync(core.class.Status, {})
const allowedTypes = new Set(taskType?.statuses ?? [])
return statuses.filter((it) => allowedTypes.has(it._id))
@@ -743,37 +738,27 @@ export class GithubWorker implements IntegrationManager {
}
projects: GithubProject[] = []
milestones: GithubMilestone[] = []
async queryProjects (): Promise<void> {
await new Promise<void>((resolve) => {
this.liveQuery.query(github.mixin.GithubProject, {}, (res) => {
let needRefresh = false
if (!equalExceptKeys(this.projects, res, ['sequence', 'modifiedOn', 'modifiedBy'])) {
needRefresh = true
this.liveQuery.query(
github.mixin.GithubProject,
{
archived: false
},
(res) => {
let needRefresh = false
if (!equalExceptKeys(this.projects, res, ['sequence', 'modifiedOn', 'modifiedBy'])) {
needRefresh = true
}
this.projects = res
resolve()
if (needRefresh || this.projects.length !== res.length) {
// Do not trigger update if only sequence is changed.
this.triggerUpdate()
}
}
this.projects = res
resolve()
if (needRefresh || this.projects.length !== res.length) {
// Do not trigger update if only sequence is changed.
this.triggerUpdate()
}
})
})
await new Promise<void>((resolve) => {
this.liveQuery.query(github.mixin.GithubMilestone, {}, (res) => {
let needRefresh = false
if (!equalExceptKeys(this.milestones, res, ['modifiedOn', 'modifiedBy'])) {
needRefresh = true
}
this.milestones = res
resolve()
if (needRefresh || this.milestones.length !== res.length) {
// Do not trigger update if only sequence is changed.
this.triggerUpdate()
}
})
)
})
}
@@ -796,7 +781,7 @@ export class GithubWorker implements IntegrationManager {
loginNodeId: inst.loginNodeId ?? '',
type: inst.type ?? 'User',
installationName: inst?.installationName ?? '',
enabled: true,
enabled: !inst.suspended,
synchronized: new Set(),
projectStructure: new Map(),
syncLock: new Map()
@@ -866,7 +851,7 @@ export class GithubWorker implements IntegrationManager {
private async queryAccounts (): Promise<void> {
const updateAccounts = async (accounts: PersonAccount[]): Promise<void> => {
const persons = await this.liveQuery.queryFind(contact.class.Person, {
const persons = await this.liveQuery.findAll(contact.class.Person, {
_id: { $in: accounts.map((it) => it.person) }
})
const h = this.client.getHierarchy()
@@ -1112,7 +1097,9 @@ export class GithubWorker implements IntegrationManager {
if (this.updateRequests > 0) {
this.updateRequests = 0 // Just in case
await this.updateIntegrations()
void this.performFullSync()
void this.performFullSync().catch((err) => {
this.ctx.error('Failed to perform full sync', { error: err })
})
}
const { projects, repositories } = await this.collectActiveProjects()
@@ -1150,7 +1137,10 @@ export class GithubWorker implements IntegrationManager {
}
}
private async performSync (projects: GithubProject[], repositories: GithubIntegrationRepository[]): Promise<boolean> {
private async performSync (
projects: GithubProject[],
repositories: Pick<GithubIntegrationRepository, '_id'>[]
): Promise<boolean> {
const _projects = toIdMap(projects)
const _repositories = repositories.map((it) => it._id)
@@ -1207,12 +1197,21 @@ export class GithubWorker implements IntegrationManager {
const projects: GithubProject[] = []
const repositories: GithubIntegrationRepository[] = []
const allProjects = await this.liveQuery.queryFind<GithubProject>(github.mixin.GithubProject, { archived: false })
const allRepositories = await this.liveQuery.queryFind(github.class.GithubIntegrationRepository, { enabled: true })
const allProjects = await this.liveQuery.findAll<GithubProject>(github.mixin.GithubProject, {
archived: false
})
const allRepositories = (await this.liveQuery.findAll(github.class.GithubIntegrationRepository, {})).filter(
(it) => it.enabled
)
for (const it of Array.from(this.integrations.values())) {
if (it.enabled) {
const _projects = allProjects.filter((p) => !syncConfig.MainProject || it.projectStructure.has(p._id))
const _projects = []
for (const p of allProjects) {
if (p.integration === it.integration._id && (!syncConfig.MainProject || it.projectStructure.has(p._id))) {
_projects.push(p)
}
}
const prjIds = new Set(_projects.map((it) => it._id))
@@ -1237,13 +1236,13 @@ export class GithubWorker implements IntegrationManager {
const integration = await this._client.findOne(github.class.GithubIntegration, {
installationId: intgr.installationId
})
if (integration === undefined && this.installations.has(intgr.installationId)) {
const installation = this.installations.get(intgr.installationId) as InstallationRecord
const installation = this.installations.get(intgr.installationId) as InstallationRecord
if (integration === undefined && installation !== undefined) {
await this._client.createDoc(
github.class.GithubIntegration,
core.space.Configuration,
{
alive: true,
alive: !installation.suspended,
installationId: intgr.installationId,
clientId: config.ClientID,
name: installation.installationName,
@@ -1257,7 +1256,7 @@ export class GithubWorker implements IntegrationManager {
this.triggerUpdate()
} else if (integration !== undefined) {
await this._client.diffUpdate(integration, {
alive: true
alive: !installation.suspended
})
}
}
@@ -1455,9 +1454,7 @@ export class GithubWorker implements IntegrationManager {
'external sync',
{ installation: integration.installationName, workspace: this.workspace.name },
async () => {
if (!integration.enabled || integration.octokit === undefined) {
return
}
const enabled = integration.enabled && integration.octokit !== undefined
const upd: DocumentUpdate<GithubIntegration> = {}
if (integration.integration.byUser !== integration.login) {
@@ -1470,14 +1467,19 @@ export class GithubWorker implements IntegrationManager {
upd.clientId = config.ClientID
}
if (integration.integration.name !== integration.installationName || !integration.integration.alive) {
if (integration.integration.name !== integration.installationName) {
upd.name = integration.installationName
upd.alive = true
}
if (integration.integration.alive !== enabled) {
upd.alive = enabled
}
if (Object.keys(upd).length > 0) {
await this._client.diffUpdate(integration.integration, upd, Date.now(), integration.integration.createdBy)
this.triggerUpdate()
}
if (!enabled) {
return
}
const derivedClient = new TxOperations(this.client, core.account.System, true)
const { projects, repositories } = await this.collectActiveProjects()
@@ -1591,7 +1593,13 @@ export class GithubWorker implements IntegrationManager {
branding
)
ctx.info('Init worker', { workspace: workspace.workspaceUrl, workspaceId: workspace.workspaceName })
void worker.init()
void worker.init().catch((err) => {
ctx.error('Failed to init worker', {
workspace: workspace.workspaceUrl,
workspaceId: workspace.workspaceName,
error: err
})
})
return worker
} catch (err: any) {
ctx.error('timeout during to connect', { workspace, error: err })